From 91433e5b9cf3d82dbb766043819c84aa7155cb41 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 17 Oct 2022 09:04:25 +0200 Subject: [PATCH 01/50] =?UTF-8?q?Added=20**=20glob=20support=20for=20recur?= =?UTF-8?q?sive=20directory=20traversal=20to=20filesystem=20and=20=20S3.?= =?UTF-8?q?=20Implementation:=20*=20Updated=20parseGlob=20to=20not=20add?= =?UTF-8?q?=20=E2=80=98/=E2=80=98=20restriction=20when=20**=20is=20used.?= =?UTF-8?q?=20*=20Updated=20S3=20&=20filesystem=20to=20fetch=20files=20and?= =?UTF-8?q?=20not=20use=20regex=20match=20if=20glob=20is=20**.=20Testing:?= =?UTF-8?q?=20*=20Added=20a=20test=20for=20filesystem=20tests/queries/0=5F?= =?UTF-8?q?stateless/02459=5Fglob=5Ffor=5Frecursive=5Fdirectory=5Ftraversa?= =?UTF-8?q?l.sh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Common/parseGlobs.cpp | 10 ++++- src/Storages/StorageFile.cpp | 24 ++++++++--- src/Storages/StorageS3.cpp | 7 ++- ...or_recursive_directory_traversal.reference | 14 ++++++ ..._glob_for_recursive_directory_traversal.sh | 43 +++++++++++++++++++ 5 files changed, 89 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/02459_glob_for_recursive_directory_traversal.reference create mode 100755 tests/queries/0_stateless/02459_glob_for_recursive_directory_traversal.sh diff --git a/src/Common/parseGlobs.cpp b/src/Common/parseGlobs.cpp index 8e9195f9842..b7c9dbe3a70 100644 --- a/src/Common/parseGlobs.cpp +++ b/src/Common/parseGlobs.cpp @@ -90,17 +90,23 @@ std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_glob oss_for_replacing << escaped_with_globs.substr(current_index); std::string almost_res = oss_for_replacing.str(); WriteBufferFromOwnString buf_final_processing; + char previous = ' '; for (const auto & letter : almost_res) { - if ((letter == '?') || (letter == '*')) + if (previous == '*' && letter == '*') + { + buf_final_processing << "[^{}]"; + } + else if ((letter == '?') || (letter == '*')) { buf_final_processing << "[^/]"; /// '?' is any symbol except '/' if (letter == '?') continue; } - if ((letter == '.') || (letter == '{') || (letter == '}')) + else if ((letter == '.') || (letter == '{') || (letter == '}')) buf_final_processing << '\\'; buf_final_processing << letter; + previous = letter; } return buf_final_processing.str(); } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 29f2d0667d9..1f0704b37fa 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -81,7 +81,8 @@ void listFilesWithRegexpMatchingImpl( const std::string & path_for_ls, const std::string & for_match, size_t & total_bytes_to_read, - std::vector & result) + std::vector & result, + bool recursive = false) { const size_t first_glob = for_match.find_first_of("*?{"); @@ -89,10 +90,17 @@ void listFilesWithRegexpMatchingImpl( const std::string suffix_with_globs = for_match.substr(end_of_path_without_globs); /// begin with '/' const size_t next_slash = suffix_with_globs.find('/', 1); - auto regexp = makeRegexpPatternFromGlobs(suffix_with_globs.substr(0, next_slash)); + const std::string current_glob = suffix_with_globs.substr(0, next_slash); + auto regexp = makeRegexpPatternFromGlobs(current_glob); + re2::RE2 matcher(regexp); + bool skip_regex = current_glob == "/*" ? true : false; + if (!recursive) + recursive = current_glob == "/**" ; + const std::string prefix_without_globs = path_for_ls + for_match.substr(1, end_of_path_without_globs); + if (!fs::exists(prefix_without_globs)) return; @@ -107,15 +115,21 @@ void listFilesWithRegexpMatchingImpl( /// Condition is_directory means what kind of path is it in current iteration of ls if (!it->is_directory() && !looking_for_directory) { - if (re2::RE2::FullMatch(file_name, matcher)) + if (skip_regex || re2::RE2::FullMatch(file_name, matcher)) { total_bytes_to_read += it->file_size(); result.push_back(it->path().string()); } } - else if (it->is_directory() && looking_for_directory) + else if (it->is_directory()) { - if (re2::RE2::FullMatch(file_name, matcher)) + if (recursive) + { + listFilesWithRegexpMatchingImpl(fs::path(full_path).append(it->path().string()) / "" , + looking_for_directory ? suffix_with_globs.substr(next_slash) : current_glob , + total_bytes_to_read, result, recursive); + } + else if (looking_for_directory && re2::RE2::FullMatch(file_name, matcher)) { /// Recursion depth is limited by pattern. '*' works only for depth = 1, for depth = 2 pattern path is '*/*'. So we do not need additional check. listFilesWithRegexpMatchingImpl(fs::path(full_path) / "", suffix_with_globs.substr(next_slash), total_bytes_to_read, result); diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 675dd548088..a7e0bfb98ad 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -139,7 +139,9 @@ public: request.SetBucket(globbed_uri.bucket); request.SetPrefix(key_prefix); + matcher = std::make_unique(makeRegexpPatternFromGlobs(globbed_uri.key)); + recursive = globbed_uri.key == "/**" ? true : false; fillInternalBufferAssumeLocked(); } @@ -197,7 +199,7 @@ private: for (const auto & row : result_batch) { const String & key = row.GetKey(); - if (re2::RE2::FullMatch(key, *matcher)) + if (recursive || re2::RE2::FullMatch(key, *matcher)) { String path = fs::path(globbed_uri.bucket) / key; if (object_infos) @@ -224,7 +226,7 @@ private: for (const auto & row : result_batch) { String key = row.GetKey(); - if (re2::RE2::FullMatch(key, *matcher)) + if (recursive || re2::RE2::FullMatch(key, *matcher)) buffer.emplace_back(std::move(key)); } } @@ -252,6 +254,7 @@ private: Aws::S3::Model::ListObjectsV2Request request; Aws::S3::Model::ListObjectsV2Outcome outcome; std::unique_ptr matcher; + bool recursive{false}; bool is_finished{false}; std::unordered_map * object_infos; Strings * read_keys; diff --git a/tests/queries/0_stateless/02459_glob_for_recursive_directory_traversal.reference b/tests/queries/0_stateless/02459_glob_for_recursive_directory_traversal.reference new file mode 100644 index 00000000000..8e146946955 --- /dev/null +++ b/tests/queries/0_stateless/02459_glob_for_recursive_directory_traversal.reference @@ -0,0 +1,14 @@ +1 1 +2 2 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +3 3 +4 4 +3 3 +4 4 +5 5 +6 6 diff --git a/tests/queries/0_stateless/02459_glob_for_recursive_directory_traversal.sh b/tests/queries/0_stateless/02459_glob_for_recursive_directory_traversal.sh new file mode 100755 index 00000000000..2c3deda2328 --- /dev/null +++ b/tests/queries/0_stateless/02459_glob_for_recursive_directory_traversal.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +mkdir $user_files_path/d1 +touch $user_files_path/d1/text1.txt + +for i in {1..2} +do + echo $i$'\t'$i >> $user_files_path/d1/text1.txt +done + +mkdir $user_files_path/d1/d2 +touch $user_files_path/d1/d2/text2.txt +for i in {3..4} +do + echo $i$'\t'$i >> $user_files_path/d1/d2/text2.txt +done + +mkdir $user_files_path/d1/d2/d3 +touch $user_files_path/d1/d2/d3/text3.txt +for i in {5..6} +do + echo $i$'\t'$i >> $user_files_path/d1/d2/d3/text3.txt +done + +${CLICKHOUSE_CLIENT} -q "SELECT * from file ('d1/*','TSV', 'Index UInt8, Number UInt8')" | sort --numeric-sort +${CLICKHOUSE_CLIENT} -q "SELECT * from file ('d1/**','TSV', 'Index UInt8, Number UInt8')" | sort --numeric-sort +${CLICKHOUSE_CLIENT} -q "SELECT * from file ('d1/*/tex*','TSV', 'Index UInt8, Number UInt8')" | sort --numeric-sort +${CLICKHOUSE_CLIENT} -q "SELECT * from file ('d1/**/tex*','TSV', 'Index UInt8, Number UInt8')" | sort --numeric-sort + + +rm $user_files_path/d1/d2/d3/text3.txt +rmdir $user_files_path/d1/d2/d3 +rm $user_files_path/d1/d2/text2.txt +rmdir $user_files_path/d1/d2 +rm $user_files_path/d1/text1.txt +rmdir $user_files_path/d1 \ No newline at end of file From 3d1ed2969ae2d25038e4f6b1e974225f90e94625 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 17 Oct 2022 18:21:03 +0000 Subject: [PATCH 02/50] Fix possible LOGICAL_ERROR in binary ariphmetics monotonicity. Fix invalid evaluation of binary monotonic function in KeyCondition. --- src/Functions/FunctionBinaryArithmetic.h | 18 ++++++++++-------- src/Storages/MergeTree/KeyCondition.cpp | 2 ++ ...2461_mullable_pk_monotonicity_bug.reference | 4 ++++ .../02461_mullable_pk_monotonicity_bug.sql | 6 ++++++ 4 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference create mode 100644 tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index baa3c65537d..cf752bbcbf8 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -1782,21 +1782,24 @@ public: { ColumnsWithTypeAndName columns_with_constant = {{left.column->cloneResized(1), left.type, left.name}, - {right.type->createColumnConst(1, point), right.type, right.name}}; + {removeNullable(right.type)->createColumnConst(1, point), removeNullable(right.type), right.name}}; + /// This is a bit dangerous to call Base::executeImpl cause it ignores `use Default Implementation For XXX` flags. + /// It was possible to check monotonicity for nullable right type which result to exception. + /// Adding removeNullable above fixes the issue, but some other inconsistency may left. auto col = Base::executeImpl(columns_with_constant, return_type, 1); Field point_transformed; col->get(0, point_transformed); return point_transformed; }; - transform(left_point); - transform(right_point); + + bool is_positive_monotonicity = applyVisitor(FieldVisitorAccurateLess(), left_point, right_point) + == applyVisitor(FieldVisitorAccurateLess(), transform(left_point), transform(right_point)); if (name_view == "plus") { // Check if there is an overflow - if (applyVisitor(FieldVisitorAccurateLess(), left_point, right_point) - == applyVisitor(FieldVisitorAccurateLess(), transform(left_point), transform(right_point))) + if (is_positive_monotonicity) return {true, true, false, true}; else return {false, true, false, false}; @@ -1804,8 +1807,7 @@ public: else { // Check if there is an overflow - if (applyVisitor(FieldVisitorAccurateLess(), left_point, right_point) - != applyVisitor(FieldVisitorAccurateLess(), transform(left_point), transform(right_point))) + if (!is_positive_monotonicity) return {true, false, false, true}; else return {false, false, false, false}; @@ -1817,7 +1819,7 @@ public: auto transform = [&](const Field & point) { ColumnsWithTypeAndName columns_with_constant - = {{left.type->createColumnConst(1, point), left.type, left.name}, + = {{removeNullable(left.type)->createColumnConst(1, point), removeNullable(left.type), left.name}, {right.column->cloneResized(1), right.type, right.name}}; auto col = Base::executeImpl(columns_with_constant, return_type, 1); diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index d7c33c8663b..ec5454973c7 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -1407,6 +1407,7 @@ public: ColumnsWithTypeAndName new_arguments; new_arguments.reserve(arguments.size() + 1); new_arguments.push_back(const_arg); + new_arguments.front().column = new_arguments.front().column->cloneResized(input_rows_count); for (const auto & arg : arguments) new_arguments.push_back(arg); return func->prepare(new_arguments)->execute(new_arguments, result_type, input_rows_count, dry_run); @@ -1415,6 +1416,7 @@ public: { auto new_arguments = arguments; new_arguments.push_back(const_arg); + new_arguments.back().column = new_arguments.back().column->cloneResized(input_rows_count); return func->prepare(new_arguments)->execute(new_arguments, result_type, input_rows_count, dry_run); } else diff --git a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference new file mode 100644 index 00000000000..099b7d91c92 --- /dev/null +++ b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference @@ -0,0 +1,4 @@ +1 +2 +1 +2 diff --git a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql new file mode 100644 index 00000000000..56ea054cc47 --- /dev/null +++ b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql @@ -0,0 +1,6 @@ +create table tab (x Nullable(UInt8)) engine = MergeTree order by x settings allow_nullable_key = 1, index_granularity = 2; +insert into tab select number from numbers(4); + +set max_rows_to_read = 2; +SELECT x + 1 FROM tab where plus(x, 1) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1, x) <= 2 order by x; From 32fad205f0a43ed43c5d97499b92b751d3a1e19e Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 18 Oct 2022 14:08:26 +0200 Subject: [PATCH 03/50] Added documentation for file and s3 - 36316 Support ** glob for recursive directory traversal --- docs/en/sql-reference/table-functions/file.md | 17 +++++++++++++++++ docs/en/sql-reference/table-functions/s3.md | 12 ++++++++++++ 2 files changed, 29 insertions(+) diff --git a/docs/en/sql-reference/table-functions/file.md b/docs/en/sql-reference/table-functions/file.md index f40107aaaca..fc81e7cf649 100644 --- a/docs/en/sql-reference/table-functions/file.md +++ b/docs/en/sql-reference/table-functions/file.md @@ -81,6 +81,7 @@ Multiple path components can have globs. For being processed file must exist and - `?` — Substitutes any single character. - `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. - `{N..M}` — Substitutes any number in range from N to M including both borders. +- `**` - Fetches all files inside the folder recursively. Constructions with `{}` are similar to the [remote](remote.md) table function. @@ -119,6 +120,22 @@ Query the data from files named `file000`, `file001`, … , `file999`: SELECT count(*) FROM file('big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String, value UInt32'); ``` +**Example** + +Query the data from all files inside `big_dir` directory recursively: + +``` sql +SELECT count(*) FROM file('big_dir/**', 'CSV', 'name String, value UInt32'); +``` + +**Example** + +Query the data from all `file002` files from any folder inside `big_dir` directory recursively: + +``` sql +SELECT count(*) FROM file('big_dir/**/file002', 'CSV', 'name String, value UInt32'); +``` + ## Virtual Columns - `_path` — Path to the file. diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md index 545037665bb..545a89223bf 100644 --- a/docs/en/sql-reference/table-functions/s3.md +++ b/docs/en/sql-reference/table-functions/s3.md @@ -127,6 +127,18 @@ INSERT INTO FUNCTION s3('https://clickhouse-public-datasets.s3.amazonaws.com/my- SELECT name, value FROM existing_table; ``` +Glob ** can be used for recursive directory traversal. Consider the below example, it will fetch all files from `my-test-bucket-768` directory recursively: + +``` sql +SELECT * FROM s3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/**', 'CSV', 'name String, value UInt32', 'gzip'); +``` + +The below get data from all `test-data.csv.gz` files from any folder inside `my-test-bucket` directory recursively: + +``` sql +SELECT * FROM s3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/**/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip'); +``` + ## Partitioned Write If you specify `PARTITION BY` expression when inserting data into `S3` table, a separate file is created for each partition value. Splitting the data into separate files helps to improve reading operations efficiency. From c457f7dff3d100c29c24e1592f665a9d03ecffed Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 18 Oct 2022 12:21:16 +0000 Subject: [PATCH 04/50] Fix and more tests. --- src/Functions/FunctionBinaryArithmetic.h | 15 +++++++++---- ...461_mullable_pk_monotonicity_bug.reference | 20 ++++++++++++++++++ .../02461_mullable_pk_monotonicity_bug.sql | 21 +++++++++++++++++++ 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index cf752bbcbf8..05f092b5060 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -39,6 +39,7 @@ #include #include #include +#include #include #if USE_EMBEDDED_COMPILER @@ -1778,11 +1779,14 @@ public: // const +|- variable if (left.column && isColumnConst(*left.column)) { + auto left_type = removeLowCardinality(removeNullable(left.type)); + auto right_type = removeLowCardinality(removeNullable(right.type)); + auto transform = [&](const Field & point) { ColumnsWithTypeAndName columns_with_constant - = {{left.column->cloneResized(1), left.type, left.name}, - {removeNullable(right.type)->createColumnConst(1, point), removeNullable(right.type), right.name}}; + = {{left_type->createColumnConst(1, (*left.column)[0]), left_type, left.name}, + {right_type->createColumnConst(1, point), right_type, right.name}}; /// This is a bit dangerous to call Base::executeImpl cause it ignores `use Default Implementation For XXX` flags. /// It was possible to check monotonicity for nullable right type which result to exception. @@ -1816,11 +1820,14 @@ public: // variable +|- constant else if (right.column && isColumnConst(*right.column)) { + auto left_type = removeLowCardinality(removeNullable(left.type)); + auto right_type = removeLowCardinality(removeNullable(right.type)); + auto transform = [&](const Field & point) { ColumnsWithTypeAndName columns_with_constant - = {{removeNullable(left.type)->createColumnConst(1, point), removeNullable(left.type), left.name}, - {right.column->cloneResized(1), right.type, right.name}}; + = {{left_type->createColumnConst(1, point), left_type, left.name}, + {right_type->createColumnConst(1, (*right.column)[0]), right_type, right.name}}; auto col = Base::executeImpl(columns_with_constant, return_type, 1); Field point_transformed; diff --git a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference index 099b7d91c92..c2983d46447 100644 --- a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference +++ b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference @@ -2,3 +2,23 @@ 2 1 2 +1 +2 +1 +2 +1 +2 +1 +2 +1 +2 +1 +2 +1 +2 +1 +2 +1 +2 +1 +2 diff --git a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql index 56ea054cc47..798868fe566 100644 --- a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql +++ b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql @@ -1,6 +1,27 @@ create table tab (x Nullable(UInt8)) engine = MergeTree order by x settings allow_nullable_key = 1, index_granularity = 2; insert into tab select number from numbers(4); +set allow_suspicious_low_cardinality_types=1; +set max_rows_to_read = 2; + +SELECT x + 1 FROM tab where plus(x, 1) <= 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::Nullable(UInt8)) <= 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::LowCardinality(UInt8)) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1, x) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1::Nullable(UInt8), x) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1::LowCardinality(UInt8), x) <= 2 order by x; + +drop table tab; + +set max_rows_to_read = 100; + +create table tab (x LowCardinality(UInt8)) engine = MergeTree order by x settings allow_nullable_key = 1, index_granularity = 2; +insert into tab select number from numbers(4); + set max_rows_to_read = 2; SELECT x + 1 FROM tab where plus(x, 1) <= 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::Nullable(UInt8)) <= 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::LowCardinality(UInt8)) <= 2 order by x; SELECT 1 + x FROM tab where plus(1, x) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1::Nullable(UInt8), x) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1::LowCardinality(UInt8), x) <= 2 order by x; From c8f9e7bd1f028be15d233571c5cff4ed2442ad64 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 19 Oct 2022 14:33:37 +0000 Subject: [PATCH 05/50] Update test. --- src/Functions/FunctionBinaryArithmetic.h | 8 ++++---- .../02461_mullable_pk_monotonicity_bug.reference | 8 ++++++++ .../0_stateless/02461_mullable_pk_monotonicity_bug.sql | 4 ++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 05f092b5060..3ffe054a439 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -1779,8 +1779,8 @@ public: // const +|- variable if (left.column && isColumnConst(*left.column)) { - auto left_type = removeLowCardinality(removeNullable(left.type)); - auto right_type = removeLowCardinality(removeNullable(right.type)); + auto left_type = removeNullable(removeLowCardinality(left.type)); + auto right_type = removeNullable(removeLowCardinality(right.type)); auto transform = [&](const Field & point) { @@ -1820,8 +1820,8 @@ public: // variable +|- constant else if (right.column && isColumnConst(*right.column)) { - auto left_type = removeLowCardinality(removeNullable(left.type)); - auto right_type = removeLowCardinality(removeNullable(right.type)); + auto left_type = removeNullable(removeLowCardinality(left.type)); + auto right_type = removeNullable(removeLowCardinality(right.type)); auto transform = [&](const Field & point) { diff --git a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference index c2983d46447..5ee6e6c67c4 100644 --- a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference +++ b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference @@ -22,3 +22,11 @@ 2 1 2 +1 +2 +1 +2 +1 +2 +1 +2 diff --git a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql index 798868fe566..4f20ec0cd35 100644 --- a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql +++ b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql @@ -7,9 +7,11 @@ set max_rows_to_read = 2; SELECT x + 1 FROM tab where plus(x, 1) <= 2 order by x; SELECT x + 1 FROM tab where plus(x, 1::Nullable(UInt8)) <= 2 order by x; SELECT x + 1 FROM tab where plus(x, 1::LowCardinality(UInt8)) <= 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::LowCardinality(Nullable(UInt8))) <= 2 order by x; SELECT 1 + x FROM tab where plus(1, x) <= 2 order by x; SELECT 1 + x FROM tab where plus(1::Nullable(UInt8), x) <= 2 order by x; SELECT 1 + x FROM tab where plus(1::LowCardinality(UInt8), x) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1::LowCardinality(Nullable(UInt8)), x) <= 2 order by x; drop table tab; @@ -22,6 +24,8 @@ set max_rows_to_read = 2; SELECT x + 1 FROM tab where plus(x, 1) <= 2 order by x; SELECT x + 1 FROM tab where plus(x, 1::Nullable(UInt8)) <= 2 order by x; SELECT x + 1 FROM tab where plus(x, 1::LowCardinality(UInt8)) <= 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::LowCardinality(Nullable(UInt8))) <= 2 order by x; SELECT 1 + x FROM tab where plus(1, x) <= 2 order by x; SELECT 1 + x FROM tab where plus(1::Nullable(UInt8), x) <= 2 order by x; SELECT 1 + x FROM tab where plus(1::LowCardinality(UInt8), x) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1::LowCardinality(Nullable(UInt8)), x) <= 2 order by x; From 58b6194a2d3ff7361617217538c573db7c529b10 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 19 Oct 2022 15:08:00 +0000 Subject: [PATCH 06/50] Fix another one case. --- src/Functions/FunctionsConversion.h | 4 ++++ .../0_stateless/02461_mullable_pk_monotonicity_bug.sql | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 8cbe3b0e532..ebf8417aea6 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -2209,6 +2209,10 @@ struct ToNumberMonotonicity /// Integer cases. + /// Do not support 128 bit integers and decimals for now. + if (!isNativeInteger(type)) + return {}; + const bool from_is_unsigned = type.isValueRepresentedByUnsignedInteger(); const bool to_is_unsigned = is_unsigned_v; diff --git a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql index 4f20ec0cd35..ce190694514 100644 --- a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql +++ b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql @@ -29,3 +29,12 @@ SELECT 1 + x FROM tab where plus(1, x) <= 2 order by x; SELECT 1 + x FROM tab where plus(1::Nullable(UInt8), x) <= 2 order by x; SELECT 1 + x FROM tab where plus(1::LowCardinality(UInt8), x) <= 2 order by x; SELECT 1 + x FROM tab where plus(1::LowCardinality(Nullable(UInt8)), x) <= 2 order by x; + +drop table tab; + +set max_rows_to_read = 100; + +create table tab (x UInt128) engine = MergeTree order by x settings allow_nullable_key = 1, index_granularity = 2; +insert into tab select number from numbers(4); + +SELECT x + 1 FROM tab WHERE (x + 1::LowCardinality(UInt8)) <= -9223372036854775808 order by x; From f485c32317b49e7c5cad9518136e6d33e3d4f103 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Wed, 19 Oct 2022 23:38:44 +0000 Subject: [PATCH 07/50] Small improvements --- src/Parsers/ExpressionListParsers.cpp | 51 ++++++++++++++------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index f7a016a59e4..1e3ffaa1c72 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -478,7 +478,10 @@ struct Operator { Operator() = default; - Operator(const std::string & function_name_, int priority_, int arity_ = 2, OperatorType type_ = OperatorType::None) + Operator(const std::string & function_name_, + int priority_, + int arity_, + OperatorType type_ = OperatorType::None) : type(type_), priority(priority_), arity(arity_), function_name(function_name_) {} OperatorType type; @@ -507,9 +510,7 @@ class Layer { public: explicit Layer(bool allow_alias_ = true, bool allow_alias_without_as_keyword_ = true) : - allow_alias(allow_alias_), allow_alias_without_as_keyword(allow_alias_without_as_keyword_) - { - } + allow_alias(allow_alias_), allow_alias_without_as_keyword(allow_alias_without_as_keyword_) {} virtual ~Layer() = default; @@ -810,10 +811,10 @@ public: }; -class OrdinaryFunctionLayer : public Layer +class FunctionLayer : public Layer { public: - explicit OrdinaryFunctionLayer(String function_name_, bool allow_function_parameters_ = true) + explicit FunctionLayer(String function_name_, bool allow_function_parameters_ = true) : function_name(function_name_), allow_function_parameters(allow_function_parameters_){} bool parse(IParser::Pos & pos, Expected & expected, Action & action) override @@ -958,7 +959,7 @@ public: if (parameters) { - function_node->parameters = parameters; + function_node->parameters = std::move(parameters); function_node->children.push_back(function_node->parameters); } @@ -991,7 +992,7 @@ public: return false; } - elements = {function_node}; + elements = {std::move(function_node)}; finished = true; } @@ -1990,9 +1991,9 @@ std::unique_ptr getFunctionLayer(ASTPtr identifier, bool is_table_functio || function_name_lowercase == "timestampdiff" || function_name_lowercase == "timestamp_diff") return std::make_unique(); else if (function_name_lowercase == "grouping") - return std::make_unique(function_name_lowercase, allow_function_parameters_); + return std::make_unique(function_name_lowercase, allow_function_parameters_); else - return std::make_unique(function_name, allow_function_parameters_); + return std::make_unique(function_name, allow_function_parameters_); } @@ -2141,22 +2142,22 @@ std::vector> ParserExpressionImpl::operators_t {"<", Operator("less", 9, 2, OperatorType::Comparison)}, {">", Operator("greater", 9, 2, OperatorType::Comparison)}, {"=", Operator("equals", 9, 2, OperatorType::Comparison)}, - {"LIKE", Operator("like", 9)}, - {"ILIKE", Operator("ilike", 9)}, - {"NOT LIKE", Operator("notLike", 9)}, - {"NOT ILIKE", Operator("notILike", 9)}, - {"IN", Operator("in", 9)}, - {"NOT IN", Operator("notIn", 9)}, - {"GLOBAL IN", Operator("globalIn", 9)}, - {"GLOBAL NOT IN", Operator("globalNotIn", 9)}, + {"LIKE", Operator("like", 9, 2)}, + {"ILIKE", Operator("ilike", 9, 2)}, + {"NOT LIKE", Operator("notLike", 9, 2)}, + {"NOT ILIKE", Operator("notILike", 9, 2)}, + {"IN", Operator("in", 9, 2)}, + {"NOT IN", Operator("notIn", 9, 2)}, + {"GLOBAL IN", Operator("globalIn", 9, 2)}, + {"GLOBAL NOT IN", Operator("globalNotIn", 9, 2)}, {"||", Operator("concat", 10, 2, OperatorType::Mergeable)}, - {"+", Operator("plus", 11)}, - {"-", Operator("minus", 11)}, - {"*", Operator("multiply", 12)}, - {"/", Operator("divide", 12)}, - {"%", Operator("modulo", 12)}, - {"MOD", Operator("modulo", 12)}, - {"DIV", Operator("intDiv", 12)}, + {"+", Operator("plus", 11, 2)}, + {"-", Operator("minus", 11, 2)}, + {"*", Operator("multiply", 12, 2)}, + {"/", Operator("divide", 12, 2)}, + {"%", Operator("modulo", 12, 2)}, + {"MOD", Operator("modulo", 12, 2)}, + {"DIV", Operator("intDiv", 12, 2)}, {".", Operator("tupleElement", 14, 2, OperatorType::TupleElement)}, {"[", Operator("arrayElement", 14, 2, OperatorType::ArrayElement)}, {"::", Operator("CAST", 14, 2, OperatorType::Cast)}, From 159a251f789fd7aaed23168ba40149f0b9856ab7 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Wed, 19 Oct 2022 23:39:35 +0000 Subject: [PATCH 08/50] Fix aliases --- src/Parsers/ExpressionListParsers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 1e3ffaa1c72..fbdfa597f82 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -509,7 +509,7 @@ enum class Checkpoint class Layer { public: - explicit Layer(bool allow_alias_ = true, bool allow_alias_without_as_keyword_ = true) : + explicit Layer(bool allow_alias_ = true, bool allow_alias_without_as_keyword_ = false) : allow_alias(allow_alias_), allow_alias_without_as_keyword(allow_alias_without_as_keyword_) {} virtual ~Layer() = default; From 0df9d2f5ccd8f88e967e830d6a7bf5c9e07e830e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 20 Oct 2022 13:41:27 +0000 Subject: [PATCH 09/50] Fix tests. --- src/Functions/FunctionsConversion.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index ebf8417aea6..b801cc070b6 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -2209,16 +2209,16 @@ struct ToNumberMonotonicity /// Integer cases. - /// Do not support 128 bit integers and decimals for now. - if (!isNativeInteger(type)) - return {}; - const bool from_is_unsigned = type.isValueRepresentedByUnsignedInteger(); const bool to_is_unsigned = is_unsigned_v; const size_t size_of_from = type.getSizeOfValueInMemory(); const size_t size_of_to = sizeof(T); + /// Do not support 128 bit integers and decimals for now. + if (size_of_from > sizeof(Int64)) + return {}; + const bool left_in_first_half = left.isNull() ? from_is_unsigned : (left.get() >= 0); From fb67a2f2ce8401288a86b0eba5fb3d9a3c6500a2 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Fri, 21 Oct 2022 00:16:24 +0000 Subject: [PATCH 10/50] Fixes & improvements --- src/Parsers/ExpressionListParsers.cpp | 210 ++++++++++++++------------ 1 file changed, 114 insertions(+), 96 deletions(-) diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index fbdfa597f82..a38d7ab0c98 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -613,13 +613,17 @@ public: /// bool mergeElement(bool push_to_elements = true) { + parsed_alias = false; + Operator cur_op; while (popOperator(cur_op)) { ASTPtr function; - // Special case of ternary operator - if (cur_op.type == OperatorType::StartIf) + // We should not meet the starting part of the operator while finishing an element + if (cur_op.type == OperatorType::StartIf || + cur_op.type == OperatorType::StartBetween || + cur_op.type == OperatorType::StartNotBetween) return false; if (cur_op.type == OperatorType::FinishIf) @@ -629,10 +633,6 @@ public: return false; } - // Special case of a BETWEEN b AND c operator - if (cur_op.type == OperatorType::StartBetween || cur_op.type == OperatorType::StartNotBetween) - return false; - if (cur_op.type == OperatorType::FinishBetween) { Operator tmp_op; @@ -728,6 +728,9 @@ public: /// In order to distinguish them we keep a counter of BETWEENs without matching ANDs. int between_counter = 0; + /// Flag we set when we parsed alias to avoid parsing next element as alias + bool parsed_alias = false; + bool allow_alias = true; bool allow_alias_without_as_keyword = true; @@ -777,7 +780,6 @@ public: } }; - /// Basic layer for a function with certain separator and end tokens: /// 1. If we parse a separator we should merge current operands and operators /// into one element and push in to 'elements' vector. @@ -787,6 +789,9 @@ template class BaseLayer : public Layer { public: + explicit BaseLayer(bool allow_alias_ = true, bool allow_alias_without_as_keyword_ = false) : + Layer(allow_alias_, allow_alias_without_as_keyword_) {} + bool parse(IParser::Pos & pos, Expected & expected, Action & action) override { if (ParserToken(separator).ignore(pos, expected)) @@ -810,7 +815,7 @@ public: } }; - +/// Layer for regular and aggregate functions without syntax sugar class FunctionLayer : public Layer { public: @@ -1091,6 +1096,8 @@ public: class CastLayer : public Layer { public: + CastLayer() : Layer(true, true) {} + bool parse(IParser::Pos & pos, Expected & expected, Action & action) override { /// CAST(x [AS alias1], T [AS alias2]) or CAST(x [AS alias1] AS T) @@ -1189,6 +1196,8 @@ public: class ExtractLayer : public BaseLayer { public: + ExtractLayer() : BaseLayer(true, true) {} + bool getResult(ASTPtr & node) override { if (state == 2) @@ -1258,6 +1267,8 @@ private: class SubstringLayer : public Layer { public: + SubstringLayer() : Layer(true, true) {} + bool getResult(ASTPtr & node) override { node = makeASTFunction("substring", std::move(elements)); @@ -1318,6 +1329,8 @@ public: class PositionLayer : public Layer { public: + PositionLayer() : Layer(true, true) {} + bool getResult(ASTPtr & node) override { if (state == 2) @@ -1383,10 +1396,11 @@ public: } }; - class ExistsLayer : public Layer { public: + ExistsLayer() : Layer(true, true) {} + bool parse(IParser::Pos & pos, Expected & expected, Action & /*action*/) override { ASTPtr node; @@ -1411,9 +1425,8 @@ public: class TrimLayer : public Layer { public: - TrimLayer(bool trim_left_, bool trim_right_) : trim_left(trim_left_), trim_right(trim_right_) - { - } + TrimLayer(bool trim_left_, bool trim_right_) + : Layer(true, true), trim_left(trim_left_), trim_right(trim_right_) {} bool getResult(ASTPtr & node) override { @@ -1571,13 +1584,11 @@ private: String function_name; }; - class DateAddLayer : public BaseLayer { public: - explicit DateAddLayer(const char * function_name_) : function_name(function_name_) - { - } + explicit DateAddLayer(const char * function_name_) + : BaseLayer(true, true), function_name(function_name_) {} bool getResult(ASTPtr & node) override { @@ -1628,10 +1639,11 @@ private: bool parsed_interval_kind = false; }; - class DateDiffLayer : public BaseLayer { public: + DateDiffLayer() : BaseLayer(true, true) {} + bool getResult(ASTPtr & node) override { if (parsed_interval_kind) @@ -1681,10 +1693,11 @@ private: bool parsed_interval_kind = false; }; - class IntervalLayer : public Layer { public: + IntervalLayer() : Layer(true, true) {} + bool parse(IParser::Pos & pos, Expected & expected, Action & action) override { /// INTERVAL 1 HOUR or INTERVAL expr HOUR @@ -1759,86 +1772,11 @@ private: IntervalKind interval_kind; }; -/// Layer for table function 'view' and 'viewIfPermitted' -class ViewLayer : public Layer -{ -public: - explicit ViewLayer(bool if_permitted_) : if_permitted(if_permitted_) {} - - bool getResult(ASTPtr & node) override - { - if (if_permitted) - node = makeASTFunction("viewIfPermitted", std::move(elements)); - else - node = makeASTFunction("view", std::move(elements)); - - return true; - } - - bool parse(IParser::Pos & pos, Expected & expected, Action & /*action*/) override - { - /// view(SELECT ...) - /// viewIfPermitted(SELECT ... ELSE func(...)) - /// - /// 0. Parse the SELECT query and if 'if_permitted' parse 'ELSE' keyword (-> 1) else (finished) - /// 1. Parse closing token - - if (state == 0) - { - ASTPtr query; - - bool maybe_an_subquery = pos->type == TokenType::OpeningRoundBracket; - - if (!ParserSelectWithUnionQuery().parse(pos, query, expected)) - return false; - - auto & select_ast = query->as(); - if (select_ast.list_of_selects->children.size() == 1 && maybe_an_subquery) - { - // It's an subquery. Bail out. - return false; - } - - pushResult(query); - - if (!if_permitted) - { - if (!ParserToken(TokenType::ClosingRoundBracket).ignore(pos, expected)) - return false; - - finished = true; - return true; - } - - if (!ParserKeyword{"ELSE"}.ignore(pos, expected)) - return false; - - state = 1; - return true; - } - - if (state == 1) - { - if (ParserToken(TokenType::ClosingRoundBracket).ignore(pos, expected)) - { - if (!mergeElement()) - return false; - - finished = true; - } - } - - return true; - } - -private: - bool if_permitted; -}; - - class CaseLayer : public Layer { public: + CaseLayer() : Layer(true, true) {} + bool parse(IParser::Pos & pos, Expected & expected, Action & action) override { /// CASE [x] WHEN expr THEN expr [WHEN expr THEN expr [...]] [ELSE expr] END @@ -1927,6 +1865,82 @@ private: bool has_case_expr; }; +/// Layer for table function 'view' and 'viewIfPermitted' +class ViewLayer : public Layer +{ +public: + explicit ViewLayer(bool if_permitted_) : if_permitted(if_permitted_) {} + + bool getResult(ASTPtr & node) override + { + if (if_permitted) + node = makeASTFunction("viewIfPermitted", std::move(elements)); + else + node = makeASTFunction("view", std::move(elements)); + + return true; + } + + bool parse(IParser::Pos & pos, Expected & expected, Action & /*action*/) override + { + /// view(SELECT ...) + /// viewIfPermitted(SELECT ... ELSE func(...)) + /// + /// 0. Parse the SELECT query and if 'if_permitted' parse 'ELSE' keyword (-> 1) else (finished) + /// 1. Parse closing token + + if (state == 0) + { + ASTPtr query; + + bool maybe_an_subquery = pos->type == TokenType::OpeningRoundBracket; + + if (!ParserSelectWithUnionQuery().parse(pos, query, expected)) + return false; + + auto & select_ast = query->as(); + if (select_ast.list_of_selects->children.size() == 1 && maybe_an_subquery) + { + // It's an subquery. Bail out. + return false; + } + + pushResult(query); + + if (!if_permitted) + { + if (!ParserToken(TokenType::ClosingRoundBracket).ignore(pos, expected)) + return false; + + finished = true; + return true; + } + + if (!ParserKeyword{"ELSE"}.ignore(pos, expected)) + return false; + + state = 1; + return true; + } + + if (state == 1) + { + if (ParserToken(TokenType::ClosingRoundBracket).ignore(pos, expected)) + { + if (!mergeElement()) + return false; + + finished = true; + } + } + + return true; + } + +private: + bool if_permitted; +}; + std::unique_ptr getFunctionLayer(ASTPtr identifier, bool is_table_function, bool allow_function_parameters_ = true) { @@ -2428,11 +2442,15 @@ Action ParserExpressionImpl::tryParseOperator(Layers & layers, IParser::Pos & po if (cur_op == operators_table.end()) { + ParserAlias alias_parser(layers.back()->allow_alias_without_as_keyword); auto old_pos = pos; - if (layers.back()->allow_alias && ParserAlias(layers.back()->allow_alias_without_as_keyword).parse(pos, tmp, expected)) + if (!layers.back()->parsed_alias && layers.back()->allow_alias && alias_parser.parse(pos, tmp, expected)) { if (layers.back()->insertAlias(tmp)) + { + layers.back()->parsed_alias = true; return Action::OPERATOR; + } } pos = old_pos; return Action::NONE; From cd7a48215ccbff4bbce45a1e43f9c69e59e88c93 Mon Sep 17 00:00:00 2001 From: flynn Date: Sat, 22 Oct 2022 16:17:05 +0000 Subject: [PATCH 11/50] Fix create set with wrong header when data type is lowcardinality --- src/Interpreters/Set.cpp | 1 + ...467_set_with_lowcardinality_type.reference | 2 ++ .../02467_set_with_lowcardinality_type.sql | 30 +++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 tests/queries/0_stateless/02467_set_with_lowcardinality_type.reference create mode 100644 tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index ded8b04a589..e75232aa0f5 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -131,6 +131,7 @@ void Set::setHeader(const ColumnsWithTypeAndName & header) if (const auto * low_cardinality_type = typeid_cast(data_types.back().get())) { data_types.back() = low_cardinality_type->getDictionaryType(); + set_elements_types.back() = low_cardinality_type->getDictionaryType(); materialized_columns.emplace_back(key_columns.back()->convertToFullColumnIfLowCardinality()); key_columns.back() = materialized_columns.back().get(); } diff --git a/tests/queries/0_stateless/02467_set_with_lowcardinality_type.reference b/tests/queries/0_stateless/02467_set_with_lowcardinality_type.reference new file mode 100644 index 00000000000..b3f28057554 --- /dev/null +++ b/tests/queries/0_stateless/02467_set_with_lowcardinality_type.reference @@ -0,0 +1,2 @@ +1 test +1 test diff --git a/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql b/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql new file mode 100644 index 00000000000..7b572df73f5 --- /dev/null +++ b/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql @@ -0,0 +1,30 @@ +DROP TABLE IF EXISTS bloom_filter_nullable_index__fuzz_0; +CREATE TABLE bloom_filter_nullable_index__fuzz_0 +( + `order_key` UInt64, + `str` Nullable(String), + INDEX idx str TYPE bloom_filter GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY order_key SETTINGS index_granularity = 6; + +INSERT INTO bloom_filter_nullable_index__fuzz_0 VALUES (1, 'test'); +INSERT INTO bloom_filter_nullable_index__fuzz_0 VALUES (2, 'test2'); + +DROP TABLE IF EXISTS bloom_filter_nullable_index__fuzz_1; +CREATE TABLE bloom_filter_nullable_index__fuzz_1 +( + `order_key` UInt64, + `str` Nullable(String), + INDEX idx str TYPE bloom_filter GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY order_key SETTINGS index_granularity = 6; + +INSERT INTO bloom_filter_nullable_index__fuzz_0 VALUES (1, 'test'); +INSERT INTO bloom_filter_nullable_index__fuzz_0 VALUES (2, 'test2'); + +DROP TABLE IF EXISTS nullable_string_value__fuzz_2; +CREATE TABLE nullable_string_value__fuzz_2 (`value` LowCardinality(String)) ENGINE = TinyLog; +INSERT INTO nullable_string_value__fuzz_2 VALUES ('test'); + +SELECT * FROM bloom_filter_nullable_index__fuzz_0 WHERE str IN (SELECT value FROM nullable_string_value__fuzz_2); +SELECT * FROM bloom_filter_nullable_index__fuzz_1 WHERE str IN (SELECT value FROM nullable_string_value__fuzz_2); From 1aa8a986dd8f7a7b2a17f5466cc94fac1637e9db Mon Sep 17 00:00:00 2001 From: flynn Date: Sat, 22 Oct 2022 16:19:49 +0000 Subject: [PATCH 12/50] update test --- tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql b/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql index 7b572df73f5..92519a48bea 100644 --- a/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql +++ b/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql @@ -1,3 +1,4 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/42460 DROP TABLE IF EXISTS bloom_filter_nullable_index__fuzz_0; CREATE TABLE bloom_filter_nullable_index__fuzz_0 ( From 5ac7538e92e32b63a635587aa7ee34f9cc5493fe Mon Sep 17 00:00:00 2001 From: flynn Date: Sat, 22 Oct 2022 16:21:28 +0000 Subject: [PATCH 13/50] update test --- .../queries/0_stateless/02467_set_with_lowcardinality_type.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql b/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql index 92519a48bea..dee6f7de74a 100644 --- a/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql +++ b/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql @@ -15,7 +15,7 @@ DROP TABLE IF EXISTS bloom_filter_nullable_index__fuzz_1; CREATE TABLE bloom_filter_nullable_index__fuzz_1 ( `order_key` UInt64, - `str` Nullable(String), + `str` String, INDEX idx str TYPE bloom_filter GRANULARITY 1 ) ENGINE = MergeTree ORDER BY order_key SETTINGS index_granularity = 6; From 2fc91fd33802ba69193e3d8cb1caa154dcefcd24 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Sat, 22 Oct 2022 18:27:24 +0200 Subject: [PATCH 14/50] fix behaviour of max_rows_to_read for trival limit queries --- src/Interpreters/InterpreterSelectQuery.cpp | 17 +++++++++----- .../QueryPlan/ReadFromMergeTree.cpp | 19 +++++++++++++++- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 4 ++++ .../MergeTree/MergeTreeSelectProcessor.cpp | 8 ------- src/Storages/SelectQueryInfo.h | 3 +++ src/Storages/System/StorageSystemNumbers.cpp | 17 +++++++++++--- ...5_limit_trivial_max_rows_to_read.reference | 7 ++++++ .../02465_limit_trivial_max_rows_to_read.sql | 22 +++++++++++++++++++ 8 files changed, 80 insertions(+), 17 deletions(-) create mode 100644 tests/queries/0_stateless/02465_limit_trivial_max_rows_to_read.reference create mode 100644 tests/queries/0_stateless/02465_limit_trivial_max_rows_to_read.sql diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 79deb38317c..adf341f5ffd 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2143,6 +2143,8 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, context); + auto local_limits = getStorageLimits(*context, options); + /** Optimization - if not specified DISTINCT, WHERE, GROUP, HAVING, ORDER, JOIN, LIMIT BY, WITH TIES * but LIMIT is specified, and limit + offset < max_block_size, * then as the block size we will use limit + offset (not to read more from the table than requested), @@ -2161,17 +2163,22 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc && !query_analyzer->hasAggregation() && !query_analyzer->hasWindow() && query.limitLength() - && limit_length <= std::numeric_limits::max() - limit_offset - && limit_length + limit_offset < max_block_size) + && limit_length <= std::numeric_limits::max() - limit_offset) { - max_block_size = std::max(1, limit_length + limit_offset); - max_threads_execute_query = max_streams = 1; + if (limit_length + limit_offset < max_block_size) + { + max_block_size = std::max(1, limit_length + limit_offset); + max_threads_execute_query = max_streams = 1; + } + if (limit_length + limit_offset < local_limits.local_limits.size_limits.max_rows) + { + query_info.limit = limit_length + limit_offset; + } } if (!max_block_size) throw Exception("Setting 'max_block_size' cannot be zero", ErrorCodes::PARAMETER_OUT_OF_BOUND); - auto local_limits = getStorageLimits(*context, options); storage_limits.emplace_back(local_limits); /// Initialize the initial data streams to which the query transforms are superimposed. Table or subquery or prepared input? diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index b340073e73d..164ec8777de 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -173,6 +173,9 @@ Pipe ReadFromMergeTree::readFromPool( total_rows += part.getRowsCount(); } + if (query_info.limit > 0 && query_info.limit < total_rows) + total_rows = query_info.limit; + const auto & settings = context->getSettingsRef(); const auto & client_info = context->getClientInfo(); MergeTreeReadPool::BackoffSettings backoff_settings(settings); @@ -246,10 +249,24 @@ ProcessorPtr ReadFromMergeTree::createSource( }; } - return std::make_shared( + auto total_rows = part.getRowsCount(); + if (query_info.limit > 0 && query_info.limit < total_rows) + total_rows = query_info.limit; + + auto source = std::make_shared( data, storage_snapshot, part.data_part, max_block_size, preferred_block_size_bytes, preferred_max_column_in_block_size_bytes, required_columns, part.ranges, use_uncompressed_cache, prewhere_info, actions_settings, reader_settings, virt_column_names, part.part_index_in_query, has_limit_below_one_block, std::move(extension)); + + /// Actually it means that parallel reading from replicas enabled + /// and we have to collaborate with initiator. + /// In this case we won't set approximate rows, because it will be accounted multiple times. + /// Also do not count amount of read rows if we read in order of sorting key, + /// because we don't know actual amount of read rows in case when limit is set. + if (!extension.has_value() && !reader_settings.read_in_order) + source -> addTotalRowsApprox(total_rows); + + return source; } Pipe ReadFromMergeTree::readInOrder( diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 91ecb3a37a0..0eddaac2fac 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1061,6 +1061,10 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd auto current_rows_estimate = ranges.getRowsCount(); size_t prev_total_rows_estimate = total_rows.fetch_add(current_rows_estimate); size_t total_rows_estimate = current_rows_estimate + prev_total_rows_estimate; + if (query_info.limit > 0 && total_rows_estimate > query_info.limit) + { + total_rows_estimate = query_info.limit; + } limits.check(total_rows_estimate, 0, "rows (controlled by 'max_rows_to_read' setting)", ErrorCodes::TOO_MANY_ROWS); leaf_limits.check( total_rows_estimate, 0, "rows (controlled by 'max_rows_to_read_leaf' setting)", ErrorCodes::TOO_MANY_ROWS); diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 59cbae3f914..2490eb77772 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -38,14 +38,6 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( has_limit_below_one_block(has_limit_below_one_block_), total_rows(data_part->index_granularity.getRowsCountInRanges(all_mark_ranges)) { - /// Actually it means that parallel reading from replicas enabled - /// and we have to collaborate with initiator. - /// In this case we won't set approximate rows, because it will be accounted multiple times. - /// Also do not count amount of read rows if we read in order of sorting key, - /// because we don't know actual amount of read rows in case when limit is set. - if (!extension_.has_value() && !reader_settings.read_in_order) - addTotalRowsApprox(total_rows); - ordered_names = header_without_virtual_columns.getNames(); } diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index f2835ab4dbf..94a8c1143f3 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -220,6 +220,9 @@ struct SelectQueryInfo Block minmax_count_projection_block; MergeTreeDataSelectAnalysisResultPtr merge_tree_select_result_ptr; + // If limit is not 0, that means it's a trival limit query. + UInt64 limit = 0; + InputOrderInfoPtr getInputOrderInfo() const { return input_order_info ? input_order_info : (projection ? projection->input_order_info : nullptr); diff --git a/src/Storages/System/StorageSystemNumbers.cpp b/src/Storages/System/StorageSystemNumbers.cpp index 523ec25b89c..fbcd449051f 100644 --- a/src/Storages/System/StorageSystemNumbers.cpp +++ b/src/Storages/System/StorageSystemNumbers.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -125,7 +126,7 @@ StorageSystemNumbers::StorageSystemNumbers(const StorageID & table_id, bool mult Pipe StorageSystemNumbers::read( const Names & column_names, const StorageSnapshotPtr & storage_snapshot, - SelectQueryInfo &, + SelectQueryInfo & query_info, ContextPtr /*context*/, QueryProcessingStage::Enum /*processed_stage*/, size_t max_block_size, @@ -154,7 +155,12 @@ Pipe StorageSystemNumbers::read( auto source = std::make_shared(state, max_block_size, max_counter); if (i == 0) - source->addTotalRowsApprox(*limit); + { + auto rows_appr = *limit; + if (query_info.limit > 0 && query_info.limit < rows_appr) + rows_appr = query_info.limit; + source->addTotalRowsApprox(rows_appr); + } pipe.addSource(std::move(source)); } @@ -167,7 +173,12 @@ Pipe StorageSystemNumbers::read( auto source = std::make_shared(max_block_size, offset + i * max_block_size, num_streams * max_block_size); if (limit && i == 0) - source->addTotalRowsApprox(*limit); + { + auto rows_appr = *limit; + if (query_info.limit > 0 && query_info.limit < rows_appr) + rows_appr = query_info.limit; + source->addTotalRowsApprox(rows_appr); + } pipe.addSource(std::move(source)); } diff --git a/tests/queries/0_stateless/02465_limit_trivial_max_rows_to_read.reference b/tests/queries/0_stateless/02465_limit_trivial_max_rows_to_read.reference new file mode 100644 index 00000000000..87370760038 --- /dev/null +++ b/tests/queries/0_stateless/02465_limit_trivial_max_rows_to_read.reference @@ -0,0 +1,7 @@ +0 +0 +1 +2 +3 +4 +0 diff --git a/tests/queries/0_stateless/02465_limit_trivial_max_rows_to_read.sql b/tests/queries/0_stateless/02465_limit_trivial_max_rows_to_read.sql new file mode 100644 index 00000000000..ee7a4e6b6b5 --- /dev/null +++ b/tests/queries/0_stateless/02465_limit_trivial_max_rows_to_read.sql @@ -0,0 +1,22 @@ +DROP TABLE IF EXISTS t_max_rows_to_read; + +CREATE TABLE t_max_rows_to_read (a UInt64) +ENGINE = MergeTree ORDER BY a +SETTINGS index_granularity = 4; + +INSERT INTO t_max_rows_to_read SELECT number FROM numbers(100); + +SET max_block_size = 10; +SET max_rows_to_read = 20; +SET read_overflow_mode = 'throw'; + +SELECT number FROM numbers(30); -- { serverError 158 } +SELECT number FROM numbers(30) LIMIT 21; -- { serverError 158 } +SELECT number FROM numbers(30) LIMIT 1; +SELECT number FROM numbers(5); + +SELECT a FROM t_max_rows_to_read LIMIT 1; +SELECT a FROM t_max_rows_to_read LIMIT 11 offset 11; -- { serverError 158 } +SELECT a FROM t_max_rows_to_read WHERE a > 50 LIMIT 1; -- { serverError 158 } + +DROP TABLE t_max_rows_to_read; From 4a8326ff0c33758570aee25702ed99143e43d853 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Sun, 23 Oct 2022 23:22:52 +0200 Subject: [PATCH 15/50] fix typo --- src/Storages/SelectQueryInfo.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 94a8c1143f3..565594569ce 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -220,7 +220,7 @@ struct SelectQueryInfo Block minmax_count_projection_block; MergeTreeDataSelectAnalysisResultPtr merge_tree_select_result_ptr; - // If limit is not 0, that means it's a trival limit query. + // If limit is not 0, that means it's a trivial limit query. UInt64 limit = 0; InputOrderInfoPtr getInputOrderInfo() const From 30f4f33e7929c7241d260aed526dca96936372fb Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Mon, 24 Oct 2022 01:39:40 +0000 Subject: [PATCH 16/50] Add test --- .../0_stateless/02469_fix_aliases_parser.reference | 2 ++ tests/queries/0_stateless/02469_fix_aliases_parser.sql | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/queries/0_stateless/02469_fix_aliases_parser.reference create mode 100644 tests/queries/0_stateless/02469_fix_aliases_parser.sql diff --git a/tests/queries/0_stateless/02469_fix_aliases_parser.reference b/tests/queries/0_stateless/02469_fix_aliases_parser.reference new file mode 100644 index 00000000000..09f584c9cd4 --- /dev/null +++ b/tests/queries/0_stateless/02469_fix_aliases_parser.reference @@ -0,0 +1,2 @@ +45 +[0] diff --git a/tests/queries/0_stateless/02469_fix_aliases_parser.sql b/tests/queries/0_stateless/02469_fix_aliases_parser.sql new file mode 100644 index 00000000000..227d8becdb6 --- /dev/null +++ b/tests/queries/0_stateless/02469_fix_aliases_parser.sql @@ -0,0 +1,9 @@ +SELECT sum(number number number) FROM numbers(10); -- { clientError 62 } +SELECT sum(number number) FROM numbers(10); -- { clientError 62 } +SELECT sum(number AS number) FROM numbers(10); + +SELECT [number number number] FROM numbers(1); -- { clientError 62 } +SELECT [number number] FROM numbers(1); -- { clientError 62 } +SELECT [number AS number] FROM numbers(1); + +SELECT cast('1234' lhs lhs, 'UInt32'), lhs; -- { clientError 62 } \ No newline at end of file From 4ddbdead4a57b28d89e30db5bd3928ab7fa71d76 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Oct 2022 10:17:56 +0200 Subject: [PATCH 17/50] Add changelog for 22.10 --- CHANGELOG.md | 129 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83c1cbf1eb4..4c7ca701d53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ### Table of Contents +**[ClickHouse release v22.10, 2022-10-25](#2210)**
**[ClickHouse release v22.9, 2022-09-22](#229)**
**[ClickHouse release v22.8-lts, 2022-08-18](#228)**
**[ClickHouse release v22.7, 2022-07-21](#227)**
@@ -10,6 +11,134 @@ **[ClickHouse release v22.1, 2022-01-18](#221)**
**[Changelog for 2021](https://clickhouse.com/docs/en/whats-new/changelog/2021/)**
+### ClickHouse release 22.10, 2022-10-25 + +#### Backward Incompatible Change +* Rename cache commands: `show caches` -> `show filesystem caches`, `describe cache` -> `describe filesystem cache`. [#41508](https://github.com/ClickHouse/ClickHouse/pull/41508) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove support for the `WITH TIMEOUT` section for `LIVE VIEW`. This closes [#40557](https://github.com/ClickHouse/ClickHouse/issues/40557). [#42173](https://github.com/ClickHouse/ClickHouse/pull/42173) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove support for the `{database}` macro from the client's prompt. It was displayed incorrectly if the database was unspecified and it was not updated on `USE` statements. This closes [#25891](https://github.com/ClickHouse/ClickHouse/issues/25891). [#42508](https://github.com/ClickHouse/ClickHouse/pull/42508) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### New Feature +* Composable protocol configuration is added. Now different protocols can be set up with different listen hosts. Protocol wrappers such as PROXYv1 can be set up over any other protocols (TCP, TCP secure, MySQL, Postgres). [#41198](https://github.com/ClickHouse/ClickHouse/pull/41198) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Add `S3` as a new type of the destination of backups. Support BACKUP to S3 with as-is path/data structure. [#42333](https://github.com/ClickHouse/ClickHouse/pull/42333) ([Vitaly Baranov](https://github.com/vitlibar)), [#42232](https://github.com/ClickHouse/ClickHouse/pull/42232) ([Azat Khuzhin](https://github.com/azat)). +* Added functions (`randUniform`, `randNormal`, `randLogNormal`, `randExponential`, `randChiSquared`, `randStudentT`, `randFisherF`, `randBernoulli`, `randBinomial`, `randNegativeBinomial`, `randPoisson` ) to generate random values according to the specified distributions. This closes [#21834](https://github.com/ClickHouse/ClickHouse/issues/21834). [#42411](https://github.com/ClickHouse/ClickHouse/pull/42411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* An improvement for ClickHouse Keeper: add support for uploading snapshots to S3. S3 information can be defined inside `keeper_server.s3_snapshot`. [#41342](https://github.com/ClickHouse/ClickHouse/pull/41342) ([Antonio Andelic](https://github.com/antonio2368)). +* Added an aggregate function `analysisOfVariance` (`anova`) to perform a statistical test over several groups of normally distributed observations to find out whether all groups have the same mean or not. Original PR [#37872](https://github.com/ClickHouse/ClickHouse/issues/37872). [#42131](https://github.com/ClickHouse/ClickHouse/pull/42131) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Support limiting of temporary data stored on disk using settings `max_temporary_data_on_disk_size_for_user`/`max_temporary_data_on_disk_size_for_query` . [#40893](https://github.com/ClickHouse/ClickHouse/pull/40893) ([Vladimir C](https://github.com/vdimir)). +* Add setting `format_json_object_each_row_column_for_object_name` to write/parse object name as column value in JSONObjectEachRow format. [#41703](https://github.com/ClickHouse/ClickHouse/pull/41703) ([Kruglov Pavel](https://github.com/Avogar)). +* Add BLAKE3 hash-function to SQL. [#33435](https://github.com/ClickHouse/ClickHouse/pull/33435) ([BoloniniD](https://github.com/BoloniniD)). +* The function `javaHash` has been extended to integers. [#41131](https://github.com/ClickHouse/ClickHouse/pull/41131) ([JackyWoo](https://github.com/JackyWoo)). +* Add OpenTelemetry support to ON CLUSTER DDL (require `distributed_ddl_entry_format_version` to be set to 4). [#41484](https://github.com/ClickHouse/ClickHouse/pull/41484) ([Frank Chen](https://github.com/FrankChen021)). +* Added system table `asynchronous_insert_log`. It contains information about asynchronous inserts (including results of queries in fire-and-forget mode (with `wait_for_async_insert=0`)) for better introspection. [#42040](https://github.com/ClickHouse/ClickHouse/pull/42040) ([Anton Popov](https://github.com/CurtizJ)). +* Add support for methods `lz4`, `bz2`, `snappy` in HTTP's `Accept-Encoding` which is a non-standard extension to HTTP protocol. [#42071](https://github.com/ClickHouse/ClickHouse/pull/42071) ([Nikolay Degterinsky](https://github.com/evillique)). + +#### Experimental Feature +* Initial implementation of Kusto Query Language. Please don't use it. [#37961](https://github.com/ClickHouse/ClickHouse/pull/37961) ([Yong Wang](https://github.com/kashwy)). + +#### Performance Improvement +* Relax the "Too many parts" threshold. This closes [#6551](https://github.com/ClickHouse/ClickHouse/issues/6551). Now ClickHouse will allow more parts in a partition if the average part size is large enough (at least 10 GiB). This allows to have up to petabytes of data in a single partition of a single table on a single server, which is possible using disk shelves or object storage. [#42002](https://github.com/ClickHouse/ClickHouse/pull/42002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Implement operator precedence element parser to resolve stack overflow issues and make the required stack size smaller. [#34892](https://github.com/ClickHouse/ClickHouse/pull/34892) ([Nikolay Degterinsky](https://github.com/evillique)). +* DISTINCT in order optimization leverage sorting properties of data streams. This improvement will enable reading in order for DISTINCT if applicable (before it was necessary to provide ORDER BY for columns in DISTINCT). [#41014](https://github.com/ClickHouse/ClickHouse/pull/41014) ([Igor Nikonov](https://github.com/devcrafter)). +* ColumnVector: optimize UInt8 index with AVX512VBMI. [#41247](https://github.com/ClickHouse/ClickHouse/pull/41247) ([Guo Wangyang](https://github.com/guowangy)). +* Optimize the lock contentions for `ThreadGroupStatus::mutex`. The performance experiments of **SSB** (Star Schema Benchmark) on the ICX device (Intel Xeon Platinum 8380 CPU, 80 cores, 160 threads) shows that this change could bring a **2.95x** improvement of the geomean of all subcases' QPS. [#41675](https://github.com/ClickHouse/ClickHouse/pull/41675) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). +* Add `ldapr` capabilities to AArch64 builds. This is supported from Graviton 2+, Azure and GCP instances. Only appeared in clang-15 [not so long ago](https://github.com/llvm/llvm-project/commit/9609b5daffe9fd28d83d83da895abc5113f76c24). [#41778](https://github.com/ClickHouse/ClickHouse/pull/41778) ([Daniel Kutenin](https://github.com/danlark1)). +* Improve performance when comparing strings and one argument is empty constant string. [#41870](https://github.com/ClickHouse/ClickHouse/pull/41870) ([Jiebin Sun](https://github.com/jiebinn)). +* Optimize `insertFrom` of ColumnAggregateFunction to share Aggregate State in some cases. [#41960](https://github.com/ClickHouse/ClickHouse/pull/41960) ([flynn](https://github.com/ucasfl)). +* Make writing to `azure_blob_storage` disks faster (respect `max_single_part_upload_size` instead of writing a block per each buffer size). Inefficiency mentioned in [#41754](https://github.com/ClickHouse/ClickHouse/issues/41754). [#42041](https://github.com/ClickHouse/ClickHouse/pull/42041) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Make thread ids in the process list and query_log unique to avoid waste. [#42180](https://github.com/ClickHouse/ClickHouse/pull/42180) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support skipping cache completely (both download to cache and reading cached data) in case the requested read range exceeds the threshold defined by cache setting `bypass_cache_threashold`, requires to be enabled with `enable_bypass_cache_with_threshold`). [#42418](https://github.com/ClickHouse/ClickHouse/pull/42418) ([Han Shukai](https://github.com/KinderRiven)). This helps on slow local disks. + +#### Improvement +* Add setting `allow_implicit_no_password`: with combination with `allow_no_password` it forbids creating a user with no password unless `IDENTIFIED WITH no_password` is explicitly specified. [#41341](https://github.com/ClickHouse/ClickHouse/pull/41341) ([Nikolay Degterinsky](https://github.com/evillique)). +* Embedded Keeper will always start in the background allowing ClickHouse to start without achieving quorum. [#40991](https://github.com/ClickHouse/ClickHouse/pull/40991) ([Antonio Andelic](https://github.com/antonio2368)). +* Made reestablishing a new connection to ZooKeeper more reactive in case of expiration of the previous one. Previously there was a task which spawns every minute by default and thus a table could be in readonly state for about this time. [#41092](https://github.com/ClickHouse/ClickHouse/pull/41092) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Now projections can be used with zero copy replication (zero-copy replication is a non-production feature). [#41147](https://github.com/ClickHouse/ClickHouse/pull/41147) ([alesapin](https://github.com/alesapin)). +* Support expression `(EXPLAIN SELECT ...)` in a subquery. Queries like `SELECT * FROM (EXPLAIN PIPELINE SELECT col FROM TABLE ORDER BY col)` became valid. [#40630](https://github.com/ClickHouse/ClickHouse/pull/40630) ([Vladimir C](https://github.com/vdimir)). +* Allow changing `async_insert_max_data_size` or `async_insert_busy_timeout_ms` in scope of query. E.g. user wants to insert data rarely and she doesn't have an access to server config to tune default settings. [#40668](https://github.com/ClickHouse/ClickHouse/pull/40668) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Improvements for reading from remote filesystems, made threadpool size for reads/writes configurable. Closes [#41070](https://github.com/ClickHouse/ClickHouse/issues/41070). [#41011](https://github.com/ClickHouse/ClickHouse/pull/41011) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Support all combinators combination in WindowTransform/arratReduce*/initializeAggregation/aggregate functions versioning. Previously combinators like `ForEach/Resample/Map` didn't work in these places, using them led to exception like`State function ... inserts results into non-state column`. [#41107](https://github.com/ClickHouse/ClickHouse/pull/41107) ([Kruglov Pavel](https://github.com/Avogar)). +* Add function `tryDecrypt` that returns NULL when decrypt fail (e.g. decrypt with incorrect key) instead of throwing exception. [#41206](https://github.com/ClickHouse/ClickHouse/pull/41206) ([Duc Canh Le](https://github.com/canhld94)). +* Add the `unreserved_space` column to the `system.disks` table to check how much space is not taken by reservations per disk. [#41254](https://github.com/ClickHouse/ClickHouse/pull/41254) ([filimonov](https://github.com/filimonov)). +* Support s3 authorization headers in table function arguments. [#41261](https://github.com/ClickHouse/ClickHouse/pull/41261) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add support for MultiRead in Keeper and internal ZooKeeper client (this is an extension to ZooKeeper protocol, only available in ClickHouse Keeper). [#41410](https://github.com/ClickHouse/ClickHouse/pull/41410) ([Antonio Andelic](https://github.com/antonio2368)). +* Add support for decimal type comparing with floating point literal in IN operator. [#41544](https://github.com/ClickHouse/ClickHouse/pull/41544) ([liang.huang](https://github.com/lhuang09287750)). +* Allow readable size values (like `1TB`) in cache config. [#41688](https://github.com/ClickHouse/ClickHouse/pull/41688) ([Kseniia Sumarokova](https://github.com/kssenii)). +* ClickHouse could cache stale DNS entries for some period of time (15 seconds by default) until the cache won't be updated asynchronously. During these period ClickHouse can nevertheless try to establish a connection and produce errors. This behaviour is fixed. [#41707](https://github.com/ClickHouse/ClickHouse/pull/41707) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add interactive history search with fzf-like utility (fzf/sk) for `clickhouse-client`/`clickhouse-local` (note you can use `FZF_DEFAULT_OPTS`/`SKIM_DEFAULT_OPTIONS` to additionally configure the behavior). [#41730](https://github.com/ClickHouse/ClickHouse/pull/41730) ([Azat Khuzhin](https://github.com/azat)). +* For client when connecting to a secure server with invalid certificate only allow to proceed with '--accept-certificate' flag. [#41743](https://github.com/ClickHouse/ClickHouse/pull/41743) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Add function `tryBase58Decode`, similar to the existing function `tryBase64Decode`. [#41824](https://github.com/ClickHouse/ClickHouse/pull/41824) ([Robert Schulze](https://github.com/rschu1ze)). +* Improve feedback when replacing partition with different primary key. Fixes [#34798](https://github.com/ClickHouse/ClickHouse/issues/34798). [#41838](https://github.com/ClickHouse/ClickHouse/pull/41838) ([Salvatore](https://github.com/tbsal)). +* Fix parallel parsing: segmentator now checks `max_block_size`. This fixed memory overallocation in case of parallel parsing and small LIMIT. [#41852](https://github.com/ClickHouse/ClickHouse/pull/41852) ([Vitaly Baranov](https://github.com/vitlibar)). +* Don't add "TABLE_IS_DROPPED" exception to `system.errors` if it's happened during SELECT from a system table and was ignored. [#41908](https://github.com/ClickHouse/ClickHouse/pull/41908) ([AlfVII](https://github.com/AlfVII)). +* Improve option `enable_extended_results_for_datetime_functions` to return results of type DateTime64 for functions `toStartOfDay`, `toStartOfHour`, `toStartOfFifteenMinutes`, `toStartOfTenMinutes`, `toStartOfFiveMinutes`, `toStartOfMinute` and `timeSlot`. [#41910](https://github.com/ClickHouse/ClickHouse/pull/41910) ([Roman Vasin](https://github.com/rvasin)). +* Improve `DateTime` type inference for text formats. Now it respect setting `date_time_input_format` and doesn't try to infer datetimes from numbers as timestamps. Closes [#41389](https://github.com/ClickHouse/ClickHouse/issues/41389) Closes [#42206](https://github.com/ClickHouse/ClickHouse/issues/42206). [#41912](https://github.com/ClickHouse/ClickHouse/pull/41912) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove confusing warning when inserting with `perform_ttl_move_on_insert` = false. [#41980](https://github.com/ClickHouse/ClickHouse/pull/41980) ([Vitaly Baranov](https://github.com/vitlibar)). +* Allow user to write `countState(*)` similar to `count(*)`. This closes [#9338](https://github.com/ClickHouse/ClickHouse/issues/9338). [#41983](https://github.com/ClickHouse/ClickHouse/pull/41983) ([Amos Bird](https://github.com/amosbird)). +* Fix `rankCorr` size overflow. [#42020](https://github.com/ClickHouse/ClickHouse/pull/42020) ([Duc Canh Le](https://github.com/canhld94)). +* Added an option to specify an arbitrary string as an environment name in the Sentry's config for more handy reports. [#42037](https://github.com/ClickHouse/ClickHouse/pull/42037) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix parsing out-of-range Date from CSV. [#42044](https://github.com/ClickHouse/ClickHouse/pull/42044) ([Andrey Zvonov](https://github.com/zvonand)). +* `parseDataTimeBestEffort` now supports comma between date and time. Closes [#42038](https://github.com/ClickHouse/ClickHouse/issues/42038). [#42049](https://github.com/ClickHouse/ClickHouse/pull/42049) ([flynn](https://github.com/ucasfl)). +* Improved stale replica recovery process for `ReplicatedMergeTree`. If a lost replica have some parts which absent on a healthy replica, but these parts should appear in future according to replication queue of the healthy replica, then lost replica will keep such parts instead of detaching them. [#42134](https://github.com/ClickHouse/ClickHouse/pull/42134) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add a possibility to use `Date32` arguments for date_diff function. Fix issue in date_diff function when using DateTime64 arguments with start date before Unix epoch and end date after Unix epoch. [#42308](https://github.com/ClickHouse/ClickHouse/pull/42308) ([Roman Vasin](https://github.com/rvasin)). +* When uploading big parts to Minio, 'Complete Multipart Upload' can take a long time. Minio sends heartbeats every 10 seconds (see https://github.com/minio/minio/pull/7198). But clickhouse times out earlier, because the default send/receive timeout is [set](https://github.com/ClickHouse/ClickHouse/blob/cc24fcd6d5dfb67f5f66f5483e986bd1010ad9cf/src/IO/S3/PocoHTTPClient.cpp#L123) to 5 seconds. [#42321](https://github.com/ClickHouse/ClickHouse/pull/42321) ([filimonov](https://github.com/filimonov)). +* Fix rarely invalid cast of aggregate state types with complex types such as Decimal. This fixes [#42408](https://github.com/ClickHouse/ClickHouse/issues/42408). [#42417](https://github.com/ClickHouse/ClickHouse/pull/42417) ([Amos Bird](https://github.com/amosbird)). +* Allow to use `Date32` arguments for `dateName` function. [#42554](https://github.com/ClickHouse/ClickHouse/pull/42554) ([Roman Vasin](https://github.com/rvasin)). +* Now filters with NULL literals will be used during index analysis. [#34063](https://github.com/ClickHouse/ClickHouse/issues/34063). [#41842](https://github.com/ClickHouse/ClickHouse/pull/41842) ([Amos Bird](https://github.com/amosbird)). + +#### Build/Testing/Packaging Improvement +* Add fuzzer for table definitions [#40096](https://github.com/ClickHouse/ClickHouse/pull/40096) ([Anton Popov](https://github.com/CurtizJ)). This represents the biggest advancement for ClickHouse testing in this year so far. +* Beta version of the ClickHouse Cloud service is released: [https://clickhouse.cloud/](https://clickhouse.cloud/). It provides the easiest way to use ClickHouse (even slightly easier than the single-command installation). +* Added support of WHERE clause generation to AST Fuzzer and possibility to add or remove ORDER BY and WHERE clause. [#38519](https://github.com/ClickHouse/ClickHouse/pull/38519) ([Ilya Yatsishin](https://github.com/qoega)). +* Aarch64 binaries now require at least ARMv8.2, released in 2016. Most notably, this enables use of ARM LSE, i.e. native atomic operations. Also, CMake build option "NO_ARMV81_OR_HIGHER" has been added to allow compilation of binaries for older ARMv8.0 hardware, e.g. Raspberry Pi 4. [#41610](https://github.com/ClickHouse/ClickHouse/pull/41610) ([Robert Schulze](https://github.com/rschu1ze)). +* Allow building ClickHouse with Musl (small changes after it was already supported but broken). [#41987](https://github.com/ClickHouse/ClickHouse/pull/41987) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add the `$CLICKHOUSE_CRONFILE` file checking to avoid running the `sed` command to get the file not found error on install. [#42081](https://github.com/ClickHouse/ClickHouse/pull/42081) ([Chun-Sheng, Li](https://github.com/peter279k)). +* Update cctz to `2022e` to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)). +* Add Rust code support into ClickHouse with BLAKE3 hash-function library as an example. [#33435](https://github.com/ClickHouse/ClickHouse/pull/33435) ([BoloniniD](https://github.com/BoloniniD)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Choose correct aggregation method for `LowCardinality` with big integer types. [#42342](https://github.com/ClickHouse/ClickHouse/pull/42342) ([Duc Canh Le](https://github.com/canhld94)). +* Several fixes for `web` disk. [#41652](https://github.com/ClickHouse/ClickHouse/pull/41652) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixes issue when docker run will fail if `https_port` is not present in config. [#41693](https://github.com/ClickHouse/ClickHouse/pull/41693) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Mutations were not cancelled properly on server shutdown or `SYSTEM STOP MERGES` query and cancellation might take long time, it's fixed. [#41699](https://github.com/ClickHouse/ClickHouse/pull/41699) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix wrong result of queries with `ORDER BY` or `GROUP BY` by columns from prefix of sorting key, wrapped into monotonic functions, with enable "read in order" optimization (settings `optimize_read_in_order` and `optimize_aggregation_in_order`). [#41701](https://github.com/ClickHouse/ClickHouse/pull/41701) ([Anton Popov](https://github.com/CurtizJ)). +* Fix possible crash in `SELECT` from `Merge` table with enabled `optimize_monotonous_functions_in_order_by` setting. Fixes [#41269](https://github.com/ClickHouse/ClickHouse/issues/41269). [#41740](https://github.com/ClickHouse/ClickHouse/pull/41740) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed "Part ... intersects part ..." error that might happen in extremely rare cases if replica was restarted just after detaching some part as broken. [#41741](https://github.com/ClickHouse/ClickHouse/pull/41741) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Don't allow to create or alter merge tree tables with column name `_row_exists`, which is reserved for lightweight delete. Fixed [#41716](https://github.com/ClickHouse/ClickHouse/issues/41716). [#41763](https://github.com/ClickHouse/ClickHouse/pull/41763) ([Jianmei Zhang](https://github.com/zhangjmruc)). +* Fix a bug that CORS headers are missing in some HTTP responses. [#41792](https://github.com/ClickHouse/ClickHouse/pull/41792) ([Frank Chen](https://github.com/FrankChen021)). +* 22.9 might fail to startup `ReplicatedMergeTree` table if that table was created by 20.3 or older version and was never altered, it's fixed. Fixes [#41742](https://github.com/ClickHouse/ClickHouse/issues/41742). [#41796](https://github.com/ClickHouse/ClickHouse/pull/41796) ([Alexander Tokmakov](https://github.com/tavplubix)). +* When the batch sending fails for some reason, it cannot be automatically recovered, and if it is not processed in time, it will lead to accumulation, and the printed error message will become longer and longer, which will cause the http thread to block. [#41813](https://github.com/ClickHouse/ClickHouse/pull/41813) ([zhongyuankai](https://github.com/zhongyuankai)). +* Fix compact parts with compressed marks setting. Fixes [#41783](https://github.com/ClickHouse/ClickHouse/issues/41783) and [#41746](https://github.com/ClickHouse/ClickHouse/issues/41746). [#41823](https://github.com/ClickHouse/ClickHouse/pull/41823) ([alesapin](https://github.com/alesapin)). +* Old versions of Replicated database doesn't have a special marker in [Zoo]Keeper. We need to check only whether the node contains come obscure data instead of special mark. [#41875](https://github.com/ClickHouse/ClickHouse/pull/41875) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix possible exception in fs cache. [#41884](https://github.com/ClickHouse/ClickHouse/pull/41884) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix `use_environment_credentials` for s3 table function. [#41970](https://github.com/ClickHouse/ClickHouse/pull/41970) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed "Directory already exists and is not empty" error on detaching broken part that might prevent `ReplicatedMergeTree` table from starting replication. Fixes [#40957](https://github.com/ClickHouse/ClickHouse/issues/40957). [#41981](https://github.com/ClickHouse/ClickHouse/pull/41981) ([Alexander Tokmakov](https://github.com/tavplubix)). +* `toDateTime64` now returns the same output with negative integer and float arguments. [#42025](https://github.com/ClickHouse/ClickHouse/pull/42025) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix write into `azure_blob_storage`. Partially closes [#41754](https://github.com/ClickHouse/ClickHouse/issues/41754). [#42034](https://github.com/ClickHouse/ClickHouse/pull/42034) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix the `bzip2` decoding issue for specific `bzip2` files. [#42046](https://github.com/ClickHouse/ClickHouse/pull/42046) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix SQL function `toLastDayOfMonth` with setting "enable_extended_results_for_datetime_functions = 1" at the beginning of the extended range (January 1900). - Fix SQL function "toRelativeWeekNum()" with setting "enable_extended_results_for_datetime_functions = 1" at the end of extended range (December 2299). - Improve the performance of for SQL functions "toISOYear()", "toFirstDayNumOfISOYearIndex()" and "toYearWeekOfNewyearMode()" by avoiding unnecessary index arithmetics. [#42084](https://github.com/ClickHouse/ClickHouse/pull/42084) ([Roman Vasin](https://github.com/rvasin)). +* The maximum size of fetches for each table accidentally was set to 8 while the pool size could be bigger. Now the maximum size of fetches for table is equal to the pool size. [#42090](https://github.com/ClickHouse/ClickHouse/pull/42090) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* A table might be shut down and a dictionary might be detached before checking if can be dropped without breaking dependencies between table, it's fixed. Fixes [#41982](https://github.com/ClickHouse/ClickHouse/issues/41982). [#42106](https://github.com/ClickHouse/ClickHouse/pull/42106) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix bad inefficiency of `remote_filesystem_read_method=read` with filesystem cache. Closes [#42125](https://github.com/ClickHouse/ClickHouse/issues/42125). [#42129](https://github.com/ClickHouse/ClickHouse/pull/42129) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible timeout exception for distributed queries with use_hedged_requests = 0. [#42130](https://github.com/ClickHouse/ClickHouse/pull/42130) ([Azat Khuzhin](https://github.com/azat)). +* Fixed a minor bug inside function `runningDifference` in case of using it with `Date32` type. Previously `Date` was used and it may cause some logical errors like `Bad cast from type DB::ColumnVector to DB::ColumnVector'`. [#42143](https://github.com/ClickHouse/ClickHouse/pull/42143) ([Alfred Xu](https://github.com/sperlingxx)). +* Fix reusing of files > 4GB from base backup. [#42146](https://github.com/ClickHouse/ClickHouse/pull/42146) ([Azat Khuzhin](https://github.com/azat)). +* DISTINCT in order fails with LOGICAL_ERROR if first column in sorting key contains function. [#42186](https://github.com/ClickHouse/ClickHouse/pull/42186) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix a bug with projections and the `aggregate_functions_null_for_empty` setting. This bug is very rare and appears only if you enable the `aggregate_functions_null_for_empty` setting in the server's config. This closes [#41647](https://github.com/ClickHouse/ClickHouse/issues/41647). [#42198](https://github.com/ClickHouse/ClickHouse/pull/42198) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix read from `Buffer` tables with read in order desc. [#42236](https://github.com/ClickHouse/ClickHouse/pull/42236) ([Duc Canh Le](https://github.com/canhld94)). +* Fix a bug which prevents ClickHouse to start when `background_pool_size setting` is set on default profile but `background_merges_mutations_concurrency_ratio` is not. [#42315](https://github.com/ClickHouse/ClickHouse/pull/42315) ([nvartolomei](https://github.com/nvartolomei)). +* `ALTER UPDATE` of attached part (with columns different from table schema) could create an invalid `columns.txt` metadata on disk. Reading from such part could fail with errors or return invalid data. Fixes [#42161](https://github.com/ClickHouse/ClickHouse/issues/42161). [#42319](https://github.com/ClickHouse/ClickHouse/pull/42319) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Setting `additional_table_filters` were not applied to `Distributed` storage. Fixes [#41692](https://github.com/ClickHouse/ClickHouse/issues/41692). [#42322](https://github.com/ClickHouse/ClickHouse/pull/42322) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix a data race in query finish/cancel. This closes [#42346](https://github.com/ClickHouse/ClickHouse/issues/42346). [#42362](https://github.com/ClickHouse/ClickHouse/pull/42362) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* This reverts [#40217](https://github.com/ClickHouse/ClickHouse/issues/40217) which introduced a regression in date/time functions. [#42367](https://github.com/ClickHouse/ClickHouse/pull/42367) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix assert cast in join on falsy condition, Close [#42380](https://github.com/ClickHouse/ClickHouse/issues/42380). [#42407](https://github.com/ClickHouse/ClickHouse/pull/42407) ([Vladimir C](https://github.com/vdimir)). +* Fix buffer overflow in the processing of Decimal data types. This closes [#42451](https://github.com/ClickHouse/ClickHouse/issues/42451). [#42465](https://github.com/ClickHouse/ClickHouse/pull/42465) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* `AggregateFunctionQuantile` now correctly works with UInt128 columns. Previously, the quantile state interpreted `UInt128` columns as `Int128` which could have led to incorrect results. [#42473](https://github.com/ClickHouse/ClickHouse/pull/42473) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix bad_cast assert during INSERT into `Annoy` indexes over non-Float32 columns. `Annoy` indices is an experimental feature. [#42485](https://github.com/ClickHouse/ClickHouse/pull/42485) ([Robert Schulze](https://github.com/rschu1ze)). +* Arithmetic operator with Date or DateTime and 128 or 256-bit integer was referencing uninitialized memory. [#42453](https://github.com/ClickHouse/ClickHouse/issues/42453). [#42573](https://github.com/ClickHouse/ClickHouse/pull/42573) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix unexpected table loading error when partition key contains alias function names during server upgrade. [#36379](https://github.com/ClickHouse/ClickHouse/pull/36379) ([Amos Bird](https://github.com/amosbird)). + + ### ClickHouse release 22.9, 2022-09-22 #### Backward Incompatible Change From b15850984f0aada23c9281f45b33488665995d86 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Oct 2022 11:22:07 +0300 Subject: [PATCH 18/50] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c7ca701d53..b02d3d9ac84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ #### New Feature * Composable protocol configuration is added. Now different protocols can be set up with different listen hosts. Protocol wrappers such as PROXYv1 can be set up over any other protocols (TCP, TCP secure, MySQL, Postgres). [#41198](https://github.com/ClickHouse/ClickHouse/pull/41198) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Add `S3` as a new type of the destination of backups. Support BACKUP to S3 with as-is path/data structure. [#42333](https://github.com/ClickHouse/ClickHouse/pull/42333) ([Vitaly Baranov](https://github.com/vitlibar)), [#42232](https://github.com/ClickHouse/ClickHouse/pull/42232) ([Azat Khuzhin](https://github.com/azat)). -* Added functions (`randUniform`, `randNormal`, `randLogNormal`, `randExponential`, `randChiSquared`, `randStudentT`, `randFisherF`, `randBernoulli`, `randBinomial`, `randNegativeBinomial`, `randPoisson` ) to generate random values according to the specified distributions. This closes [#21834](https://github.com/ClickHouse/ClickHouse/issues/21834). [#42411](https://github.com/ClickHouse/ClickHouse/pull/42411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Added functions (`randUniform`, `randNormal`, `randLogNormal`, `randExponential`, `randChiSquared`, `randStudentT`, `randFisherF`, `randBernoulli`, `randBinomial`, `randNegativeBinomial`, `randPoisson`) to generate random values according to the specified distributions. This closes [#21834](https://github.com/ClickHouse/ClickHouse/issues/21834). [#42411](https://github.com/ClickHouse/ClickHouse/pull/42411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * An improvement for ClickHouse Keeper: add support for uploading snapshots to S3. S3 information can be defined inside `keeper_server.s3_snapshot`. [#41342](https://github.com/ClickHouse/ClickHouse/pull/41342) ([Antonio Andelic](https://github.com/antonio2368)). * Added an aggregate function `analysisOfVariance` (`anova`) to perform a statistical test over several groups of normally distributed observations to find out whether all groups have the same mean or not. Original PR [#37872](https://github.com/ClickHouse/ClickHouse/issues/37872). [#42131](https://github.com/ClickHouse/ClickHouse/pull/42131) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Support limiting of temporary data stored on disk using settings `max_temporary_data_on_disk_size_for_user`/`max_temporary_data_on_disk_size_for_query` . [#40893](https://github.com/ClickHouse/ClickHouse/pull/40893) ([Vladimir C](https://github.com/vdimir)). From 5ebdf847ff444a59fa653d6e1b1410aad31baf95 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Oct 2022 11:27:31 +0300 Subject: [PATCH 19/50] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b02d3d9ac84..f2b8e9004b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,7 @@ #### Performance Improvement * Relax the "Too many parts" threshold. This closes [#6551](https://github.com/ClickHouse/ClickHouse/issues/6551). Now ClickHouse will allow more parts in a partition if the average part size is large enough (at least 10 GiB). This allows to have up to petabytes of data in a single partition of a single table on a single server, which is possible using disk shelves or object storage. [#42002](https://github.com/ClickHouse/ClickHouse/pull/42002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Implement operator precedence element parser to resolve stack overflow issues and make the required stack size smaller. [#34892](https://github.com/ClickHouse/ClickHouse/pull/34892) ([Nikolay Degterinsky](https://github.com/evillique)). +* Implement operator precedence element parser to make the required stack size smaller. [#34892](https://github.com/ClickHouse/ClickHouse/pull/34892) ([Nikolay Degterinsky](https://github.com/evillique)). * DISTINCT in order optimization leverage sorting properties of data streams. This improvement will enable reading in order for DISTINCT if applicable (before it was necessary to provide ORDER BY for columns in DISTINCT). [#41014](https://github.com/ClickHouse/ClickHouse/pull/41014) ([Igor Nikonov](https://github.com/devcrafter)). * ColumnVector: optimize UInt8 index with AVX512VBMI. [#41247](https://github.com/ClickHouse/ClickHouse/pull/41247) ([Guo Wangyang](https://github.com/guowangy)). * Optimize the lock contentions for `ThreadGroupStatus::mutex`. The performance experiments of **SSB** (Star Schema Benchmark) on the ICX device (Intel Xeon Platinum 8380 CPU, 80 cores, 160 threads) shows that this change could bring a **2.95x** improvement of the geomean of all subcases' QPS. [#41675](https://github.com/ClickHouse/ClickHouse/pull/41675) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). From 8f00d0d1cc709733b2571b23cba6bead445fed72 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Mon, 24 Oct 2022 12:56:36 +0200 Subject: [PATCH 20/50] fix tidy --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 164ec8777de..ba7b0e963eb 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -253,17 +253,19 @@ ProcessorPtr ReadFromMergeTree::createSource( if (query_info.limit > 0 && query_info.limit < total_rows) total_rows = query_info.limit; - auto source = std::make_shared( - data, storage_snapshot, part.data_part, max_block_size, preferred_block_size_bytes, - preferred_max_column_in_block_size_bytes, required_columns, part.ranges, use_uncompressed_cache, prewhere_info, - actions_settings, reader_settings, virt_column_names, part.part_index_in_query, has_limit_below_one_block, std::move(extension)); - /// Actually it means that parallel reading from replicas enabled /// and we have to collaborate with initiator. /// In this case we won't set approximate rows, because it will be accounted multiple times. /// Also do not count amount of read rows if we read in order of sorting key, /// because we don't know actual amount of read rows in case when limit is set. - if (!extension.has_value() && !reader_settings.read_in_order) + bool set_rows_approx = !extension.has_value() && !reader_settings.read_in_order; + + auto source = std::make_shared( + data, storage_snapshot, part.data_part, max_block_size, preferred_block_size_bytes, + preferred_max_column_in_block_size_bytes, required_columns, part.ranges, use_uncompressed_cache, prewhere_info, + actions_settings, reader_settings, virt_column_names, part.part_index_in_query, has_limit_below_one_block, std::move(extension)); + + if (set_rows_approx) source -> addTotalRowsApprox(total_rows); return source; From 14e3bb6354f4acf4341731b67ba6a626752c69e3 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 24 Oct 2022 13:58:26 +0000 Subject: [PATCH 21/50] Another case. --- src/Functions/FunctionBinaryArithmetic.h | 6 ++-- ...461_mullable_pk_monotonicity_bug.reference | 32 +++++++++++++++++++ .../02461_mullable_pk_monotonicity_bug.sql | 32 ++++++++++++++++--- 3 files changed, 63 insertions(+), 7 deletions(-) diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 3ffe054a439..1c1d0453230 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -1781,6 +1781,7 @@ public: { auto left_type = removeNullable(removeLowCardinality(left.type)); auto right_type = removeNullable(removeLowCardinality(right.type)); + auto ret_type = removeNullable(removeLowCardinality(return_type)); auto transform = [&](const Field & point) { @@ -1791,7 +1792,7 @@ public: /// This is a bit dangerous to call Base::executeImpl cause it ignores `use Default Implementation For XXX` flags. /// It was possible to check monotonicity for nullable right type which result to exception. /// Adding removeNullable above fixes the issue, but some other inconsistency may left. - auto col = Base::executeImpl(columns_with_constant, return_type, 1); + auto col = Base::executeImpl(columns_with_constant, ret_type, 1); Field point_transformed; col->get(0, point_transformed); return point_transformed; @@ -1822,6 +1823,7 @@ public: { auto left_type = removeNullable(removeLowCardinality(left.type)); auto right_type = removeNullable(removeLowCardinality(right.type)); + auto ret_type = removeNullable(removeLowCardinality(return_type)); auto transform = [&](const Field & point) { @@ -1829,7 +1831,7 @@ public: = {{left_type->createColumnConst(1, point), left_type, left.name}, {right_type->createColumnConst(1, (*right.column)[0]), right_type, right.name}}; - auto col = Base::executeImpl(columns_with_constant, return_type, 1); + auto col = Base::executeImpl(columns_with_constant, ret_type, 1); Field point_transformed; col->get(0, point_transformed); return point_transformed; diff --git a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference index 5ee6e6c67c4..29da9bc651b 100644 --- a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference +++ b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference @@ -30,3 +30,35 @@ 2 1 2 +1 +2 +1 +2 +1 +2 +1 +2 +1 +2 +1 +2 +1 +2 +1 +2 +1970-01-01 00:00:01 +1970-01-01 00:00:02 +1970-01-01 00:00:01 +1970-01-01 00:00:02 +1970-01-01 00:00:01 +1970-01-01 00:00:02 +1970-01-01 00:00:01 +1970-01-01 00:00:02 +1970-01-01 00:00:01 +1970-01-01 00:00:02 +1970-01-01 00:00:01 +1970-01-01 00:00:02 +1970-01-01 00:00:01 +1970-01-01 00:00:02 +1970-01-01 00:00:01 +1970-01-01 00:00:02 diff --git a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql index ce190694514..af2d5b859b3 100644 --- a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql +++ b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql @@ -1,6 +1,5 @@ create table tab (x Nullable(UInt8)) engine = MergeTree order by x settings allow_nullable_key = 1, index_granularity = 2; insert into tab select number from numbers(4); - set allow_suspicious_low_cardinality_types=1; set max_rows_to_read = 2; @@ -14,9 +13,7 @@ SELECT 1 + x FROM tab where plus(1::LowCardinality(UInt8), x) <= 2 order by x; SELECT 1 + x FROM tab where plus(1::LowCardinality(Nullable(UInt8)), x) <= 2 order by x; drop table tab; - set max_rows_to_read = 100; - create table tab (x LowCardinality(UInt8)) engine = MergeTree order by x settings allow_nullable_key = 1, index_granularity = 2; insert into tab select number from numbers(4); @@ -31,10 +28,35 @@ SELECT 1 + x FROM tab where plus(1::LowCardinality(UInt8), x) <= 2 order by x; SELECT 1 + x FROM tab where plus(1::LowCardinality(Nullable(UInt8)), x) <= 2 order by x; drop table tab; - set max_rows_to_read = 100; - create table tab (x UInt128) engine = MergeTree order by x settings allow_nullable_key = 1, index_granularity = 2; insert into tab select number from numbers(4); +set max_rows_to_read = 2; +SELECT x + 1 FROM tab where plus(x, 1) <= 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::Nullable(UInt8)) <= 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::LowCardinality(UInt8)) <= 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::LowCardinality(Nullable(UInt8))) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1, x) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1::Nullable(UInt8), x) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1::LowCardinality(UInt8), x) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1::LowCardinality(Nullable(UInt8)), x) <= 2 order by x; + +set max_rows_to_read = 100; SELECT x + 1 FROM tab WHERE (x + 1::LowCardinality(UInt8)) <= -9223372036854775808 order by x; + +drop table tab; +create table tab (x DateTime) engine = MergeTree order by x settings allow_nullable_key = 1, index_granularity = 2; +insert into tab select number from numbers(4); + +set max_rows_to_read = 2; +SELECT x + 1 FROM tab where plus(x, 1) <= 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::Nullable(UInt8)) <= 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::LowCardinality(UInt8)) <= 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::LowCardinality(Nullable(UInt8))) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1, x) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1::Nullable(UInt8), x) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1::LowCardinality(UInt8), x) <= 2 order by x; +SELECT 1 + x FROM tab where plus(1::LowCardinality(Nullable(UInt8)), x) <= 2 order by x; + +SELECT x + 1 FROM tab WHERE (x + CAST('1', 'Nullable(UInt8)')) <= -2147483647 ORDER BY x ASC NULLS FIRST; From 9532b6024bcfc81c57c7064a702b6ed587e76afd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Oct 2022 19:02:05 +0300 Subject: [PATCH 22/50] Update CHANGELOG.md Co-authored-by: Dan Roscigno --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2b8e9004b0..ea568985d31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,7 +42,7 @@ * ColumnVector: optimize UInt8 index with AVX512VBMI. [#41247](https://github.com/ClickHouse/ClickHouse/pull/41247) ([Guo Wangyang](https://github.com/guowangy)). * Optimize the lock contentions for `ThreadGroupStatus::mutex`. The performance experiments of **SSB** (Star Schema Benchmark) on the ICX device (Intel Xeon Platinum 8380 CPU, 80 cores, 160 threads) shows that this change could bring a **2.95x** improvement of the geomean of all subcases' QPS. [#41675](https://github.com/ClickHouse/ClickHouse/pull/41675) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). * Add `ldapr` capabilities to AArch64 builds. This is supported from Graviton 2+, Azure and GCP instances. Only appeared in clang-15 [not so long ago](https://github.com/llvm/llvm-project/commit/9609b5daffe9fd28d83d83da895abc5113f76c24). [#41778](https://github.com/ClickHouse/ClickHouse/pull/41778) ([Daniel Kutenin](https://github.com/danlark1)). -* Improve performance when comparing strings and one argument is empty constant string. [#41870](https://github.com/ClickHouse/ClickHouse/pull/41870) ([Jiebin Sun](https://github.com/jiebinn)). +* Improve performance when comparing strings and one argument is an empty constant string. [#41870](https://github.com/ClickHouse/ClickHouse/pull/41870) ([Jiebin Sun](https://github.com/jiebinn)). * Optimize `insertFrom` of ColumnAggregateFunction to share Aggregate State in some cases. [#41960](https://github.com/ClickHouse/ClickHouse/pull/41960) ([flynn](https://github.com/ucasfl)). * Make writing to `azure_blob_storage` disks faster (respect `max_single_part_upload_size` instead of writing a block per each buffer size). Inefficiency mentioned in [#41754](https://github.com/ClickHouse/ClickHouse/issues/41754). [#42041](https://github.com/ClickHouse/ClickHouse/pull/42041) ([Kseniia Sumarokova](https://github.com/kssenii)). * Make thread ids in the process list and query_log unique to avoid waste. [#42180](https://github.com/ClickHouse/ClickHouse/pull/42180) ([Alexey Milovidov](https://github.com/alexey-milovidov)). From 182925071119174f783529f90ebcb09cda4dd4f9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Oct 2022 19:02:16 +0300 Subject: [PATCH 23/50] Update CHANGELOG.md Co-authored-by: Dan Roscigno --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea568985d31..192dafd20ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,7 +49,7 @@ * Support skipping cache completely (both download to cache and reading cached data) in case the requested read range exceeds the threshold defined by cache setting `bypass_cache_threashold`, requires to be enabled with `enable_bypass_cache_with_threshold`). [#42418](https://github.com/ClickHouse/ClickHouse/pull/42418) ([Han Shukai](https://github.com/KinderRiven)). This helps on slow local disks. #### Improvement -* Add setting `allow_implicit_no_password`: with combination with `allow_no_password` it forbids creating a user with no password unless `IDENTIFIED WITH no_password` is explicitly specified. [#41341](https://github.com/ClickHouse/ClickHouse/pull/41341) ([Nikolay Degterinsky](https://github.com/evillique)). +* Add setting `allow_implicit_no_password`: in combination with `allow_no_password` it forbids creating a user with no password unless `IDENTIFIED WITH no_password` is explicitly specified. [#41341](https://github.com/ClickHouse/ClickHouse/pull/41341) ([Nikolay Degterinsky](https://github.com/evillique)). * Embedded Keeper will always start in the background allowing ClickHouse to start without achieving quorum. [#40991](https://github.com/ClickHouse/ClickHouse/pull/40991) ([Antonio Andelic](https://github.com/antonio2368)). * Made reestablishing a new connection to ZooKeeper more reactive in case of expiration of the previous one. Previously there was a task which spawns every minute by default and thus a table could be in readonly state for about this time. [#41092](https://github.com/ClickHouse/ClickHouse/pull/41092) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Now projections can be used with zero copy replication (zero-copy replication is a non-production feature). [#41147](https://github.com/ClickHouse/ClickHouse/pull/41147) ([alesapin](https://github.com/alesapin)). From 440d302a8c5c286a7394bd747f3b0af16a427bff Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Oct 2022 19:02:28 +0300 Subject: [PATCH 24/50] Update CHANGELOG.md Co-authored-by: Dan Roscigno --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 192dafd20ff..8049037c99b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,7 +54,7 @@ * Made reestablishing a new connection to ZooKeeper more reactive in case of expiration of the previous one. Previously there was a task which spawns every minute by default and thus a table could be in readonly state for about this time. [#41092](https://github.com/ClickHouse/ClickHouse/pull/41092) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Now projections can be used with zero copy replication (zero-copy replication is a non-production feature). [#41147](https://github.com/ClickHouse/ClickHouse/pull/41147) ([alesapin](https://github.com/alesapin)). * Support expression `(EXPLAIN SELECT ...)` in a subquery. Queries like `SELECT * FROM (EXPLAIN PIPELINE SELECT col FROM TABLE ORDER BY col)` became valid. [#40630](https://github.com/ClickHouse/ClickHouse/pull/40630) ([Vladimir C](https://github.com/vdimir)). -* Allow changing `async_insert_max_data_size` or `async_insert_busy_timeout_ms` in scope of query. E.g. user wants to insert data rarely and she doesn't have an access to server config to tune default settings. [#40668](https://github.com/ClickHouse/ClickHouse/pull/40668) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Allow changing `async_insert_max_data_size` or `async_insert_busy_timeout_ms` in scope of query. E.g. user wants to insert data rarely and she doesn't have access to the server config to tune default settings. [#40668](https://github.com/ClickHouse/ClickHouse/pull/40668) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Improvements for reading from remote filesystems, made threadpool size for reads/writes configurable. Closes [#41070](https://github.com/ClickHouse/ClickHouse/issues/41070). [#41011](https://github.com/ClickHouse/ClickHouse/pull/41011) ([Kseniia Sumarokova](https://github.com/kssenii)). * Support all combinators combination in WindowTransform/arratReduce*/initializeAggregation/aggregate functions versioning. Previously combinators like `ForEach/Resample/Map` didn't work in these places, using them led to exception like`State function ... inserts results into non-state column`. [#41107](https://github.com/ClickHouse/ClickHouse/pull/41107) ([Kruglov Pavel](https://github.com/Avogar)). * Add function `tryDecrypt` that returns NULL when decrypt fail (e.g. decrypt with incorrect key) instead of throwing exception. [#41206](https://github.com/ClickHouse/ClickHouse/pull/41206) ([Duc Canh Le](https://github.com/canhld94)). From 6d255b3430d3d441803ed2be9570c73517040cc4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Oct 2022 19:02:41 +0300 Subject: [PATCH 25/50] Update CHANGELOG.md Co-authored-by: Dan Roscigno --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8049037c99b..e71dd5dd737 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,7 +57,7 @@ * Allow changing `async_insert_max_data_size` or `async_insert_busy_timeout_ms` in scope of query. E.g. user wants to insert data rarely and she doesn't have access to the server config to tune default settings. [#40668](https://github.com/ClickHouse/ClickHouse/pull/40668) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Improvements for reading from remote filesystems, made threadpool size for reads/writes configurable. Closes [#41070](https://github.com/ClickHouse/ClickHouse/issues/41070). [#41011](https://github.com/ClickHouse/ClickHouse/pull/41011) ([Kseniia Sumarokova](https://github.com/kssenii)). * Support all combinators combination in WindowTransform/arratReduce*/initializeAggregation/aggregate functions versioning. Previously combinators like `ForEach/Resample/Map` didn't work in these places, using them led to exception like`State function ... inserts results into non-state column`. [#41107](https://github.com/ClickHouse/ClickHouse/pull/41107) ([Kruglov Pavel](https://github.com/Avogar)). -* Add function `tryDecrypt` that returns NULL when decrypt fail (e.g. decrypt with incorrect key) instead of throwing exception. [#41206](https://github.com/ClickHouse/ClickHouse/pull/41206) ([Duc Canh Le](https://github.com/canhld94)). +* Add function `tryDecrypt` that returns NULL when decrypt fails (e.g. decrypt with incorrect key) instead of throwing an exception. [#41206](https://github.com/ClickHouse/ClickHouse/pull/41206) ([Duc Canh Le](https://github.com/canhld94)). * Add the `unreserved_space` column to the `system.disks` table to check how much space is not taken by reservations per disk. [#41254](https://github.com/ClickHouse/ClickHouse/pull/41254) ([filimonov](https://github.com/filimonov)). * Support s3 authorization headers in table function arguments. [#41261](https://github.com/ClickHouse/ClickHouse/pull/41261) ([Kseniia Sumarokova](https://github.com/kssenii)). * Add support for MultiRead in Keeper and internal ZooKeeper client (this is an extension to ZooKeeper protocol, only available in ClickHouse Keeper). [#41410](https://github.com/ClickHouse/ClickHouse/pull/41410) ([Antonio Andelic](https://github.com/antonio2368)). From 4d490fa088f7e639f579b234833aadabdc4ca15f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Oct 2022 19:02:49 +0300 Subject: [PATCH 26/50] Update CHANGELOG.md Co-authored-by: Dan Roscigno --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e71dd5dd737..85363153e77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,7 +63,7 @@ * Add support for MultiRead in Keeper and internal ZooKeeper client (this is an extension to ZooKeeper protocol, only available in ClickHouse Keeper). [#41410](https://github.com/ClickHouse/ClickHouse/pull/41410) ([Antonio Andelic](https://github.com/antonio2368)). * Add support for decimal type comparing with floating point literal in IN operator. [#41544](https://github.com/ClickHouse/ClickHouse/pull/41544) ([liang.huang](https://github.com/lhuang09287750)). * Allow readable size values (like `1TB`) in cache config. [#41688](https://github.com/ClickHouse/ClickHouse/pull/41688) ([Kseniia Sumarokova](https://github.com/kssenii)). -* ClickHouse could cache stale DNS entries for some period of time (15 seconds by default) until the cache won't be updated asynchronously. During these period ClickHouse can nevertheless try to establish a connection and produce errors. This behaviour is fixed. [#41707](https://github.com/ClickHouse/ClickHouse/pull/41707) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* ClickHouse could cache stale DNS entries for some period of time (15 seconds by default) until the cache won't be updated asynchronously. During these periods ClickHouse can nevertheless try to establish a connection and produce errors. This behavior is fixed. [#41707](https://github.com/ClickHouse/ClickHouse/pull/41707) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Add interactive history search with fzf-like utility (fzf/sk) for `clickhouse-client`/`clickhouse-local` (note you can use `FZF_DEFAULT_OPTS`/`SKIM_DEFAULT_OPTIONS` to additionally configure the behavior). [#41730](https://github.com/ClickHouse/ClickHouse/pull/41730) ([Azat Khuzhin](https://github.com/azat)). * For client when connecting to a secure server with invalid certificate only allow to proceed with '--accept-certificate' flag. [#41743](https://github.com/ClickHouse/ClickHouse/pull/41743) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Add function `tryBase58Decode`, similar to the existing function `tryBase64Decode`. [#41824](https://github.com/ClickHouse/ClickHouse/pull/41824) ([Robert Schulze](https://github.com/rschu1ze)). From 55e0a9115ee344955425d53d305ccf8fce33243a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Oct 2022 19:04:21 +0300 Subject: [PATCH 27/50] Update CHANGELOG.md Co-authored-by: Dan Roscigno --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85363153e77..e0354f4ebb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,7 +65,8 @@ * Allow readable size values (like `1TB`) in cache config. [#41688](https://github.com/ClickHouse/ClickHouse/pull/41688) ([Kseniia Sumarokova](https://github.com/kssenii)). * ClickHouse could cache stale DNS entries for some period of time (15 seconds by default) until the cache won't be updated asynchronously. During these periods ClickHouse can nevertheless try to establish a connection and produce errors. This behavior is fixed. [#41707](https://github.com/ClickHouse/ClickHouse/pull/41707) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Add interactive history search with fzf-like utility (fzf/sk) for `clickhouse-client`/`clickhouse-local` (note you can use `FZF_DEFAULT_OPTS`/`SKIM_DEFAULT_OPTIONS` to additionally configure the behavior). [#41730](https://github.com/ClickHouse/ClickHouse/pull/41730) ([Azat Khuzhin](https://github.com/azat)). -* For client when connecting to a secure server with invalid certificate only allow to proceed with '--accept-certificate' flag. [#41743](https://github.com/ClickHouse/ClickHouse/pull/41743) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* +Only allow clients connecting to a secure server with an invalid certificate only to proceed with the '--accept-certificate' flag. [#41743](https://github.com/ClickHouse/ClickHouse/pull/41743) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Add function `tryBase58Decode`, similar to the existing function `tryBase64Decode`. [#41824](https://github.com/ClickHouse/ClickHouse/pull/41824) ([Robert Schulze](https://github.com/rschu1ze)). * Improve feedback when replacing partition with different primary key. Fixes [#34798](https://github.com/ClickHouse/ClickHouse/issues/34798). [#41838](https://github.com/ClickHouse/ClickHouse/pull/41838) ([Salvatore](https://github.com/tbsal)). * Fix parallel parsing: segmentator now checks `max_block_size`. This fixed memory overallocation in case of parallel parsing and small LIMIT. [#41852](https://github.com/ClickHouse/ClickHouse/pull/41852) ([Vitaly Baranov](https://github.com/vitlibar)). From e28f9402c79f0d8085268ec4b77e709419dd5c2e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Oct 2022 19:04:31 +0300 Subject: [PATCH 28/50] Update CHANGELOG.md Co-authored-by: Dan Roscigno --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0354f4ebb0..c9ec975df9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,7 +72,7 @@ Only allow clients connecting to a secure server with an invalid certificate onl * Fix parallel parsing: segmentator now checks `max_block_size`. This fixed memory overallocation in case of parallel parsing and small LIMIT. [#41852](https://github.com/ClickHouse/ClickHouse/pull/41852) ([Vitaly Baranov](https://github.com/vitlibar)). * Don't add "TABLE_IS_DROPPED" exception to `system.errors` if it's happened during SELECT from a system table and was ignored. [#41908](https://github.com/ClickHouse/ClickHouse/pull/41908) ([AlfVII](https://github.com/AlfVII)). * Improve option `enable_extended_results_for_datetime_functions` to return results of type DateTime64 for functions `toStartOfDay`, `toStartOfHour`, `toStartOfFifteenMinutes`, `toStartOfTenMinutes`, `toStartOfFiveMinutes`, `toStartOfMinute` and `timeSlot`. [#41910](https://github.com/ClickHouse/ClickHouse/pull/41910) ([Roman Vasin](https://github.com/rvasin)). -* Improve `DateTime` type inference for text formats. Now it respect setting `date_time_input_format` and doesn't try to infer datetimes from numbers as timestamps. Closes [#41389](https://github.com/ClickHouse/ClickHouse/issues/41389) Closes [#42206](https://github.com/ClickHouse/ClickHouse/issues/42206). [#41912](https://github.com/ClickHouse/ClickHouse/pull/41912) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve `DateTime` type inference for text formats. Now it respects setting `date_time_input_format` and doesn't try to infer datetimes from numbers as timestamps. Closes [#41389](https://github.com/ClickHouse/ClickHouse/issues/41389) Closes [#42206](https://github.com/ClickHouse/ClickHouse/issues/42206). [#41912](https://github.com/ClickHouse/ClickHouse/pull/41912) ([Kruglov Pavel](https://github.com/Avogar)). * Remove confusing warning when inserting with `perform_ttl_move_on_insert` = false. [#41980](https://github.com/ClickHouse/ClickHouse/pull/41980) ([Vitaly Baranov](https://github.com/vitlibar)). * Allow user to write `countState(*)` similar to `count(*)`. This closes [#9338](https://github.com/ClickHouse/ClickHouse/issues/9338). [#41983](https://github.com/ClickHouse/ClickHouse/pull/41983) ([Amos Bird](https://github.com/amosbird)). * Fix `rankCorr` size overflow. [#42020](https://github.com/ClickHouse/ClickHouse/pull/42020) ([Duc Canh Le](https://github.com/canhld94)). From f6139c82b6bc166ffd8ee894c79ce37d19f9e39b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Oct 2022 19:04:37 +0300 Subject: [PATCH 29/50] Update CHANGELOG.md Co-authored-by: Dan Roscigno --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9ec975df9b..ea74314ef57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,7 +79,7 @@ Only allow clients connecting to a secure server with an invalid certificate onl * Added an option to specify an arbitrary string as an environment name in the Sentry's config for more handy reports. [#42037](https://github.com/ClickHouse/ClickHouse/pull/42037) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Fix parsing out-of-range Date from CSV. [#42044](https://github.com/ClickHouse/ClickHouse/pull/42044) ([Andrey Zvonov](https://github.com/zvonand)). * `parseDataTimeBestEffort` now supports comma between date and time. Closes [#42038](https://github.com/ClickHouse/ClickHouse/issues/42038). [#42049](https://github.com/ClickHouse/ClickHouse/pull/42049) ([flynn](https://github.com/ucasfl)). -* Improved stale replica recovery process for `ReplicatedMergeTree`. If a lost replica have some parts which absent on a healthy replica, but these parts should appear in future according to replication queue of the healthy replica, then lost replica will keep such parts instead of detaching them. [#42134](https://github.com/ClickHouse/ClickHouse/pull/42134) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Improved stale replica recovery process for `ReplicatedMergeTree`. If a lost replica has some parts which are absent from a healthy replica, but these parts should appear in the future according to the replication queue of the healthy replica, then the lost replica will keep such parts instead of detaching them. [#42134](https://github.com/ClickHouse/ClickHouse/pull/42134) ([Alexander Tokmakov](https://github.com/tavplubix)). * Add a possibility to use `Date32` arguments for date_diff function. Fix issue in date_diff function when using DateTime64 arguments with start date before Unix epoch and end date after Unix epoch. [#42308](https://github.com/ClickHouse/ClickHouse/pull/42308) ([Roman Vasin](https://github.com/rvasin)). * When uploading big parts to Minio, 'Complete Multipart Upload' can take a long time. Minio sends heartbeats every 10 seconds (see https://github.com/minio/minio/pull/7198). But clickhouse times out earlier, because the default send/receive timeout is [set](https://github.com/ClickHouse/ClickHouse/blob/cc24fcd6d5dfb67f5f66f5483e986bd1010ad9cf/src/IO/S3/PocoHTTPClient.cpp#L123) to 5 seconds. [#42321](https://github.com/ClickHouse/ClickHouse/pull/42321) ([filimonov](https://github.com/filimonov)). * Fix rarely invalid cast of aggregate state types with complex types such as Decimal. This fixes [#42408](https://github.com/ClickHouse/ClickHouse/issues/42408). [#42417](https://github.com/ClickHouse/ClickHouse/pull/42417) ([Amos Bird](https://github.com/amosbird)). From aa8434610787b8e9a2d164499be8f20843f24002 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Oct 2022 19:04:43 +0300 Subject: [PATCH 30/50] Update CHANGELOG.md Co-authored-by: Dan Roscigno --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea74314ef57..4a1c8ed9c67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,7 +80,7 @@ Only allow clients connecting to a secure server with an invalid certificate onl * Fix parsing out-of-range Date from CSV. [#42044](https://github.com/ClickHouse/ClickHouse/pull/42044) ([Andrey Zvonov](https://github.com/zvonand)). * `parseDataTimeBestEffort` now supports comma between date and time. Closes [#42038](https://github.com/ClickHouse/ClickHouse/issues/42038). [#42049](https://github.com/ClickHouse/ClickHouse/pull/42049) ([flynn](https://github.com/ucasfl)). * Improved stale replica recovery process for `ReplicatedMergeTree`. If a lost replica has some parts which are absent from a healthy replica, but these parts should appear in the future according to the replication queue of the healthy replica, then the lost replica will keep such parts instead of detaching them. [#42134](https://github.com/ClickHouse/ClickHouse/pull/42134) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Add a possibility to use `Date32` arguments for date_diff function. Fix issue in date_diff function when using DateTime64 arguments with start date before Unix epoch and end date after Unix epoch. [#42308](https://github.com/ClickHouse/ClickHouse/pull/42308) ([Roman Vasin](https://github.com/rvasin)). +* Add a possibility to use `Date32` arguments for date_diff function. Fix issue in date_diff function when using DateTime64 arguments with a start date before Unix epoch and end date after Unix epoch. [#42308](https://github.com/ClickHouse/ClickHouse/pull/42308) ([Roman Vasin](https://github.com/rvasin)). * When uploading big parts to Minio, 'Complete Multipart Upload' can take a long time. Minio sends heartbeats every 10 seconds (see https://github.com/minio/minio/pull/7198). But clickhouse times out earlier, because the default send/receive timeout is [set](https://github.com/ClickHouse/ClickHouse/blob/cc24fcd6d5dfb67f5f66f5483e986bd1010ad9cf/src/IO/S3/PocoHTTPClient.cpp#L123) to 5 seconds. [#42321](https://github.com/ClickHouse/ClickHouse/pull/42321) ([filimonov](https://github.com/filimonov)). * Fix rarely invalid cast of aggregate state types with complex types such as Decimal. This fixes [#42408](https://github.com/ClickHouse/ClickHouse/issues/42408). [#42417](https://github.com/ClickHouse/ClickHouse/pull/42417) ([Amos Bird](https://github.com/amosbird)). * Allow to use `Date32` arguments for `dateName` function. [#42554](https://github.com/ClickHouse/ClickHouse/pull/42554) ([Roman Vasin](https://github.com/rvasin)). From dd4ff1e0e5978e3ccb7feecbd4890efbdcb013f5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Oct 2022 19:04:51 +0300 Subject: [PATCH 31/50] Update CHANGELOG.md Co-authored-by: Dan Roscigno --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a1c8ed9c67..131804043e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,7 +100,7 @@ Only allow clients connecting to a secure server with an invalid certificate onl * Choose correct aggregation method for `LowCardinality` with big integer types. [#42342](https://github.com/ClickHouse/ClickHouse/pull/42342) ([Duc Canh Le](https://github.com/canhld94)). * Several fixes for `web` disk. [#41652](https://github.com/ClickHouse/ClickHouse/pull/41652) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fixes issue when docker run will fail if `https_port` is not present in config. [#41693](https://github.com/ClickHouse/ClickHouse/pull/41693) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fixes an issue that causes docker run to fail if `https_port` is not present in config. [#41693](https://github.com/ClickHouse/ClickHouse/pull/41693) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Mutations were not cancelled properly on server shutdown or `SYSTEM STOP MERGES` query and cancellation might take long time, it's fixed. [#41699](https://github.com/ClickHouse/ClickHouse/pull/41699) ([Alexander Tokmakov](https://github.com/tavplubix)). * Fix wrong result of queries with `ORDER BY` or `GROUP BY` by columns from prefix of sorting key, wrapped into monotonic functions, with enable "read in order" optimization (settings `optimize_read_in_order` and `optimize_aggregation_in_order`). [#41701](https://github.com/ClickHouse/ClickHouse/pull/41701) ([Anton Popov](https://github.com/CurtizJ)). * Fix possible crash in `SELECT` from `Merge` table with enabled `optimize_monotonous_functions_in_order_by` setting. Fixes [#41269](https://github.com/ClickHouse/ClickHouse/issues/41269). [#41740](https://github.com/ClickHouse/ClickHouse/pull/41740) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). From 67212ad575feb4930b3082ada7a627f439a5b9b1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Oct 2022 19:04:59 +0300 Subject: [PATCH 32/50] Update CHANGELOG.md Co-authored-by: Dan Roscigno --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 131804043e2..c2def2f787e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,7 +110,7 @@ Only allow clients connecting to a secure server with an invalid certificate onl * 22.9 might fail to startup `ReplicatedMergeTree` table if that table was created by 20.3 or older version and was never altered, it's fixed. Fixes [#41742](https://github.com/ClickHouse/ClickHouse/issues/41742). [#41796](https://github.com/ClickHouse/ClickHouse/pull/41796) ([Alexander Tokmakov](https://github.com/tavplubix)). * When the batch sending fails for some reason, it cannot be automatically recovered, and if it is not processed in time, it will lead to accumulation, and the printed error message will become longer and longer, which will cause the http thread to block. [#41813](https://github.com/ClickHouse/ClickHouse/pull/41813) ([zhongyuankai](https://github.com/zhongyuankai)). * Fix compact parts with compressed marks setting. Fixes [#41783](https://github.com/ClickHouse/ClickHouse/issues/41783) and [#41746](https://github.com/ClickHouse/ClickHouse/issues/41746). [#41823](https://github.com/ClickHouse/ClickHouse/pull/41823) ([alesapin](https://github.com/alesapin)). -* Old versions of Replicated database doesn't have a special marker in [Zoo]Keeper. We need to check only whether the node contains come obscure data instead of special mark. [#41875](https://github.com/ClickHouse/ClickHouse/pull/41875) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Old versions of Replicated database don't have a special marker in [Zoo]Keeper. We need to check only whether the node contains come obscure data instead of special mark. [#41875](https://github.com/ClickHouse/ClickHouse/pull/41875) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Fix possible exception in fs cache. [#41884](https://github.com/ClickHouse/ClickHouse/pull/41884) ([Kseniia Sumarokova](https://github.com/kssenii)). * Fix `use_environment_credentials` for s3 table function. [#41970](https://github.com/ClickHouse/ClickHouse/pull/41970) ([Kseniia Sumarokova](https://github.com/kssenii)). * Fixed "Directory already exists and is not empty" error on detaching broken part that might prevent `ReplicatedMergeTree` table from starting replication. Fixes [#40957](https://github.com/ClickHouse/ClickHouse/issues/40957). [#41981](https://github.com/ClickHouse/ClickHouse/pull/41981) ([Alexander Tokmakov](https://github.com/tavplubix)). From 171ed8dd4e1e050f34a19b3702381a112a00e159 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Mon, 24 Oct 2022 19:52:51 +0000 Subject: [PATCH 33/50] Apply suggestions from review --- src/Parsers/ExpressionListParsers.cpp | 32 +++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index a38d7ab0c98..6336b92de1b 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -1096,7 +1096,7 @@ public: class CastLayer : public Layer { public: - CastLayer() : Layer(true, true) {} + CastLayer() : Layer(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true) {} bool parse(IParser::Pos & pos, Expected & expected, Action & action) override { @@ -1196,7 +1196,7 @@ public: class ExtractLayer : public BaseLayer { public: - ExtractLayer() : BaseLayer(true, true) {} + ExtractLayer() : BaseLayer(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true) {} bool getResult(ASTPtr & node) override { @@ -1267,7 +1267,7 @@ private: class SubstringLayer : public Layer { public: - SubstringLayer() : Layer(true, true) {} + SubstringLayer() : Layer(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true) {} bool getResult(ASTPtr & node) override { @@ -1329,7 +1329,7 @@ public: class PositionLayer : public Layer { public: - PositionLayer() : Layer(true, true) {} + PositionLayer() : Layer(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true) {} bool getResult(ASTPtr & node) override { @@ -1399,7 +1399,7 @@ public: class ExistsLayer : public Layer { public: - ExistsLayer() : Layer(true, true) {} + ExistsLayer() : Layer(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true) {} bool parse(IParser::Pos & pos, Expected & expected, Action & /*action*/) override { @@ -1426,7 +1426,7 @@ class TrimLayer : public Layer { public: TrimLayer(bool trim_left_, bool trim_right_) - : Layer(true, true), trim_left(trim_left_), trim_right(trim_right_) {} + : Layer(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true), trim_left(trim_left_), trim_right(trim_right_) {} bool getResult(ASTPtr & node) override { @@ -1588,7 +1588,7 @@ class DateAddLayer : public BaseLayer { public: - DateDiffLayer() : BaseLayer(true, true) {} + DateDiffLayer() : BaseLayer(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true) {} bool getResult(ASTPtr & node) override { @@ -1696,7 +1696,7 @@ private: class IntervalLayer : public Layer { public: - IntervalLayer() : Layer(true, true) {} + IntervalLayer() : Layer(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true) {} bool parse(IParser::Pos & pos, Expected & expected, Action & action) override { @@ -1775,7 +1775,7 @@ private: class CaseLayer : public Layer { public: - CaseLayer() : Layer(true, true) {} + CaseLayer() : Layer(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true) {} bool parse(IParser::Pos & pos, Expected & expected, Action & action) override { @@ -2444,13 +2444,13 @@ Action ParserExpressionImpl::tryParseOperator(Layers & layers, IParser::Pos & po { ParserAlias alias_parser(layers.back()->allow_alias_without_as_keyword); auto old_pos = pos; - if (!layers.back()->parsed_alias && layers.back()->allow_alias && alias_parser.parse(pos, tmp, expected)) + if (layers.back()->allow_alias && + !layers.back()->parsed_alias && + alias_parser.parse(pos, tmp, expected) && + layers.back()->insertAlias(tmp)) { - if (layers.back()->insertAlias(tmp)) - { - layers.back()->parsed_alias = true; - return Action::OPERATOR; - } + layers.back()->parsed_alias = true; + return Action::OPERATOR; } pos = old_pos; return Action::NONE; From eed21ad4caa803b6ead095399d1e2fef83da3b5f Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 25 Oct 2022 01:30:32 +0200 Subject: [PATCH 34/50] Revert "Low cardinality cases moved to the function for its corresponding type" --- src/Functions/FunctionsJSON.cpp | 123 +++--------------- .../performance/low_cardinality_from_json.xml | 73 ----------- .../02452_check_low_cardinality.reference | 8 -- .../02452_check_low_cardinality.sql | 73 ----------- ...ct_fixed_string_from_nested_json.reference | 1 - ..._extract_fixed_string_from_nested_json.sql | 6 - 6 files changed, 21 insertions(+), 263 deletions(-) delete mode 100644 tests/performance/low_cardinality_from_json.xml delete mode 100644 tests/queries/0_stateless/02452_check_low_cardinality.reference delete mode 100644 tests/queries/0_stateless/02452_check_low_cardinality.sql delete mode 100644 tests/queries/0_stateless/02455_extract_fixed_string_from_nested_json.reference delete mode 100644 tests/queries/0_stateless/02455_extract_fixed_string_from_nested_json.sql diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index 2f0a0889847..493fc36ca3c 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -696,16 +695,8 @@ public: else return false; - if (dest.getDataType() == TypeIndex::LowCardinality) - { - ColumnLowCardinality & col_low = assert_cast(dest); - col_low.insertData(reinterpret_cast(&value), sizeof(value)); - } - else - { - auto & col_vec = assert_cast &>(dest); - col_vec.insertValue(value); - } + auto & col_vec = assert_cast &>(dest); + col_vec.insertValue(value); return true; } }; @@ -782,17 +773,8 @@ public: return JSONExtractRawImpl::insertResultToColumn(dest, element, {}); auto str = element.getString(); - - if (dest.getDataType() == TypeIndex::LowCardinality) - { - ColumnLowCardinality & col_low = assert_cast(dest); - col_low.insertData(str.data(), str.size()); - } - else - { - ColumnString & col_str = assert_cast(dest); - col_str.insertData(str.data(), str.size()); - } + ColumnString & col_str = assert_cast(dest); + col_str.insertData(str.data(), str.size()); return true; } }; @@ -821,33 +803,25 @@ struct JSONExtractTree } }; - class LowCardinalityFixedStringNode : public Node + class LowCardinalityNode : public Node { public: - explicit LowCardinalityFixedStringNode(const size_t fixed_length_) : fixed_length(fixed_length_) {} + LowCardinalityNode(DataTypePtr dictionary_type_, std::unique_ptr impl_) + : dictionary_type(dictionary_type_), impl(std::move(impl_)) {} bool insertResultToColumn(IColumn & dest, const Element & element) override { - // If element is an object we delegate the insertion to JSONExtractRawImpl - if (element.isObject()) - return JSONExtractRawImpl::insertResultToLowCardinalityFixedStringColumn(dest, element, fixed_length); - else if (!element.isString()) - return false; - - auto str = element.getString(); - if (str.size() > fixed_length) - return false; - - // For the non low cardinality case of FixedString, the padding is done in the FixedString Column implementation. - // In order to avoid having to pass the data to a FixedString Column and read it back (which would slow down the execution) - // the data is padded here and written directly to the Low Cardinality Column - auto padded_str = str.data() + std::string(fixed_length - std::min(fixed_length, str.length()), '\0'); - - assert_cast(dest).insertData(padded_str.data(), padded_str.size()); - return true; + auto from_col = dictionary_type->createColumn(); + if (impl->insertResultToColumn(*from_col, element)) + { + std::string_view value = from_col->getDataAt(0).toView(); + assert_cast(dest).insertData(value.data(), value.size()); + return true; + } + return false; } - private: - const size_t fixed_length; + DataTypePtr dictionary_type; + std::unique_ptr impl; }; class UUIDNode : public Node @@ -859,15 +833,7 @@ struct JSONExtractTree return false; auto uuid = parseFromString(element.getString()); - if (dest.getDataType() == TypeIndex::LowCardinality) - { - ColumnLowCardinality & col_low = assert_cast(dest); - col_low.insertData(reinterpret_cast(&uuid), sizeof(uuid)); - } - else - { - assert_cast(dest).insert(uuid); - } + assert_cast(dest).insert(uuid); return true; } }; @@ -887,7 +853,6 @@ struct JSONExtractTree assert_cast &>(dest).insert(result); return true; } - private: DataTypePtr data_type; }; @@ -906,18 +871,13 @@ struct JSONExtractTree public: bool insertResultToColumn(IColumn & dest, const Element & element) override { - if (element.isNull()) - return false; - if (!element.isString()) - return JSONExtractRawImpl::insertResultToFixedStringColumn(dest, element, {}); - - auto str = element.getString(); + return false; auto & col_str = assert_cast(dest); + auto str = element.getString(); if (str.size() > col_str.getN()) return false; col_str.insertData(str.data(), str.size()); - return true; } }; @@ -1139,19 +1099,9 @@ struct JSONExtractTree case TypeIndex::UUID: return std::make_unique(); case TypeIndex::LowCardinality: { - // The low cardinality case is treated in two different ways: - // For FixedString type, an especial class is implemented for inserting the data in the destination column, - // as the string length must be passed in order to check and pad the incoming data. - // For the rest of low cardinality types, the insertion is done in their corresponding class, adapting the data - // as needed for the insertData function of the ColumnLowCardinality. auto dictionary_type = typeid_cast(type.get())->getDictionaryType(); - if ((*dictionary_type).getTypeId() == TypeIndex::FixedString) - { - auto fixed_length = typeid_cast(dictionary_type.get())->getN(); - return std::make_unique(fixed_length); - } auto impl = build(function_name, dictionary_type); - return impl; + return std::make_unique(dictionary_type, std::move(impl)); } case TypeIndex::Decimal256: return std::make_unique>(type); case TypeIndex::Decimal128: return std::make_unique>(type); @@ -1313,37 +1263,6 @@ public: return true; } - // We use insertResultToFixedStringColumn in case we are inserting raw data in a FixedString column - static bool insertResultToFixedStringColumn(IColumn & dest, const Element & element, std::string_view) - { - ColumnFixedString & col_str = assert_cast(dest); - auto & chars = col_str.getChars(); - WriteBufferFromVector buf(chars, AppendModeTag()); - traverse(element, buf); - buf.finalize(); - col_str.insertDefault(); - return true; - } - - // We use insertResultToLowCardinalityFixedStringColumn in case we are inserting raw data in a Low Cardinality FixedString column - static bool insertResultToLowCardinalityFixedStringColumn(IColumn & dest, const Element & element, size_t fixed_length) - { - if (element.getObject().size() > fixed_length) - return false; - - ColumnFixedString::Chars chars; - WriteBufferFromVector buf(chars, AppendModeTag()); - traverse(element, buf); - buf.finalize(); - chars.push_back(0); - std::string str = reinterpret_cast(chars.data()); - - auto padded_str = str + std::string(fixed_length - std::min(fixed_length, str.length()), '\0'); - assert_cast(dest).insertData(padded_str.data(), padded_str.size()); - - return true; - } - private: static void traverse(const Element & element, WriteBuffer & buf) { diff --git a/tests/performance/low_cardinality_from_json.xml b/tests/performance/low_cardinality_from_json.xml deleted file mode 100644 index ac6542ac503..00000000000 --- a/tests/performance/low_cardinality_from_json.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - - - string_json - - '{"a": "hi", "b": "hello", "c": "hola", "d": "see you, bye, bye"}' - - - - int_json - - '{"a": 11, "b": 2222, "c": 33333333, "d": 4444444444444444}' - - - - uuid_json - - '{"a": "2d49dc6e-ddce-4cd0-afb8-790956df54c4", "b": "2d49dc6e-ddce-4cd0-afb8-790956df54c3", "c": "2d49dc6e-ddce-4cd0-afb8-790956df54c1", "d": "2d49dc6e-ddce-4cd0-afb8-790956df54c1"}' - - - - low_cardinality_tuple_string - - 'Tuple(a LowCardinality(String), b LowCardinality(String), c LowCardinality(String), d LowCardinality(String) )' - - - - low_cardinality_tuple_fixed_string - - 'Tuple(a LowCardinality(FixedString(20)), b LowCardinality(FixedString(20)), c LowCardinality(FixedString(20)), d LowCardinality(FixedString(20)) )' - - - - low_cardinality_tuple_int8 - - 'Tuple(a LowCardinality(Int8), b LowCardinality(Int8), c LowCardinality(Int8), d LowCardinality(Int8) )' - - - - low_cardinality_tuple_int16 - - 'Tuple(a LowCardinality(Int16), b LowCardinality(Int16), c LowCardinality(Int16), d LowCardinality(Int16) )' - - - - low_cardinality_tuple_int32 - - 'Tuple(a LowCardinality(Int32), b LowCardinality(Int32), c LowCardinality(Int32), d LowCardinality(Int32) )' - - - - low_cardinality_tuple_int64 - - 'Tuple(a LowCardinality(Int64), b LowCardinality(Int64), c LowCardinality(Int64), d LowCardinality(Int64) )' - - - - low_cardinality_tuple_uuid - - 'Tuple(a LowCardinality(UUID), b LowCardinality(UUID), c LowCardinality(UUID), d LowCardinality(UUID) )' - - - - - SELECT 'fixed_string_json' FROM zeros(500000) WHERE NOT ignore(JSONExtract(materialize({string_json}), {low_cardinality_tuple_fixed_string})) FORMAT Null - SELECT 'string_json' FROM zeros(500000) WHERE NOT ignore(JSONExtract(materialize({string_json}), {low_cardinality_tuple_string})) FORMAT Null - SELECT 'int8_json' FROM zeros(500000) WHERE NOT ignore(JSONExtract(materialize({int_json}), {low_cardinality_tuple_int8})) FORMAT Null - SELECT 'int16_json' FROM zeros(500000) WHERE NOT ignore(JSONExtract(materialize({int_json}), {low_cardinality_tuple_int16})) FORMAT Null - SELECT 'int32_json' FROM zeros(500000) WHERE NOT ignore(JSONExtract(materialize({int_json}), {low_cardinality_tuple_int32})) FORMAT Null - SELECT 'int64_json' FROM zeros(500000) WHERE NOT ignore(JSONExtract(materialize({int_json}), {low_cardinality_tuple_int64})) FORMAT Null - SELECT 'uuid_json' FROM zeros(500000) WHERE NOT ignore(JSONExtract(materialize({uuid_json}), {low_cardinality_tuple_uuid})) FORMAT Null - \ No newline at end of file diff --git a/tests/queries/0_stateless/02452_check_low_cardinality.reference b/tests/queries/0_stateless/02452_check_low_cardinality.reference deleted file mode 100644 index dc5553d7886..00000000000 --- a/tests/queries/0_stateless/02452_check_low_cardinality.reference +++ /dev/null @@ -1,8 +0,0 @@ -('hi','hello','hola','see you, bye, bye') -('hi\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','hello\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','hola\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','see you, bye, bye\0\0\0') -(11,0,0,0) -(11,2222,0,0) -(11,2222,33333333,0) -(11,2222,33333333,4444444444444444) -('2d49dc6e-ddce-4cd0-afb8-790956df54c4','2d49dc6e-ddce-4cd0-afb8-790956df54c3','2d49dc6e-ddce-4cd0-afb8-790956df54c1','2d49dc6e-ddce-4cd0-afb8-790956df54c1') -('{"b":{"c":1,"d":"str"}}\0\0\0\0\0\0\0','\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0') diff --git a/tests/queries/0_stateless/02452_check_low_cardinality.sql b/tests/queries/0_stateless/02452_check_low_cardinality.sql deleted file mode 100644 index d62f418139e..00000000000 --- a/tests/queries/0_stateless/02452_check_low_cardinality.sql +++ /dev/null @@ -1,73 +0,0 @@ --- Tags: no-fasttest -DROP TABLE IF EXISTS test_low_cardinality_string; -DROP TABLE IF EXISTS test_low_cardinality_string_nested_json; -DROP TABLE IF EXISTS test_low_cardinality_uuid; -DROP TABLE IF EXISTS test_low_cardinality_int; -CREATE TABLE test_low_cardinality_string (data String) ENGINE MergeTree ORDER BY data; -CREATE TABLE test_low_cardinality_string_nested_json (data String) ENGINE MergeTree ORDER BY data; -CREATE TABLE test_low_cardinality_uuid (data String) ENGINE MergeTree ORDER BY data; -CREATE TABLE test_low_cardinality_int (data String) ENGINE MergeTree ORDER BY data; -INSERT INTO test_low_cardinality_string (data) VALUES ('{"a": "hi", "b": "hello", "c": "hola", "d": "see you, bye, bye"}'); -INSERT INTO test_low_cardinality_string_nested_json (data) VALUES ('{"a" : {"b" : {"c" : 1, "d" : "str"}}}'); -INSERT INTO test_low_cardinality_int (data) VALUES ('{"a": 11, "b": 2222, "c": 33333333, "d": 4444444444444444}'); -INSERT INTO test_low_cardinality_uuid (data) VALUES ('{"a": "2d49dc6e-ddce-4cd0-afb8-790956df54c4", "b": "2d49dc6e-ddce-4cd0-afb8-790956df54c3", "c": "2d49dc6e-ddce-4cd0-afb8-790956df54c1", "d": "2d49dc6e-ddce-4cd0-afb8-790956df54c1"}'); -SELECT JSONExtract(data, 'Tuple( - a LowCardinality(String), - b LowCardinality(String), - c LowCardinality(String), - d LowCardinality(String) - - )') AS json FROM test_low_cardinality_string; -SELECT JSONExtract(data, 'Tuple( - a LowCardinality(FixedString(20)), - b LowCardinality(FixedString(20)), - c LowCardinality(FixedString(20)), - d LowCardinality(FixedString(20)) - - )') AS json FROM test_low_cardinality_string; -SELECT JSONExtract(data, 'Tuple( - a LowCardinality(Int8), - b LowCardinality(Int8), - c LowCardinality(Int8), - d LowCardinality(Int8) - - )') AS json FROM test_low_cardinality_int; -SELECT JSONExtract(data, 'Tuple( - a LowCardinality(Int16), - b LowCardinality(Int16), - c LowCardinality(Int16), - d LowCardinality(Int16) - - )') AS json FROM test_low_cardinality_int; -SELECT JSONExtract(data, 'Tuple( - a LowCardinality(Int32), - b LowCardinality(Int32), - c LowCardinality(Int32), - d LowCardinality(Int32) - - )') AS json FROM test_low_cardinality_int; -SELECT JSONExtract(data, 'Tuple( - a LowCardinality(Int64), - b LowCardinality(Int64), - c LowCardinality(Int64), - d LowCardinality(Int64) - - )') AS json FROM test_low_cardinality_int; -SELECT JSONExtract(data, 'Tuple( - a LowCardinality(UUID), - b LowCardinality(UUID), - c LowCardinality(UUID), - d LowCardinality(UUID) - - )') AS json FROM test_low_cardinality_uuid; -SELECT JSONExtract('{"a" : {"b" : {"c" : 1, "d" : "str"}}}', 'Tuple( - a LowCardinality(FixedString(30)), - b LowCardinality(FixedString(20)), - c LowCardinality(FixedString(20)), - d LowCardinality(FixedString(20)) - - )') AS json FROM test_low_cardinality_string; -DROP TABLE test_low_cardinality_string_nested_json; -DROP TABLE test_low_cardinality_string; -DROP TABLE test_low_cardinality_uuid; -DROP TABLE test_low_cardinality_int; diff --git a/tests/queries/0_stateless/02455_extract_fixed_string_from_nested_json.reference b/tests/queries/0_stateless/02455_extract_fixed_string_from_nested_json.reference deleted file mode 100644 index 3a528a24821..00000000000 --- a/tests/queries/0_stateless/02455_extract_fixed_string_from_nested_json.reference +++ /dev/null @@ -1 +0,0 @@ -('{"b":{"c":1,"d":"str"}}\0') diff --git a/tests/queries/0_stateless/02455_extract_fixed_string_from_nested_json.sql b/tests/queries/0_stateless/02455_extract_fixed_string_from_nested_json.sql deleted file mode 100644 index c3eee60f161..00000000000 --- a/tests/queries/0_stateless/02455_extract_fixed_string_from_nested_json.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Tags: no-fasttest -DROP TABLE IF EXISTS test_fixed_string_nested_json; -CREATE TABLE test_fixed_string_nested_json (data String) ENGINE MergeTree ORDER BY data; -INSERT INTO test_fixed_string_nested_json (data) VALUES ('{"a" : {"b" : {"c" : 1, "d" : "str"}}}'); -SELECT JSONExtract(data, 'Tuple(a FixedString(24))') AS json FROM test_fixed_string_nested_json; -DROP TABLE test_fixed_string_nested_json; From 646f2f8f9c5ad4e1b018c5efec9bde6819049984 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 25 Oct 2022 05:20:16 +0200 Subject: [PATCH 35/50] Add .rgignore for test data --- tests/.rgignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/.rgignore diff --git a/tests/.rgignore b/tests/.rgignore new file mode 100644 index 00000000000..26cb6f9025d --- /dev/null +++ b/tests/.rgignore @@ -0,0 +1 @@ +data_json From 62a33b12d50e7dd263fa9c0b5b573ac5b666ec21 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> Date: Tue, 25 Oct 2022 11:24:42 +0200 Subject: [PATCH 36/50] Small refactor --- src/Parsers/ExpressionListParsers.cpp | 30 +++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index a13282bc3d5..4f5438d5b5c 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -786,10 +786,10 @@ public: /// 2. If we parse an ending token, we should merge everything as in (1) and /// also set 'finished' flag. template -class BaseLayer : public Layer +class LayerWithSeparator : public Layer { public: - explicit BaseLayer(bool allow_alias_ = true, bool allow_alias_without_as_keyword_ = false) : + explicit LayerWithSeparator(bool allow_alias_ = true, bool allow_alias_without_as_keyword_ = false) : Layer(allow_alias_, allow_alias_without_as_keyword_) {} bool parse(IParser::Pos & pos, Expected & expected, Action & action) override @@ -1066,7 +1066,7 @@ private: }; /// Layer for array square brackets operator -class ArrayLayer : public BaseLayer +class ArrayLayer : public LayerWithSeparator { public: bool getResult(ASTPtr & node) override @@ -1077,19 +1077,19 @@ public: bool parse(IParser::Pos & pos, Expected & expected, Action & action) override { - return BaseLayer::parse(pos, expected, action); + return LayerWithSeparator::parse(pos, expected, action); } }; /// Layer for arrayElement square brackets operator /// This layer does not create a function, it is only needed to parse closing token /// and return only one element. -class ArrayElementLayer : public BaseLayer +class ArrayElementLayer : public LayerWithSeparator { public: bool parse(IParser::Pos & pos, Expected & expected, Action & action) override { - return BaseLayer::parse(pos, expected, action); + return LayerWithSeparator::parse(pos, expected, action); } }; @@ -1193,10 +1193,10 @@ public: } }; -class ExtractLayer : public BaseLayer +class ExtractLayer : public LayerWithSeparator { public: - ExtractLayer() : BaseLayer(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true) {} + ExtractLayer() : LayerWithSeparator(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true) {} bool getResult(ASTPtr & node) override { @@ -1242,7 +1242,7 @@ public: if (state == 1) { - return BaseLayer::parse(pos, expected, action); + return LayerWithSeparator::parse(pos, expected, action); } if (state == 2) @@ -1584,11 +1584,11 @@ private: String function_name; }; -class DateAddLayer : public BaseLayer +class DateAddLayer : public LayerWithSeparator { public: explicit DateAddLayer(const char * function_name_) - : BaseLayer(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true), function_name(function_name_) {} + : LayerWithSeparator(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true), function_name(function_name_) {} bool getResult(ASTPtr & node) override { @@ -1630,7 +1630,7 @@ public: if (state == 1) { - return BaseLayer::parse(pos, expected, action); + return LayerWithSeparator::parse(pos, expected, action); } return true; @@ -1642,10 +1642,10 @@ private: bool parsed_interval_kind = false; }; -class DateDiffLayer : public BaseLayer +class DateDiffLayer : public LayerWithSeparator { public: - DateDiffLayer() : BaseLayer(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true) {} + DateDiffLayer() : LayerWithSeparator(/*allow_alias*/ true, /*allow_alias_without_as_keyword*/ true) {} bool getResult(ASTPtr & node) override { @@ -1685,7 +1685,7 @@ public: if (state == 1) { - return BaseLayer::parse(pos, expected, action); + return LayerWithSeparator::parse(pos, expected, action); } return true; From e6bfff3de2e719af596cda9265ffebb3c16cd177 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 25 Oct 2022 12:35:25 +0200 Subject: [PATCH 37/50] Analyzer change setting into allow_experimental_analyzer --- src/Core/Settings.h | 2 +- src/Interpreters/InterpreterDescribeQuery.cpp | 2 +- src/Interpreters/InterpreterExplainQuery.cpp | 4 ++-- src/Interpreters/InterpreterFactory.cpp | 4 ++-- .../Optimizations/reuseStorageOrderingForWindowFunctions.cpp | 2 +- tests/queries/0_stateless/02337_analyzer_columns_basic.sql | 2 +- tests/queries/0_stateless/02338_analyzer_constants_basic.sql | 2 +- tests/queries/0_stateless/02339_analyzer_matcher_basic.sql | 2 +- tests/queries/0_stateless/02340_analyzer_functions.sql | 2 +- tests/queries/0_stateless/02341_analyzer_aliases_basics.sql | 2 +- tests/queries/0_stateless/02342_analyzer_compound_types.sql | 2 +- .../0_stateless/02343_analyzer_column_transformers_strict.sql | 2 +- tests/queries/0_stateless/02343_analyzer_lambdas.sql | 2 +- .../02344_analyzer_multiple_aliases_for_expression.sql | 2 +- tests/queries/0_stateless/02345_analyzer_subqueries.sql | 4 ++-- tests/queries/0_stateless/02366_explain_query_tree.sql | 2 +- .../0_stateless/02367_analyzer_table_alias_columns.sql | 2 +- tests/queries/0_stateless/02368_analyzer_table_functions.sql | 2 +- .../0_stateless/02369_analyzer_array_join_function.sql | 2 +- tests/queries/0_stateless/02370_analyzer_in_function.sql | 2 +- tests/queries/0_stateless/02371_analyzer_join_cross.sql | 2 +- tests/queries/0_stateless/02372_analyzer_join.sql.j2 | 2 +- tests/queries/0_stateless/02373_analyzer_join_use_nulls.sql | 2 +- tests/queries/0_stateless/02374_analyzer_array_join.sql | 2 +- tests/queries/0_stateless/02374_analyzer_join_using.sql.j2 | 2 +- tests/queries/0_stateless/02375_analyzer_union.sql | 2 +- .../0_stateless/02376_analyzer_in_function_subquery.sql | 2 +- tests/queries/0_stateless/02377_analyzer_in_function_set.sql | 2 +- tests/queries/0_stateless/02378_analyzer_projection_names.sql | 2 +- tests/queries/0_stateless/02379_analyzer_subquery_depth.sql | 2 +- tests/queries/0_stateless/02380_analyzer_join_sample.sql | 2 +- tests/queries/0_stateless/02381_analyzer_join_final.sql | 2 +- .../queries/0_stateless/02382_analyzer_matcher_join_using.sql | 2 +- .../0_stateless/02383_analyzer_merge_tree_self_join.sql | 2 +- .../queries/0_stateless/02384_analyzer_dict_get_join_get.sql | 2 +- .../02385_analyzer_aliases_compound_expression.sql | 2 +- .../02386_analyzer_in_function_nested_subqueries.sql | 2 +- tests/queries/0_stateless/02387_analyzer_cte.sql | 2 +- tests/queries/0_stateless/02388_analyzer_recursive_lambda.sql | 2 +- tests/queries/0_stateless/02389_analyzer_nested_lambda.sql | 2 +- 40 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index bf117163431..148dc56a084 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -302,7 +302,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \ M(Bool, opentelemetry_trace_processors, false, "Collect OpenTelemetry spans for processors.", 0) \ M(Bool, prefer_column_name_to_alias, false, "Prefer using column names instead of aliases if possible.", 0) \ - M(Bool, use_analyzer, false, "Use analyzer", 0) \ + M(Bool, allow_experimental_analyzer, false, "Allow experimental analyzer", 0) \ M(Bool, prefer_global_in_and_join, false, "If enabled, all IN/JOIN operators will be rewritten as GLOBAL IN/JOIN. It's useful when the to-be-joined tables are only available on the initiator and we need to always scatter their data on-the-fly during distributed processing with the GLOBAL keyword. It's also useful to reduce the need to access the external sources joining external tables.", 0) \ \ \ diff --git a/src/Interpreters/InterpreterDescribeQuery.cpp b/src/Interpreters/InterpreterDescribeQuery.cpp index 60894d331a1..512f9be6fa1 100644 --- a/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/src/Interpreters/InterpreterDescribeQuery.cpp @@ -75,7 +75,7 @@ BlockIO InterpreterDescribeQuery::execute() auto select_query = table_expression.subquery->children.at(0); auto current_context = getContext(); - if (settings.use_analyzer) + if (settings.allow_experimental_analyzer) { SelectQueryOptions select_query_options; names_and_types = InterpreterSelectQueryAnalyzer(select_query, select_query_options, current_context).getSampleBlock().getNamesAndTypesList(); diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index 6ea1b6a62b3..fb8d3c6049f 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -419,7 +419,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl() auto settings = checkAndGetSettings(ast.getSettings()); QueryPlan plan; - if (getContext()->getSettingsRef().use_analyzer) + if (getContext()->getSettingsRef().allow_experimental_analyzer) { InterpreterSelectQueryAnalyzer interpreter(ast.getExplainedQuery(), options, getContext()); plan = std::move(interpreter).extractQueryPlan(); @@ -462,7 +462,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl() auto settings = checkAndGetSettings(ast.getSettings()); QueryPlan plan; - if (getContext()->getSettingsRef().use_analyzer) + if (getContext()->getSettingsRef().allow_experimental_analyzer) { InterpreterSelectQueryAnalyzer interpreter(ast.getExplainedQuery(), options, getContext()); plan = std::move(interpreter).extractQueryPlan(); diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 3e074ec8713..06d5746af59 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -119,7 +119,7 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextMut if (query->as()) { - if (context->getSettingsRef().use_analyzer) + if (context->getSettingsRef().allow_experimental_analyzer) return std::make_unique(query, options, context); /// This is internal part of ASTSelectWithUnionQuery. @@ -130,7 +130,7 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextMut { ProfileEvents::increment(ProfileEvents::SelectQuery); - if (context->getSettingsRef().use_analyzer) + if (context->getSettingsRef().allow_experimental_analyzer) return std::make_unique(query, options, context); return std::make_unique(query, context, options); diff --git a/src/Processors/QueryPlan/Optimizations/reuseStorageOrderingForWindowFunctions.cpp b/src/Processors/QueryPlan/Optimizations/reuseStorageOrderingForWindowFunctions.cpp index f5e7a438877..c74b5ed915b 100644 --- a/src/Processors/QueryPlan/Optimizations/reuseStorageOrderingForWindowFunctions.cpp +++ b/src/Processors/QueryPlan/Optimizations/reuseStorageOrderingForWindowFunctions.cpp @@ -62,7 +62,7 @@ size_t tryReuseStorageOrderingForWindowFunctions(QueryPlan::Node * parent_node, } auto context = read_from_merge_tree->getContext(); - if (!context->getSettings().optimize_read_in_window_order || context->getSettingsRef().use_analyzer) + if (!context->getSettings().optimize_read_in_window_order || context->getSettingsRef().allow_experimental_analyzer) { return 0; } diff --git a/tests/queries/0_stateless/02337_analyzer_columns_basic.sql b/tests/queries/0_stateless/02337_analyzer_columns_basic.sql index 00dc6c1351e..76f9f8b25e4 100644 --- a/tests/queries/0_stateless/02337_analyzer_columns_basic.sql +++ b/tests/queries/0_stateless/02337_analyzer_columns_basic.sql @@ -1,6 +1,6 @@ -- Tags: no-parallel -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; -- Empty from section diff --git a/tests/queries/0_stateless/02338_analyzer_constants_basic.sql b/tests/queries/0_stateless/02338_analyzer_constants_basic.sql index 56f82763f1b..6d6249538a4 100644 --- a/tests/queries/0_stateless/02338_analyzer_constants_basic.sql +++ b/tests/queries/0_stateless/02338_analyzer_constants_basic.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DESCRIBE (SELECT 1); SELECT 1; diff --git a/tests/queries/0_stateless/02339_analyzer_matcher_basic.sql b/tests/queries/0_stateless/02339_analyzer_matcher_basic.sql index 862cb2f88ff..9d7c486b28a 100644 --- a/tests/queries/0_stateless/02339_analyzer_matcher_basic.sql +++ b/tests/queries/0_stateless/02339_analyzer_matcher_basic.sql @@ -1,6 +1,6 @@ -- Tags: no-parallel -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; SELECT 'Matchers without FROM section'; diff --git a/tests/queries/0_stateless/02340_analyzer_functions.sql b/tests/queries/0_stateless/02340_analyzer_functions.sql index b1bfeabd836..101a5bfcc86 100644 --- a/tests/queries/0_stateless/02340_analyzer_functions.sql +++ b/tests/queries/0_stateless/02340_analyzer_functions.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DESCRIBE (SELECT 1 + 1); SELECT 1 + 1; diff --git a/tests/queries/0_stateless/02341_analyzer_aliases_basics.sql b/tests/queries/0_stateless/02341_analyzer_aliases_basics.sql index 636936d875a..52a1cd1dae8 100644 --- a/tests/queries/0_stateless/02341_analyzer_aliases_basics.sql +++ b/tests/queries/0_stateless/02341_analyzer_aliases_basics.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; SELECT 'Aliases to constants'; diff --git a/tests/queries/0_stateless/02342_analyzer_compound_types.sql b/tests/queries/0_stateless/02342_analyzer_compound_types.sql index 6b4f323fd66..0fd96928496 100644 --- a/tests/queries/0_stateless/02342_analyzer_compound_types.sql +++ b/tests/queries/0_stateless/02342_analyzer_compound_types.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; SELECT 'Constant tuple'; diff --git a/tests/queries/0_stateless/02343_analyzer_column_transformers_strict.sql b/tests/queries/0_stateless/02343_analyzer_column_transformers_strict.sql index 657e2098b4a..98ee7bc8f58 100644 --- a/tests/queries/0_stateless/02343_analyzer_column_transformers_strict.sql +++ b/tests/queries/0_stateless/02343_analyzer_column_transformers_strict.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table diff --git a/tests/queries/0_stateless/02343_analyzer_lambdas.sql b/tests/queries/0_stateless/02343_analyzer_lambdas.sql index 1c6200e5eb6..b90f7b32b57 100644 --- a/tests/queries/0_stateless/02343_analyzer_lambdas.sql +++ b/tests/queries/0_stateless/02343_analyzer_lambdas.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table diff --git a/tests/queries/0_stateless/02344_analyzer_multiple_aliases_for_expression.sql b/tests/queries/0_stateless/02344_analyzer_multiple_aliases_for_expression.sql index 2569dc8ba9c..cd1bca8285b 100644 --- a/tests/queries/0_stateless/02344_analyzer_multiple_aliases_for_expression.sql +++ b/tests/queries/0_stateless/02344_analyzer_multiple_aliases_for_expression.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table diff --git a/tests/queries/0_stateless/02345_analyzer_subqueries.sql b/tests/queries/0_stateless/02345_analyzer_subqueries.sql index 3694028e5d2..c0cc242b57b 100644 --- a/tests/queries/0_stateless/02345_analyzer_subqueries.sql +++ b/tests/queries/0_stateless/02345_analyzer_subqueries.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table @@ -32,7 +32,7 @@ WITH 1 AS global_a SELECT b.a FROM (SELECT global_a AS a) AS b; SELECT * FROM (SELECT * FROM (SELECT * FROM test_table)); SELECT * FROM (SELECT id, value FROM (SELECT * FROM test_table)); -WITH 1 AS a SELECT (SELECT * FROM (SELECT * FROM (SELECT a + 1))) SETTINGS use_analyzer=1; +WITH 1 AS a SELECT (SELECT * FROM (SELECT * FROM (SELECT a + 1))); SELECT 'Subqueries CTE'; diff --git a/tests/queries/0_stateless/02366_explain_query_tree.sql b/tests/queries/0_stateless/02366_explain_query_tree.sql index 23df6d12121..c942f0cac37 100644 --- a/tests/queries/0_stateless/02366_explain_query_tree.sql +++ b/tests/queries/0_stateless/02366_explain_query_tree.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; EXPLAIN QUERY TREE SELECT 1; diff --git a/tests/queries/0_stateless/02367_analyzer_table_alias_columns.sql b/tests/queries/0_stateless/02367_analyzer_table_alias_columns.sql index 769ba769f4d..f41680cd9f4 100644 --- a/tests/queries/0_stateless/02367_analyzer_table_alias_columns.sql +++ b/tests/queries/0_stateless/02367_analyzer_table_alias_columns.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table diff --git a/tests/queries/0_stateless/02368_analyzer_table_functions.sql b/tests/queries/0_stateless/02368_analyzer_table_functions.sql index 6d9accef30c..456e095c6c1 100644 --- a/tests/queries/0_stateless/02368_analyzer_table_functions.sql +++ b/tests/queries/0_stateless/02368_analyzer_table_functions.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; SELECT c1, c2, c3, c4 FROM format('CSV', '1,2,"[1,2,3]","[[\'abc\'], [], [\'d\', \'e\']]"'); SELECT f.c1, f.c2, f.c3, f.c4 FROM format('CSV', '1,2,"[1,2,3]","[[\'abc\'], [], [\'d\', \'e\']]"') AS f; diff --git a/tests/queries/0_stateless/02369_analyzer_array_join_function.sql b/tests/queries/0_stateless/02369_analyzer_array_join_function.sql index db9884f9cdd..9a9939d2a2f 100644 --- a/tests/queries/0_stateless/02369_analyzer_array_join_function.sql +++ b/tests/queries/0_stateless/02369_analyzer_array_join_function.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; SELECT arrayJoin([1, 2, 3]); diff --git a/tests/queries/0_stateless/02370_analyzer_in_function.sql b/tests/queries/0_stateless/02370_analyzer_in_function.sql index 01e5cf498cd..a7128ced449 100644 --- a/tests/queries/0_stateless/02370_analyzer_in_function.sql +++ b/tests/queries/0_stateless/02370_analyzer_in_function.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; SELECT 1 IN 1; SELECT 1 IN (1); diff --git a/tests/queries/0_stateless/02371_analyzer_join_cross.sql b/tests/queries/0_stateless/02371_analyzer_join_cross.sql index 0c0709fb951..8261572cdf2 100644 --- a/tests/queries/0_stateless/02371_analyzer_join_cross.sql +++ b/tests/queries/0_stateless/02371_analyzer_join_cross.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table_join_1; CREATE TABLE test_table_join_1 diff --git a/tests/queries/0_stateless/02372_analyzer_join.sql.j2 b/tests/queries/0_stateless/02372_analyzer_join.sql.j2 index 970c0d3da17..9b3c212562b 100644 --- a/tests/queries/0_stateless/02372_analyzer_join.sql.j2 +++ b/tests/queries/0_stateless/02372_analyzer_join.sql.j2 @@ -1,6 +1,6 @@ -- Tags: long -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table_join_1; CREATE TABLE test_table_join_1 diff --git a/tests/queries/0_stateless/02373_analyzer_join_use_nulls.sql b/tests/queries/0_stateless/02373_analyzer_join_use_nulls.sql index a67a11bf8b0..db7895084e8 100644 --- a/tests/queries/0_stateless/02373_analyzer_join_use_nulls.sql +++ b/tests/queries/0_stateless/02373_analyzer_join_use_nulls.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; SET join_use_nulls = 1; DROP TABLE IF EXISTS test_table_join_1; diff --git a/tests/queries/0_stateless/02374_analyzer_array_join.sql b/tests/queries/0_stateless/02374_analyzer_array_join.sql index b7e19bceadf..8ebfdef349c 100644 --- a/tests/queries/0_stateless/02374_analyzer_array_join.sql +++ b/tests/queries/0_stateless/02374_analyzer_array_join.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table diff --git a/tests/queries/0_stateless/02374_analyzer_join_using.sql.j2 b/tests/queries/0_stateless/02374_analyzer_join_using.sql.j2 index a8e9cc9eefc..26fb52716ff 100644 --- a/tests/queries/0_stateless/02374_analyzer_join_using.sql.j2 +++ b/tests/queries/0_stateless/02374_analyzer_join_using.sql.j2 @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table_join_1; CREATE TABLE test_table_join_1 diff --git a/tests/queries/0_stateless/02375_analyzer_union.sql b/tests/queries/0_stateless/02375_analyzer_union.sql index 4983040bd23..5e41f07d217 100644 --- a/tests/queries/0_stateless/02375_analyzer_union.sql +++ b/tests/queries/0_stateless/02375_analyzer_union.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 0; +SET allow_experimental_analyzer = 0; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table diff --git a/tests/queries/0_stateless/02376_analyzer_in_function_subquery.sql b/tests/queries/0_stateless/02376_analyzer_in_function_subquery.sql index 90b2f75b6c6..72a4edb8567 100644 --- a/tests/queries/0_stateless/02376_analyzer_in_function_subquery.sql +++ b/tests/queries/0_stateless/02376_analyzer_in_function_subquery.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table diff --git a/tests/queries/0_stateless/02377_analyzer_in_function_set.sql b/tests/queries/0_stateless/02377_analyzer_in_function_set.sql index e5c27e72ea1..e3cbcf75a9c 100644 --- a/tests/queries/0_stateless/02377_analyzer_in_function_set.sql +++ b/tests/queries/0_stateless/02377_analyzer_in_function_set.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table diff --git a/tests/queries/0_stateless/02378_analyzer_projection_names.sql b/tests/queries/0_stateless/02378_analyzer_projection_names.sql index a31f5afd8fb..907cc79dcec 100644 --- a/tests/queries/0_stateless/02378_analyzer_projection_names.sql +++ b/tests/queries/0_stateless/02378_analyzer_projection_names.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table diff --git a/tests/queries/0_stateless/02379_analyzer_subquery_depth.sql b/tests/queries/0_stateless/02379_analyzer_subquery_depth.sql index 3fea94e1f1c..c2109f543eb 100644 --- a/tests/queries/0_stateless/02379_analyzer_subquery_depth.sql +++ b/tests/queries/0_stateless/02379_analyzer_subquery_depth.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; SELECT (SELECT a FROM (SELECT 1 AS a)) SETTINGS max_subquery_depth = 1; -- { serverError 162 } SELECT (SELECT a FROM (SELECT 1 AS a)) SETTINGS max_subquery_depth = 2; diff --git a/tests/queries/0_stateless/02380_analyzer_join_sample.sql b/tests/queries/0_stateless/02380_analyzer_join_sample.sql index 1cf5a4a04df..e417f47d173 100644 --- a/tests/queries/0_stateless/02380_analyzer_join_sample.sql +++ b/tests/queries/0_stateless/02380_analyzer_join_sample.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table_join_1; CREATE TABLE test_table_join_1 diff --git a/tests/queries/0_stateless/02381_analyzer_join_final.sql b/tests/queries/0_stateless/02381_analyzer_join_final.sql index 6870b6e5f98..57fc3aedd8f 100644 --- a/tests/queries/0_stateless/02381_analyzer_join_final.sql +++ b/tests/queries/0_stateless/02381_analyzer_join_final.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table_join_1; CREATE TABLE test_table_join_1 diff --git a/tests/queries/0_stateless/02382_analyzer_matcher_join_using.sql b/tests/queries/0_stateless/02382_analyzer_matcher_join_using.sql index 8b6d9832086..25d493dc422 100644 --- a/tests/queries/0_stateless/02382_analyzer_matcher_join_using.sql +++ b/tests/queries/0_stateless/02382_analyzer_matcher_join_using.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table_join_1; CREATE TABLE test_table_join_1 diff --git a/tests/queries/0_stateless/02383_analyzer_merge_tree_self_join.sql b/tests/queries/0_stateless/02383_analyzer_merge_tree_self_join.sql index adb858c217f..c22a0f4244b 100644 --- a/tests/queries/0_stateless/02383_analyzer_merge_tree_self_join.sql +++ b/tests/queries/0_stateless/02383_analyzer_merge_tree_self_join.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table_join_1; CREATE TABLE test_table_join_1 diff --git a/tests/queries/0_stateless/02384_analyzer_dict_get_join_get.sql b/tests/queries/0_stateless/02384_analyzer_dict_get_join_get.sql index 79aa0703aaa..ff6e417d756 100644 --- a/tests/queries/0_stateless/02384_analyzer_dict_get_join_get.sql +++ b/tests/queries/0_stateless/02384_analyzer_dict_get_join_get.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table diff --git a/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql b/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql index ec2155d241b..1a195bbfffe 100644 --- a/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql +++ b/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; SELECT cast(tuple(1, 'Value'), 'Tuple(first UInt64, second String)') AS value, value.first, value.second; diff --git a/tests/queries/0_stateless/02386_analyzer_in_function_nested_subqueries.sql b/tests/queries/0_stateless/02386_analyzer_in_function_nested_subqueries.sql index bffbc46809a..c8ca3ff21d4 100644 --- a/tests/queries/0_stateless/02386_analyzer_in_function_nested_subqueries.sql +++ b/tests/queries/0_stateless/02386_analyzer_in_function_nested_subqueries.sql @@ -1,3 +1,3 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; SELECT (NULL IN (SELECT 9223372036854775806 IN (SELECT 65536), inf, NULL IN (NULL))) IN (SELECT NULL IN (NULL)); diff --git a/tests/queries/0_stateless/02387_analyzer_cte.sql b/tests/queries/0_stateless/02387_analyzer_cte.sql index 725e18af315..1f10ac10438 100644 --- a/tests/queries/0_stateless/02387_analyzer_cte.sql +++ b/tests/queries/0_stateless/02387_analyzer_cte.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table diff --git a/tests/queries/0_stateless/02388_analyzer_recursive_lambda.sql b/tests/queries/0_stateless/02388_analyzer_recursive_lambda.sql index dd342b32d1f..6fc8ff2aae0 100644 --- a/tests/queries/0_stateless/02388_analyzer_recursive_lambda.sql +++ b/tests/queries/0_stateless/02388_analyzer_recursive_lambda.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; WITH x -> plus(lambda(1), x) AS lambda SELECT lambda(1048576); -- { serverError 1 }; diff --git a/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql b/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql index 3be07a35425..8f8b5537da9 100644 --- a/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql +++ b/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql @@ -1,4 +1,4 @@ -SET use_analyzer = 1; +SET allow_experimental_analyzer = 1; -- { echoOn } From a28d1f01fd23e67eef8ab57f245b50f200d96c70 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 25 Oct 2022 12:54:17 +0200 Subject: [PATCH 38/50] Analyzer IQueryTreeNode remove getName method --- src/Analyzer/ColumnNode.h | 5 -- src/Analyzer/ConstantNode.h | 5 -- src/Analyzer/FunctionNode.cpp | 21 ----- src/Analyzer/FunctionNode.h | 2 - src/Analyzer/IQueryTreeNode.h | 8 -- src/Analyzer/IdentifierNode.h | 5 -- src/Analyzer/InterpolateNode.cpp | 9 --- src/Analyzer/InterpolateNode.h | 2 - src/Analyzer/LambdaNode.cpp | 5 -- src/Analyzer/LambdaNode.h | 2 - src/Analyzer/ListNode.cpp | 18 ----- src/Analyzer/ListNode.h | 2 - src/Analyzer/MatcherNode.cpp | 49 ------------ src/Analyzer/MatcherNode.h | 2 - src/Analyzer/Passes/QueryAnalysisPass.cpp | 2 +- src/Analyzer/QueryNode.cpp | 94 ----------------------- src/Analyzer/QueryNode.h | 2 - src/Analyzer/SortNode.cpp | 32 -------- src/Analyzer/SortNode.h | 3 - src/Analyzer/TableFunctionNode.cpp | 12 --- src/Analyzer/TableFunctionNode.h | 2 - src/Analyzer/TableNode.cpp | 5 -- src/Analyzer/TableNode.h | 2 - src/Analyzer/UnionNode.cpp | 40 ---------- src/Analyzer/UnionNode.h | 2 - src/Analyzer/WindowNode.cpp | 69 ----------------- src/Analyzer/WindowNode.h | 2 - src/Planner/PlannerActionsVisitor.cpp | 11 ++- 28 files changed, 11 insertions(+), 402 deletions(-) diff --git a/src/Analyzer/ColumnNode.h b/src/Analyzer/ColumnNode.h index fbd788ae6fd..e378bc5f3d0 100644 --- a/src/Analyzer/ColumnNode.h +++ b/src/Analyzer/ColumnNode.h @@ -112,11 +112,6 @@ public: return QueryTreeNodeType::COLUMN; } - String getName() const override - { - return column.name; - } - DataTypePtr getResultType() const override { return column.type; diff --git a/src/Analyzer/ConstantNode.h b/src/Analyzer/ConstantNode.h index 29e8cd25532..0a6868b72d4 100644 --- a/src/Analyzer/ConstantNode.h +++ b/src/Analyzer/ConstantNode.h @@ -51,11 +51,6 @@ public: return QueryTreeNodeType::CONSTANT; } - String getName() const override - { - return value_string; - } - DataTypePtr getResultType() const override { return constant_value->getType(); diff --git a/src/Analyzer/FunctionNode.cpp b/src/Analyzer/FunctionNode.cpp index 7468141b3d5..55e097cc5f4 100644 --- a/src/Analyzer/FunctionNode.cpp +++ b/src/Analyzer/FunctionNode.cpp @@ -93,27 +93,6 @@ void FunctionNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state } } -String FunctionNode::getName() const -{ - String name = function_name; - - const auto & parameters = getParameters(); - const auto & parameters_nodes = parameters.getNodes(); - if (!parameters_nodes.empty()) - { - name += '('; - name += parameters.getName(); - name += ')'; - } - - const auto & arguments = getArguments(); - name += '('; - name += arguments.getName(); - name += ')'; - - return name; -} - bool FunctionNode::isEqualImpl(const IQueryTreeNode & rhs) const { const auto & rhs_typed = assert_cast(rhs); diff --git a/src/Analyzer/FunctionNode.h b/src/Analyzer/FunctionNode.h index 18b4c6d445c..cb1bf0d8cc5 100644 --- a/src/Analyzer/FunctionNode.h +++ b/src/Analyzer/FunctionNode.h @@ -203,8 +203,6 @@ public: return result_type; } - String getName() const override; - void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: diff --git a/src/Analyzer/IQueryTreeNode.h b/src/Analyzer/IQueryTreeNode.h index 2ba96d27575..c3b067ab005 100644 --- a/src/Analyzer/IQueryTreeNode.h +++ b/src/Analyzer/IQueryTreeNode.h @@ -82,14 +82,6 @@ public: return toString(getNodeType()); } - /** Get name of query tree node that can be used as part of expression. - * TODO: Projection name, expression name must be refactored in better interface. - */ - virtual String getName() const - { - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Method getName is not supported for {} query node", getNodeTypeName()); - } - /** Get result type of query tree node that can be used as part of expression. * If node does not support this method exception is thrown. * TODO: Maybe this can be a part of ExpressionQueryTreeNode. diff --git a/src/Analyzer/IdentifierNode.h b/src/Analyzer/IdentifierNode.h index 7a2351f0ece..358511d1f90 100644 --- a/src/Analyzer/IdentifierNode.h +++ b/src/Analyzer/IdentifierNode.h @@ -50,11 +50,6 @@ public: return QueryTreeNodeType::IDENTIFIER; } - String getName() const override - { - return identifier.getFullName(); - } - void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: diff --git a/src/Analyzer/InterpolateNode.cpp b/src/Analyzer/InterpolateNode.cpp index dcc14d6b6d5..c8c61b05853 100644 --- a/src/Analyzer/InterpolateNode.cpp +++ b/src/Analyzer/InterpolateNode.cpp @@ -17,15 +17,6 @@ InterpolateNode::InterpolateNode(QueryTreeNodePtr expression_, QueryTreeNodePtr children[interpolate_expression_child_index] = std::move(interpolate_expression_); } -String InterpolateNode::getName() const -{ - String result = getExpression()->getName(); - result += " AS "; - result += getInterpolateExpression()->getName(); - - return result; -} - void InterpolateNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const { buffer << std::string(indent, ' ') << "INTERPOLATE id: " << format_state.getNodeId(this); diff --git a/src/Analyzer/InterpolateNode.h b/src/Analyzer/InterpolateNode.h index 5bc8eded0bb..5764ea561c0 100644 --- a/src/Analyzer/InterpolateNode.h +++ b/src/Analyzer/InterpolateNode.h @@ -50,8 +50,6 @@ public: return QueryTreeNodeType::INTERPOLATE; } - String getName() const override; - void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: diff --git a/src/Analyzer/LambdaNode.cpp b/src/Analyzer/LambdaNode.cpp index ccc43e75644..809f73072d2 100644 --- a/src/Analyzer/LambdaNode.cpp +++ b/src/Analyzer/LambdaNode.cpp @@ -44,11 +44,6 @@ void LambdaNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, getExpression()->dumpTreeImpl(buffer, format_state, indent + 4); } -String LambdaNode::getName() const -{ - return "lambda(" + children[arguments_child_index]->getName() + ") -> " + children[expression_child_index]->getName(); -} - bool LambdaNode::isEqualImpl(const IQueryTreeNode & rhs) const { const auto & rhs_typed = assert_cast(rhs); diff --git a/src/Analyzer/LambdaNode.h b/src/Analyzer/LambdaNode.h index 6061e854ab0..ad80c645656 100644 --- a/src/Analyzer/LambdaNode.h +++ b/src/Analyzer/LambdaNode.h @@ -84,8 +84,6 @@ public: return QueryTreeNodeType::LAMBDA; } - String getName() const override; - DataTypePtr getResultType() const override { return getExpression()->getResultType(); diff --git a/src/Analyzer/ListNode.cpp b/src/Analyzer/ListNode.cpp index b98e29deecd..7bbb884fa7f 100644 --- a/src/Analyzer/ListNode.cpp +++ b/src/Analyzer/ListNode.cpp @@ -38,24 +38,6 @@ void ListNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, si } } -String ListNode::getName() const -{ - if (children.empty()) - return ""; - - std::string result; - for (const auto & node : children) - { - result += node->getName(); - result += ", "; - } - - result.pop_back(); - result.pop_back(); - - return result; -} - bool ListNode::isEqualImpl(const IQueryTreeNode &) const { /// No state diff --git a/src/Analyzer/ListNode.h b/src/Analyzer/ListNode.h index 7d941731b25..5b328d0f870 100644 --- a/src/Analyzer/ListNode.h +++ b/src/Analyzer/ListNode.h @@ -39,8 +39,6 @@ public: return QueryTreeNodeType::LIST; } - String getName() const override; - void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: diff --git a/src/Analyzer/MatcherNode.cpp b/src/Analyzer/MatcherNode.cpp index 0bed9110d25..9d822771087 100644 --- a/src/Analyzer/MatcherNode.cpp +++ b/src/Analyzer/MatcherNode.cpp @@ -146,55 +146,6 @@ void MatcherNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, } } -String MatcherNode::getName() const -{ - WriteBufferFromOwnString buffer; - - if (!qualified_identifier.empty()) - buffer << qualified_identifier.getFullName() << '.'; - - if (matcher_type == MatcherNodeType::ASTERISK) - { - buffer << '*'; - } - else - { - buffer << "COLUMNS("; - - if (columns_matcher) - { - buffer << ' ' << columns_matcher->pattern(); - } - else if (matcher_type == MatcherNodeType::COLUMNS_LIST) - { - size_t columns_identifiers_size = columns_identifiers.size(); - for (size_t i = 0; i < columns_identifiers_size; ++i) - { - buffer << columns_identifiers[i].getFullName(); - - if (i + 1 != columns_identifiers_size) - buffer << ", "; - } - } - } - - buffer << ')'; - - const auto & column_transformers = getColumnTransformers().getNodes(); - size_t column_transformers_size = column_transformers.size(); - - for (size_t i = 0; i < column_transformers_size; ++i) - { - const auto & column_transformer = column_transformers[i]; - buffer << column_transformer->getName(); - - if (i + 1 != column_transformers_size) - buffer << ' '; - } - - return buffer.str(); -} - bool MatcherNode::isEqualImpl(const IQueryTreeNode & rhs) const { const auto & rhs_typed = assert_cast(rhs); diff --git a/src/Analyzer/MatcherNode.h b/src/Analyzer/MatcherNode.h index 3635d038549..e79c1cb4bf2 100644 --- a/src/Analyzer/MatcherNode.h +++ b/src/Analyzer/MatcherNode.h @@ -139,8 +139,6 @@ public: return QueryTreeNodeType::MATCHER; } - String getName() const override; - void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index e55f296e1b7..6b91d0f8053 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -2255,7 +2255,7 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo for (auto & join_using_node : join_using_list.getNodes()) { auto & column_node = join_using_node->as(); - join_using_column_name_to_column_node.emplace(column_node.getName(), std::static_pointer_cast(join_using_node)); + join_using_column_name_to_column_node.emplace(column_node.getColumnName(), std::static_pointer_cast(join_using_node)); } } diff --git a/src/Analyzer/QueryNode.cpp b/src/Analyzer/QueryNode.cpp index 378bd7259bf..557baad2654 100644 --- a/src/Analyzer/QueryNode.cpp +++ b/src/Analyzer/QueryNode.cpp @@ -32,100 +32,6 @@ QueryNode::QueryNode() children[limit_by_child_index] = std::make_shared(); } -String QueryNode::getName() const -{ - WriteBufferFromOwnString buffer; - - if (hasWith()) - { - buffer << getWith().getName(); - buffer << ' '; - } - - buffer << "SELECT "; - buffer << getProjection().getName(); - - if (getJoinTree()) - { - buffer << " FROM "; - buffer << getJoinTree()->getName(); - } - - if (getPrewhere()) - { - buffer << " PREWHERE "; - buffer << getPrewhere()->getName(); - } - - if (getWhere()) - { - buffer << " WHERE "; - buffer << getWhere()->getName(); - } - - if (hasGroupBy()) - { - buffer << " GROUP BY "; - buffer << getGroupBy().getName(); - } - - if (hasHaving()) - { - buffer << " HAVING "; - buffer << getHaving()->getName(); - } - - if (hasWindow()) - { - buffer << " WINDOW "; - buffer << getWindow().getName(); - } - - if (hasOrderBy()) - { - buffer << " ORDER BY "; - buffer << getOrderByNode()->getName(); - } - - if (hasInterpolate()) - { - buffer << " INTERPOLATE "; - buffer << getInterpolate()->getName(); - } - - if (hasLimitByLimit()) - { - buffer << "LIMIT "; - buffer << getLimitByLimit()->getName(); - } - - if (hasLimitByOffset()) - { - buffer << "OFFSET "; - buffer << getLimitByOffset()->getName(); - } - - if (hasLimitBy()) - { - buffer << " BY "; - buffer << getLimitBy().getName(); - } - - if (hasLimit()) - { - buffer << " LIMIT "; - buffer << getLimit()->getName(); - } - - if (hasOffset()) - { - buffer << " OFFSET "; - buffer << getOffset()->getName(); - } - - return buffer.str(); -} - void QueryNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const { buffer << std::string(indent, ' ') << "QUERY id: " << format_state.getNodeId(this); diff --git a/src/Analyzer/QueryNode.h b/src/Analyzer/QueryNode.h index 05d393b4212..6bb6613fc2b 100644 --- a/src/Analyzer/QueryNode.h +++ b/src/Analyzer/QueryNode.h @@ -559,8 +559,6 @@ public: return QueryTreeNodeType::QUERY; } - String getName() const override; - DataTypePtr getResultType() const override { if (constant_value) diff --git a/src/Analyzer/SortNode.cpp b/src/Analyzer/SortNode.cpp index f6faccdb7c5..3f91724e9b7 100644 --- a/src/Analyzer/SortNode.cpp +++ b/src/Analyzer/SortNode.cpp @@ -35,38 +35,6 @@ SortNode::SortNode(QueryTreeNodePtr expression_, children[sort_expression_child_index] = std::move(expression_); } -String SortNode::getName() const -{ - String result = getExpression()->getName(); - - if (sort_direction == SortDirection::ASCENDING) - result += " ASC"; - else - result += " DESC"; - - if (nulls_sort_direction) - { - if (*nulls_sort_direction == SortDirection::ASCENDING) - result += " NULLS FIRST"; - else - result += " NULLS LAST"; - } - - if (with_fill) - result += " WITH FILL"; - - if (hasFillFrom()) - result += " FROM " + getFillFrom()->getName(); - - if (hasFillStep()) - result += " STEP " + getFillStep()->getName(); - - if (hasFillTo()) - result += " TO " + getFillTo()->getName(); - - return result; -} - void SortNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const { buffer << std::string(indent, ' ') << "SORT id: " << format_state.getNodeId(this); diff --git a/src/Analyzer/SortNode.h b/src/Analyzer/SortNode.h index b0fe835cf45..04f9fe798e1 100644 --- a/src/Analyzer/SortNode.h +++ b/src/Analyzer/SortNode.h @@ -128,11 +128,8 @@ public: return QueryTreeNodeType::SORT; } - String getName() const override; - void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; - protected: bool isEqualImpl(const IQueryTreeNode & rhs) const override; diff --git a/src/Analyzer/TableFunctionNode.cpp b/src/Analyzer/TableFunctionNode.cpp index 2056e10ff6f..c8cd05cf685 100644 --- a/src/Analyzer/TableFunctionNode.cpp +++ b/src/Analyzer/TableFunctionNode.cpp @@ -50,18 +50,6 @@ const StorageSnapshotPtr & TableFunctionNode::getStorageSnapshot() const return storage_snapshot; } -String TableFunctionNode::getName() const -{ - String name = table_function_name; - - const auto & arguments = getArguments(); - name += '('; - name += arguments.getName(); - name += ')'; - - return name; -} - void TableFunctionNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const { buffer << std::string(indent, ' ') << "TABLE_FUNCTION id: " << format_state.getNodeId(this); diff --git a/src/Analyzer/TableFunctionNode.h b/src/Analyzer/TableFunctionNode.h index a9f08ed4d1a..292ab740c5b 100644 --- a/src/Analyzer/TableFunctionNode.h +++ b/src/Analyzer/TableFunctionNode.h @@ -127,8 +127,6 @@ public: return QueryTreeNodeType::TABLE_FUNCTION; } - String getName() const override; - void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: diff --git a/src/Analyzer/TableNode.cpp b/src/Analyzer/TableNode.cpp index 3075bce238a..0d9a351e9a2 100644 --- a/src/Analyzer/TableNode.cpp +++ b/src/Analyzer/TableNode.cpp @@ -66,11 +66,6 @@ void TableNode::updateTreeHashImpl(HashState & state) const table_expression_modifiers->updateTreeHash(state); } -String TableNode::getName() const -{ - return storage->getStorageID().getFullNameNotQuoted(); -} - QueryTreeNodePtr TableNode::cloneImpl() const { auto result_table_node = std::make_shared(storage, storage_id, storage_lock, storage_snapshot); diff --git a/src/Analyzer/TableNode.h b/src/Analyzer/TableNode.h index f8e897378d6..c7feedd908f 100644 --- a/src/Analyzer/TableNode.h +++ b/src/Analyzer/TableNode.h @@ -76,8 +76,6 @@ public: return QueryTreeNodeType::TABLE; } - String getName() const override; - void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: diff --git a/src/Analyzer/UnionNode.cpp b/src/Analyzer/UnionNode.cpp index fe913373981..b8ed46c645e 100644 --- a/src/Analyzer/UnionNode.cpp +++ b/src/Analyzer/UnionNode.cpp @@ -79,46 +79,6 @@ NamesAndTypes UnionNode::computeProjectionColumns() const return result_columns; } -String UnionNode::getName() const -{ - WriteBufferFromOwnString buffer; - - auto query_nodes = getQueries().getNodes(); - size_t query_nodes_size = query_nodes.size(); - - for (size_t i = 0; i < query_nodes_size; ++i) - { - const auto & query_node = query_nodes[i]; - buffer << query_node->getName(); - - if (i == 0) - continue; - - auto query_union_mode = union_modes.at(i - 1); - - if (query_union_mode == SelectUnionMode::UNION_DEFAULT) - buffer << "UNION"; - else if (query_union_mode == SelectUnionMode::UNION_ALL) - buffer << "UNION ALL"; - else if (query_union_mode == SelectUnionMode::UNION_DISTINCT) - buffer << "UNION DISTINCT"; - else if (query_union_mode == SelectUnionMode::EXCEPT_DEFAULT) - buffer << "EXCEPT"; - else if (query_union_mode == SelectUnionMode::EXCEPT_ALL) - buffer << "EXCEPT ALL"; - else if (query_union_mode == SelectUnionMode::EXCEPT_DISTINCT) - buffer << "EXCEPT DISTINCT"; - else if (query_union_mode == SelectUnionMode::INTERSECT_DEFAULT) - buffer << "INTERSECT"; - else if (query_union_mode == SelectUnionMode::INTERSECT_ALL) - buffer << "INTERSECT ALL"; - else if (query_union_mode == SelectUnionMode::INTERSECT_DISTINCT) - buffer << "INTERSECT DISTINCT"; - } - - return buffer.str(); -} - void UnionNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const { buffer << std::string(indent, ' ') << "UNION id: " << format_state.getNodeId(this); diff --git a/src/Analyzer/UnionNode.h b/src/Analyzer/UnionNode.h index 9c502c8fe5b..05e70b87a27 100644 --- a/src/Analyzer/UnionNode.h +++ b/src/Analyzer/UnionNode.h @@ -154,8 +154,6 @@ public: return QueryTreeNodeType::UNION; } - String getName() const override; - DataTypePtr getResultType() const override { if (constant_value) diff --git a/src/Analyzer/WindowNode.cpp b/src/Analyzer/WindowNode.cpp index ccf49bc37d8..3e8537302e5 100644 --- a/src/Analyzer/WindowNode.cpp +++ b/src/Analyzer/WindowNode.cpp @@ -18,75 +18,6 @@ WindowNode::WindowNode(WindowFrame window_frame_) children[order_by_child_index] = std::make_shared(); } -String WindowNode::getName() const -{ - String result; - - if (hasPartitionBy()) - { - result += "PARTITION BY"; - result += getPartitionBy().getName(); - } - - if (hasOrderBy()) - { - result += "ORDER BY"; - result += getOrderBy().getName(); - } - - if (!window_frame.is_default) - { - if (hasPartitionBy() || hasOrderBy()) - result += ' '; - - if (window_frame.type == WindowFrame::FrameType::ROWS) - result += "ROWS"; - else if (window_frame.type == WindowFrame::FrameType::GROUPS) - result += "GROUPS"; - else if (window_frame.type == WindowFrame::FrameType::RANGE) - result += "RANGE"; - - result += " BETWEEN "; - if (window_frame.begin_type == WindowFrame::BoundaryType::Current) - { - result += "CURRENT ROW"; - } - else if (window_frame.begin_type == WindowFrame::BoundaryType::Unbounded) - { - result += "UNBOUNDED"; - result += " "; - result += (window_frame.begin_preceding ? "PRECEDING" : "FOLLOWING"); - } - else - { - result += getFrameBeginOffsetNode()->getName(); - result += " "; - result += (window_frame.begin_preceding ? "PRECEDING" : "FOLLOWING"); - } - - result += " AND "; - - if (window_frame.end_type == WindowFrame::BoundaryType::Current) - { - result += "CURRENT ROW"; - } - else if (window_frame.end_type == WindowFrame::BoundaryType::Unbounded) - { - result += "UNBOUNDED"; - result += " "; - result += (window_frame.end_preceding ? "PRECEDING" : "FOLLOWING"); - } - else - { - result += getFrameEndOffsetNode()->getName(); - result += " "; - result += (window_frame.end_preceding ? "PRECEDING" : "FOLLOWING"); - } - } - - return result; -} - void WindowNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const { buffer << std::string(indent, ' ') << "WINDOW id: " << format_state.getNodeId(this); diff --git a/src/Analyzer/WindowNode.h b/src/Analyzer/WindowNode.h index 2f99524eebd..9dfb3e6ef2a 100644 --- a/src/Analyzer/WindowNode.h +++ b/src/Analyzer/WindowNode.h @@ -166,8 +166,6 @@ public: return QueryTreeNodeType::WINDOW; } - String getName() const override; - void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index 7bc53127101..a6f1a74f251 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -502,7 +502,16 @@ String calculateActionNodeName(const QueryTreeNodePtr & node, const PlannerConte case QueryTreeNodeType::COLUMN: { const auto * column_identifier = planner_context.getColumnNodeIdentifierOrNull(node); - result = column_identifier ? *column_identifier : node->getName(); + + if (column_identifier) + { + result = *column_identifier; + } + else + { + const auto & column_node = node->as(); + result = column_node.getColumnName(); + } break; } From e124920d02c9e591207e86b59b3269fb4a471012 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Tue, 25 Oct 2022 12:08:51 +0000 Subject: [PATCH 39/50] Remove flaky queries from 02457_datediff_via_unix_epoch test --- .../0_stateless/02457_datediff_via_unix_epoch.reference | 4 ---- tests/queries/0_stateless/02457_datediff_via_unix_epoch.sql | 5 ----- 2 files changed, 9 deletions(-) diff --git a/tests/queries/0_stateless/02457_datediff_via_unix_epoch.reference b/tests/queries/0_stateless/02457_datediff_via_unix_epoch.reference index ba12c868037..8b2de7a3eec 100644 --- a/tests/queries/0_stateless/02457_datediff_via_unix_epoch.reference +++ b/tests/queries/0_stateless/02457_datediff_via_unix_epoch.reference @@ -8,9 +8,5 @@ week 1 week 1 day 11 day 11 -hour 264 -hour 264 minute 1440 -minute 20 second 86400 -second 1200 diff --git a/tests/queries/0_stateless/02457_datediff_via_unix_epoch.sql b/tests/queries/0_stateless/02457_datediff_via_unix_epoch.sql index 796b4cc6e8f..ce977712943 100644 --- a/tests/queries/0_stateless/02457_datediff_via_unix_epoch.sql +++ b/tests/queries/0_stateless/02457_datediff_via_unix_epoch.sql @@ -13,11 +13,6 @@ select 'week', date_diff('week', toDateTime64('1969-12-25 10:00:00.000', 3), toD select 'day', date_diff('day', toDate32('1969-12-25'), toDate32('1970-01-05')); select 'day', date_diff('day', toDateTime64('1969-12-25 10:00:00.000', 3), toDateTime64('1970-01-05 10:00:00.000', 3)); -select 'hour', date_diff('hour', toDate32('1969-12-25'), toDate32('1970-01-05')); -select 'hour', date_diff('hour', toDateTime64('1969-12-25 10:00:00.000', 3), toDateTime64('1970-01-05 10:00:00.000', 3)); - select 'minute', date_diff('minute', toDate32('1969-12-31'), toDate32('1970-01-01')); -select 'minute', date_diff('minute', toDateTime64('1969-12-31 23:50:00.000', 3), toDateTime64('1970-01-01 00:10:00.000', 3)); select 'second', date_diff('second', toDate32('1969-12-31'), toDate32('1970-01-01')); -select 'second', date_diff('second', toDateTime64('1969-12-31 23:50:00.000', 3), toDateTime64('1970-01-01 00:10:00.000', 3)); From 9c9b8f4400bf88c26d2cc8ed0d50e4ef1bff533e Mon Sep 17 00:00:00 2001 From: Jordi Villar Date: Tue, 25 Oct 2022 15:23:28 +0200 Subject: [PATCH 40/50] Minor fix iotest_nonblock build --- utils/iotest/iotest_nonblock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/iotest/iotest_nonblock.cpp b/utils/iotest/iotest_nonblock.cpp index 33fab4d04e6..6db00045e03 100644 --- a/utils/iotest/iotest_nonblock.cpp +++ b/utils/iotest/iotest_nonblock.cpp @@ -101,7 +101,7 @@ int mainImpl(int argc, char ** argv) size_t ops = 0; while (ops < count) { - if (poll(polls.data(), descriptors, -1) <= 0) + if (poll(polls.data(), static_cast(descriptors), -1) <= 0) throwFromErrno("poll failed", ErrorCodes::SYSTEM_ERROR); for (size_t i = 0; i < descriptors; ++i) { From 40ac6fda7420073c1ec717e0ebab199195c98c78 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 25 Oct 2022 13:43:14 +0000 Subject: [PATCH 41/50] Fixing test. --- ...461_mullable_pk_monotonicity_bug.reference | 32 +++++++++---------- .../02461_mullable_pk_monotonicity_bug.sql | 18 +++++------ 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference index 29da9bc651b..c0d3de1806a 100644 --- a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference +++ b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.reference @@ -46,19 +46,19 @@ 2 1 2 -1970-01-01 00:00:01 -1970-01-01 00:00:02 -1970-01-01 00:00:01 -1970-01-01 00:00:02 -1970-01-01 00:00:01 -1970-01-01 00:00:02 -1970-01-01 00:00:01 -1970-01-01 00:00:02 -1970-01-01 00:00:01 -1970-01-01 00:00:02 -1970-01-01 00:00:01 -1970-01-01 00:00:02 -1970-01-01 00:00:01 -1970-01-01 00:00:02 -1970-01-01 00:00:01 -1970-01-01 00:00:02 +2022-02-02 00:00:01 +2022-02-02 00:00:02 +2022-02-02 00:00:01 +2022-02-02 00:00:02 +2022-02-02 00:00:01 +2022-02-02 00:00:02 +2022-02-02 00:00:01 +2022-02-02 00:00:02 +2022-02-02 00:00:01 +2022-02-02 00:00:02 +2022-02-02 00:00:01 +2022-02-02 00:00:02 +2022-02-02 00:00:01 +2022-02-02 00:00:02 +2022-02-02 00:00:01 +2022-02-02 00:00:02 diff --git a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql index af2d5b859b3..75c8cb2b7e7 100644 --- a/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql +++ b/tests/queries/0_stateless/02461_mullable_pk_monotonicity_bug.sql @@ -47,16 +47,16 @@ SELECT x + 1 FROM tab WHERE (x + 1::LowCardinality(UInt8)) <= -92233720368547758 drop table tab; create table tab (x DateTime) engine = MergeTree order by x settings allow_nullable_key = 1, index_granularity = 2; -insert into tab select number from numbers(4); +insert into tab select toDateTime('2022-02-02') + number from numbers(4); set max_rows_to_read = 2; -SELECT x + 1 FROM tab where plus(x, 1) <= 2 order by x; -SELECT x + 1 FROM tab where plus(x, 1::Nullable(UInt8)) <= 2 order by x; -SELECT x + 1 FROM tab where plus(x, 1::LowCardinality(UInt8)) <= 2 order by x; -SELECT x + 1 FROM tab where plus(x, 1::LowCardinality(Nullable(UInt8))) <= 2 order by x; -SELECT 1 + x FROM tab where plus(1, x) <= 2 order by x; -SELECT 1 + x FROM tab where plus(1::Nullable(UInt8), x) <= 2 order by x; -SELECT 1 + x FROM tab where plus(1::LowCardinality(UInt8), x) <= 2 order by x; -SELECT 1 + x FROM tab where plus(1::LowCardinality(Nullable(UInt8)), x) <= 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1) <= toDateTime('2022-02-02') + 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::Nullable(UInt8)) <= toDateTime('2022-02-02') + 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::LowCardinality(UInt8)) <= toDateTime('2022-02-02') + 2 order by x; +SELECT x + 1 FROM tab where plus(x, 1::LowCardinality(Nullable(UInt8))) <= toDateTime('2022-02-02') + 2 order by x; +SELECT 1 + x FROM tab where plus(1, x) <= toDateTime('2022-02-02') + 2 order by x; +SELECT 1 + x FROM tab where plus(1::Nullable(UInt8), x) <= toDateTime('2022-02-02') + 2 order by x; +SELECT 1 + x FROM tab where plus(1::LowCardinality(UInt8), x) <= toDateTime('2022-02-02') + 2 order by x; +SELECT 1 + x FROM tab where plus(1::LowCardinality(Nullable(UInt8)), x) <= toDateTime('2022-02-02') + 2 order by x; SELECT x + 1 FROM tab WHERE (x + CAST('1', 'Nullable(UInt8)')) <= -2147483647 ORDER BY x ASC NULLS FIRST; From 97e4d5a7d826fca0a5a0dcb7bf67034efef1a60f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 26 Oct 2022 06:19:39 +0300 Subject: [PATCH 42/50] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2def2f787e..8b97622dcd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ * Add support for methods `lz4`, `bz2`, `snappy` in HTTP's `Accept-Encoding` which is a non-standard extension to HTTP protocol. [#42071](https://github.com/ClickHouse/ClickHouse/pull/42071) ([Nikolay Degterinsky](https://github.com/evillique)). #### Experimental Feature +* Added new infrastructure for query analysis and planning under the `allow_experimental_analyzer` setting. [#31796](https://github.com/ClickHouse/ClickHouse/pull/31796) ([Maksim Kita](https://github.com/kitaisreal)). * Initial implementation of Kusto Query Language. Please don't use it. [#37961](https://github.com/ClickHouse/ClickHouse/pull/37961) ([Yong Wang](https://github.com/kashwy)). #### Performance Improvement From cca0d0a4b8564c5034b1914e7d22dac8b0ee10b1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 26 Oct 2022 06:24:57 +0300 Subject: [PATCH 43/50] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b97622dcd1..22f6afc4901 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ **[ClickHouse release v22.1, 2022-01-18](#221)**
**[Changelog for 2021](https://clickhouse.com/docs/en/whats-new/changelog/2021/)**
-### ClickHouse release 22.10, 2022-10-25 +### ClickHouse release 22.10, 2022-10-26 #### Backward Incompatible Change * Rename cache commands: `show caches` -> `show filesystem caches`, `describe cache` -> `describe filesystem cache`. [#41508](https://github.com/ClickHouse/ClickHouse/pull/41508) ([Kseniia Sumarokova](https://github.com/kssenii)). From 117674c0cef594c125b472020a9e8339538d92ca Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 26 Oct 2022 08:48:12 +0000 Subject: [PATCH 44/50] Update version_date.tsv and changelogs after v22.10.1.1875-stable --- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v22.10.1.1875-stable.md | 351 ++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 1 + 4 files changed, 354 insertions(+), 2 deletions(-) create mode 100644 docs/changelogs/v22.10.1.1875-stable.md diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 5b597f927a2..9b633b66188 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="22.9.3.18" +ARG VERSION="22.10.1.1875" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index c6254b898ed..35c78763a31 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -21,7 +21,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="22.9.3.18" +ARG VERSION="22.10.1.1875" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v22.10.1.1875-stable.md b/docs/changelogs/v22.10.1.1875-stable.md new file mode 100644 index 00000000000..49f93168a00 --- /dev/null +++ b/docs/changelogs/v22.10.1.1875-stable.md @@ -0,0 +1,351 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + +### ClickHouse release v22.10.1.1875-stable (011ae8675a2) FIXME as compared to v22.9.1.2603-stable (3030d4c7ff0) + +#### Backward Incompatible Change +* Rename cache commands: `show caches` -> `show filesystem caches`, `describe cache` -> `describe filesystem cache`. [#41508](https://github.com/ClickHouse/ClickHouse/pull/41508) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove support for the `WITH TIMEOUT` section for `LIVE VIEW`. This closes [#40557](https://github.com/ClickHouse/ClickHouse/issues/40557). [#42173](https://github.com/ClickHouse/ClickHouse/pull/42173) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### New Feature +* Add Rust code support into ClickHouse with BLAKE3 hash-function library as an example. [#33435](https://github.com/ClickHouse/ClickHouse/pull/33435) ([BoloniniD](https://github.com/BoloniniD)). +* This is the initial implement of Kusto Query Language. (MVP). [#37961](https://github.com/ClickHouse/ClickHouse/pull/37961) ([Yong Wang](https://github.com/kashwy)). +* * Support limiting of temporary data stored on disk using settings `max_temporary_data_on_disk_size_for_user`/`max_temporary_data_on_disk_size_for_query` . [#40893](https://github.com/ClickHouse/ClickHouse/pull/40893) ([Vladimir C](https://github.com/vdimir)). +* Support Java integers hashing in `javaHash`. [#41131](https://github.com/ClickHouse/ClickHouse/pull/41131) ([JackyWoo](https://github.com/JackyWoo)). +* This PR is to support the OpenSSL in-house build like the BoringSSL submodule. Build flag i.e. ENABLE_CH_BUNDLE_BORINGSSL is used to choose between BoringSSL and OpenSSL. By default, the BoringSSL in-house build will be used. [#41142](https://github.com/ClickHouse/ClickHouse/pull/41142) ([MeenaRenganathan22](https://github.com/MeenaRenganathan22)). +* Composable protocol configuration is added. [#41198](https://github.com/ClickHouse/ClickHouse/pull/41198) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Add OpenTelemetry support to ON CLUSTER DDL(require `distributed_ddl_entry_format_version` to be set to 4). [#41484](https://github.com/ClickHouse/ClickHouse/pull/41484) ([Frank Chen](https://github.com/FrankChen021)). +* Add setting `format_json_object_each_row_column_for_object_name` to write/parse object name as column value in JSONObjectEachRow format. [#41703](https://github.com/ClickHouse/ClickHouse/pull/41703) ([Kruglov Pavel](https://github.com/Avogar)). +* adds Morton Coding (ZCurve) encode/decode functions. [#41753](https://github.com/ClickHouse/ClickHouse/pull/41753) ([Constantine Peresypkin](https://github.com/pkit)). +* Implement support for different UUID binary formats with support for the two most prevalent ones: the default big-endian and Microsoft's mixed-endian as specified in [RFC 4122](https://datatracker.ietf.org/doc/html/rfc4122#section-4.1.1). [#42108](https://github.com/ClickHouse/ClickHouse/pull/42108) ([ltrk2](https://github.com/ltrk2)). +* Added an aggregate function `analysisOfVariance` (`anova`) to perform a statistical test over several groups of normally distributed observations to find out whether all groups have the same mean or not. Original PR [#37872](https://github.com/ClickHouse/ClickHouse/issues/37872). [#42131](https://github.com/ClickHouse/ClickHouse/pull/42131) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add support for `SET setting_name = DEFAULT`. [#42187](https://github.com/ClickHouse/ClickHouse/pull/42187) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* * Add `URL` Functions which conform rfc. Functions include: `cutToFirstSignificantSubdomainCustomRFC`, `cutToFirstSignificantSubdomainCustomWithWWWRFC`, `cutToFirstSignificantSubdomainRFC`, `cutToFirstSignificantSubdomainWithWWWRFC`, `domainRFC`, `domainWithoutWWWRFC`, `firstSignificantSubdomainCustomRFC`, `firstSignificantSubdomainRFC`, `portRFC`, `topLevelDomainRFC`. [#42274](https://github.com/ClickHouse/ClickHouse/pull/42274) ([Quanfa Fu](https://github.com/dentiscalprum)). +* Added functions (`randUniform`, `randNormal`, `randLogNormal`, `randExponential`, `randChiSquared`, `randStudentT`, `randFisherF`, `randBernoulli`, `randBinomial`, `randNegativeBinomial`, `randPoisson` ) to generate random values according to the specified distributions. This closes [#21834](https://github.com/ClickHouse/ClickHouse/issues/21834). [#42411](https://github.com/ClickHouse/ClickHouse/pull/42411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### Performance Improvement +* Implement operator precedence element parser to resolve stack overflow issues and make the required stack size smaller. [#34892](https://github.com/ClickHouse/ClickHouse/pull/34892) ([Nikolay Degterinsky](https://github.com/evillique)). +* DISTINCT in order optimization leverage sorting properties of data streams. This improvement will enable reading in order for DISTINCT if applicable (before it was necessary to provide ORDER BY for columns in DISTINCT). [#41014](https://github.com/ClickHouse/ClickHouse/pull/41014) ([Igor Nikonov](https://github.com/devcrafter)). +* ColumnVector: optimize UInt8 index with AVX512VBMI. [#41247](https://github.com/ClickHouse/ClickHouse/pull/41247) ([Guo Wangyang](https://github.com/guowangy)). +* The performance experiments of **SSB** (Star Schema Benchmark) on the ICX device (Intel Xeon Platinum 8380 CPU, 80 cores, 160 threads) shows that this change could bring a **2.95x** improvement of the geomean of all subcases' QPS. [#41675](https://github.com/ClickHouse/ClickHouse/pull/41675) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). +* Fixed slowness in JSONExtract with LowCardinality(String) tuples. [#41726](https://github.com/ClickHouse/ClickHouse/pull/41726) ([AlfVII](https://github.com/AlfVII)). +* Add ldapr capabilities to AArch64 builds. This is supported from Graviton 2+, Azure and GCP instances. Only appeared in clang-15 [not so long ago](https://github.com/llvm/llvm-project/commit/9609b5daffe9fd28d83d83da895abc5113f76c24). [#41778](https://github.com/ClickHouse/ClickHouse/pull/41778) ([Daniel Kutenin](https://github.com/danlark1)). +* Improve performance when comparing strings and one argument is empty constant string. [#41870](https://github.com/ClickHouse/ClickHouse/pull/41870) ([Jiebin Sun](https://github.com/jiebinn)). +* optimize insertFrom of ColumnAggregateFunction to share Aggregate State in some cases. [#41960](https://github.com/ClickHouse/ClickHouse/pull/41960) ([flynn](https://github.com/ucasfl)). +* Relax the "Too many parts" threshold. This closes [#6551](https://github.com/ClickHouse/ClickHouse/issues/6551). Now ClickHouse will allow more parts in a partition if the average part size is large enough (at least 10 GiB). This allows to have up to petabytes of data in a single partition of a single table on a single server, which is possible using disk shelves or object storage. [#42002](https://github.com/ClickHouse/ClickHouse/pull/42002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make writing to AzureBlobStorage more efficient (respect `max_single_part_upload_size` instead of writing a block per each buffer size). Inefficiency mentioned in [#41754](https://github.com/ClickHouse/ClickHouse/issues/41754). [#42041](https://github.com/ClickHouse/ClickHouse/pull/42041) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Make thread ids in the process list and query_log unique to avoid waste. [#42180](https://github.com/ClickHouse/ClickHouse/pull/42180) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Improvement +* Added new infrastructure for query analysis and planning under `allow_experimental_analyzer` setting. [#31796](https://github.com/ClickHouse/ClickHouse/pull/31796) ([Maksim Kita](https://github.com/kitaisreal)). +* * Support expression `(EXPLAIN SELECT ...)` in a subquery. Queries like `SELECT * FROM (EXPLAIN PIPELINE SELECT col FROM TABLE ORDER BY col)` became valid. [#40630](https://github.com/ClickHouse/ClickHouse/pull/40630) ([Vladimir C](https://github.com/vdimir)). +* Currently changing `async_insert_max_data_size` or `async_insert_busy_timeout_ms` in scope of query makes no sense and this leads to bad user experience. E.g. user wants to insert data rarely and he doesn't have an access to server config to tune default settings. [#40668](https://github.com/ClickHouse/ClickHouse/pull/40668) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Embedded Keeper will always start in the background allowing ClickHouse to start without achieving quorum. [#40991](https://github.com/ClickHouse/ClickHouse/pull/40991) ([Antonio Andelic](https://github.com/antonio2368)). +* Improvements for reading from remote filesystems, made threadpool size for reads/writes configurable. Closes [#41070](https://github.com/ClickHouse/ClickHouse/issues/41070). [#41011](https://github.com/ClickHouse/ClickHouse/pull/41011) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Made reestablishing a new connection more reactive in case of expiration of the previous one. Previously there was a task which spawns every minute by default and thus a table could be in readonly state for about this time. [#41092](https://github.com/ClickHouse/ClickHouse/pull/41092) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Support all combinators combination in WindowTransform/arratReduce*/initializeAggregation/aggregate functions versioning. Previously combinators like `ForEach/Resample/Map` didn't work in these places, using them led to exception like`State function ... inserts results into non-state column`. [#41107](https://github.com/ClickHouse/ClickHouse/pull/41107) ([Kruglov Pavel](https://github.com/Avogar)). +* Now projections can be used with zero copy replication. [#41147](https://github.com/ClickHouse/ClickHouse/pull/41147) ([alesapin](https://github.com/alesapin)). +* - Add function tryDecrypt that returns NULL when decrypt fail (e.g. decrypt with incorrect key) instead of throwing exception. [#41206](https://github.com/ClickHouse/ClickHouse/pull/41206) ([Duc Canh Le](https://github.com/canhld94)). +* Add the `unreserved_space` column to the `system.disks` table to check how much space is not taken by reservations per disk. [#41254](https://github.com/ClickHouse/ClickHouse/pull/41254) ([filimonov](https://github.com/filimonov)). +* Support s3 authorisation headers from ast arguments. [#41261](https://github.com/ClickHouse/ClickHouse/pull/41261) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add setting 'allow_implicit_no_password' that forbids creating a user with no password unless 'IDENTIFIED WITH no_password' is explicitly specified. [#41341](https://github.com/ClickHouse/ClickHouse/pull/41341) ([Nikolay Degterinsky](https://github.com/evillique)). +* keeper-improvement: add support for uploading snapshots to S3. S3 information can be defined inside `keeper_server.s3_snapshot`. [#41342](https://github.com/ClickHouse/ClickHouse/pull/41342) ([Antonio Andelic](https://github.com/antonio2368)). +* Add support for MultiRead in Keeper and internal ZooKeeper client. [#41410](https://github.com/ClickHouse/ClickHouse/pull/41410) ([Antonio Andelic](https://github.com/antonio2368)). +* add a support for decimal type comparing with floating point literal in IN operator. [#41544](https://github.com/ClickHouse/ClickHouse/pull/41544) ([liang.huang](https://github.com/lhuang09287750)). +* Allow readable size values in cache config. [#41688](https://github.com/ClickHouse/ClickHouse/pull/41688) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Check file path for path traversal attacks in errors logger for input formats. [#41694](https://github.com/ClickHouse/ClickHouse/pull/41694) ([Kruglov Pavel](https://github.com/Avogar)). +* ClickHouse could cache stale DNS entries for some period of time (15 seconds by default) until the cache won't be updated asynchronously. During these period ClickHouse can nevertheless try to establish a connection and produce errors. This behaviour is fixed. [#41707](https://github.com/ClickHouse/ClickHouse/pull/41707) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add interactive history search with fzf-like utility (fzf/sk) for `clickhouse-client`/`clickhouse-local` (note you can use `FZF_DEFAULT_OPTS`/`SKIM_DEFAULT_OPTIONS` to additionally configure the behavior). [#41730](https://github.com/ClickHouse/ClickHouse/pull/41730) ([Azat Khuzhin](https://github.com/azat)). +* For client when connecting to a secure server with invalid certificate only allow to proceed with '--accept-certificate' flag. [#41743](https://github.com/ClickHouse/ClickHouse/pull/41743) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Add function "tryBase58Decode()", similar to the existing function "tryBase64Decode()". [#41824](https://github.com/ClickHouse/ClickHouse/pull/41824) ([Robert Schulze](https://github.com/rschu1ze)). +* Improve feedback when replacing partition with different primary key. Fixes [#34798](https://github.com/ClickHouse/ClickHouse/issues/34798). [#41838](https://github.com/ClickHouse/ClickHouse/pull/41838) ([Salvatore](https://github.com/tbsal)). +* Replace back `clickhouse su` command with `sudo -u` in start in order to respect limits in `/etc/security/limits.conf`. [#41847](https://github.com/ClickHouse/ClickHouse/pull/41847) ([Eugene Konkov](https://github.com/ekonkov)). +* Fix parallel parsing: segmentator now checks max_block_size. [#41852](https://github.com/ClickHouse/ClickHouse/pull/41852) ([Vitaly Baranov](https://github.com/vitlibar)). +* Don't report TABLE_IS_DROPPED exception in order to skip table in case is was just dropped. [#41908](https://github.com/ClickHouse/ClickHouse/pull/41908) ([AlfVII](https://github.com/AlfVII)). +* Improve option enable_extended_results_for_datetime_functions to return results of type DateTime64 for functions toStartOfDay, toStartOfHour, toStartOfFifteenMinutes, toStartOfTenMinutes, toStartOfFiveMinutes, toStartOfMinute and timeSlot. [#41910](https://github.com/ClickHouse/ClickHouse/pull/41910) ([Roman Vasin](https://github.com/rvasin)). +* Improve DateTime type inference for text formats. Now it respect setting `date_time_input_format` and doesn't try to infer datetimes from numbers as timestamps. Closes [#41389](https://github.com/ClickHouse/ClickHouse/issues/41389) Closes [#42206](https://github.com/ClickHouse/ClickHouse/issues/42206). [#41912](https://github.com/ClickHouse/ClickHouse/pull/41912) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove confusing warning when inserting with `perform_ttl_move_on_insert`=false. [#41980](https://github.com/ClickHouse/ClickHouse/pull/41980) ([Vitaly Baranov](https://github.com/vitlibar)). +* Allow user to write `countState(*)` similar to `count(*)`. This closes [#9338](https://github.com/ClickHouse/ClickHouse/issues/9338). [#41983](https://github.com/ClickHouse/ClickHouse/pull/41983) ([Amos Bird](https://github.com/amosbird)). +* - Fix rankCorr size overflow. [#42020](https://github.com/ClickHouse/ClickHouse/pull/42020) ([Duc Canh Le](https://github.com/canhld94)). +* Added an option to specify an arbitrary string as an environment name in the Sentry's config for more handy reports. [#42037](https://github.com/ClickHouse/ClickHouse/pull/42037) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Added system table `asynchronous_insert_log `. It contains information about asynchronous inserts (including results of queries in fire-and-forget mode (with `wait_for_async_insert=0`)) for better introspection. [#42040](https://github.com/ClickHouse/ClickHouse/pull/42040) ([Anton Popov](https://github.com/CurtizJ)). +* Fix parsing out-of-range Date from CSV:. [#42044](https://github.com/ClickHouse/ClickHouse/pull/42044) ([Andrey Zvonov](https://github.com/zvonand)). +* parseDataTimeBestEffort support comma between date and time. Closes [#42038](https://github.com/ClickHouse/ClickHouse/issues/42038). [#42049](https://github.com/ClickHouse/ClickHouse/pull/42049) ([flynn](https://github.com/ucasfl)). +* Add support for methods lz4, bz2, snappy in 'Accept-Encoding'. [#42071](https://github.com/ClickHouse/ClickHouse/pull/42071) ([Nikolay Degterinsky](https://github.com/evillique)). +* Various minor fixes for BLAKE3 function. [#42073](https://github.com/ClickHouse/ClickHouse/pull/42073) ([BoloniniD](https://github.com/BoloniniD)). +* Improved stale replica recovery process for `ReplicatedMergeTree`. If lost replica have some parts which absent on a healthy replica, but these parts should appear in future according to replication queue of the healthy replica, then lost replica will keep such parts instead of detaching them. [#42134](https://github.com/ClickHouse/ClickHouse/pull/42134) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Support BACKUP to S3 with as-is path/data structure. [#42232](https://github.com/ClickHouse/ClickHouse/pull/42232) ([Azat Khuzhin](https://github.com/azat)). +* Add a possibility to use Date32 arguments for date_diff function. Fix issue in date_diff function when using DateTime64 arguments with start date before Unix epoch and end date after Unix epoch. [#42308](https://github.com/ClickHouse/ClickHouse/pull/42308) ([Roman Vasin](https://github.com/rvasin)). +* When uploading big parts to minio, 'Complete Multipart Upload' can take a long time. Minio sends heartbeats every 10 seconds (see https://github.com/minio/minio/pull/7198). But clickhouse times out earlier, because the default send/receive timeout is [set](https://github.com/ClickHouse/ClickHouse/blob/cc24fcd6d5dfb67f5f66f5483e986bd1010ad9cf/src/IO/S3/PocoHTTPClient.cpp#L123) to 5 seconds. [#42321](https://github.com/ClickHouse/ClickHouse/pull/42321) ([filimonov](https://github.com/filimonov)). +* Add `S3` as a new type of the destination of backups. [#42333](https://github.com/ClickHouse/ClickHouse/pull/42333) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix rarely invalid cast of aggregate state types with complex types such as Decimal. This fixes [#42408](https://github.com/ClickHouse/ClickHouse/issues/42408). [#42417](https://github.com/ClickHouse/ClickHouse/pull/42417) ([Amos Bird](https://github.com/amosbird)). +* Support skipping cache completely (both download to cache and reading cached data) in case the requested read range exceeds the threshold defined by cache setting `bypass_cache_threashold`, requires to be enabled with `enable_bypass_cache_with_threshold`). [#42418](https://github.com/ClickHouse/ClickHouse/pull/42418) ([Han Shukai](https://github.com/KinderRiven)). +* Merge parts if every part in the range is older than a certain threshold. The threshold can be set by using `min_age_to_force_merge_seconds`. This closes [#35836](https://github.com/ClickHouse/ClickHouse/issues/35836). [#42423](https://github.com/ClickHouse/ClickHouse/pull/42423) ([Antonio Andelic](https://github.com/antonio2368)). +* Enabled CompiledExpressionCache in clickhouse-local. [#42477](https://github.com/ClickHouse/ClickHouse/pull/42477) ([AlfVII](https://github.com/AlfVII)). +* Remove support for the `{database}` macro from the client's prompt. It was displayed incorrectly if the database was unspecified and it was not updated on `USE` statements. This closes [#25891](https://github.com/ClickHouse/ClickHouse/issues/25891). [#42508](https://github.com/ClickHouse/ClickHouse/pull/42508) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* - Improve the time to recover lost keeper connections. [#42541](https://github.com/ClickHouse/ClickHouse/pull/42541) ([Raúl Marín](https://github.com/Algunenano)). +* Allow to use Date32 arguments for dateName function. [#42554](https://github.com/ClickHouse/ClickHouse/pull/42554) ([Roman Vasin](https://github.com/rvasin)). + +#### Bug Fix +* Now filters with NULL literals will be used during index analysis. This closes https://github.com/ClickHouse/ClickHouse/pull/41814 [#34063](https://github.com/ClickHouse/ClickHouse/issues/34063). [#41842](https://github.com/ClickHouse/ClickHouse/pull/41842) ([Amos Bird](https://github.com/amosbird)). +* - Choose correct aggregation method for LowCardinality with BigInt. [#42342](https://github.com/ClickHouse/ClickHouse/pull/42342) ([Duc Canh Le](https://github.com/canhld94)). +* Fix using subqueries in row policy filters. This PR fixes [#32463](https://github.com/ClickHouse/ClickHouse/issues/32463). [#42562](https://github.com/ClickHouse/ClickHouse/pull/42562) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Build/Testing/Packaging Improvement +* Added support of WHERE clause generation to AST Fuzzer and possibility to add or remove ORDER BY and WHERE clause. [#38519](https://github.com/ClickHouse/ClickHouse/pull/38519) ([Ilya Yatsishin](https://github.com/qoega)). +* Aarch64 binaries now require at least ARMv8.2, released in 2016. Most notably, this enables use of ARM LSE, i.e. native atomic operations. Also, CMake build option "NO_ARMV81_OR_HIGHER" has been added to allow compilation of binaries for older ARMv8.0 hardware, e.g. Raspberry Pi 4. [#41610](https://github.com/ClickHouse/ClickHouse/pull/41610) ([Robert Schulze](https://github.com/rschu1ze)). +* After updating runners to 22.04 cgroups stopped to work in privileged mode, here's the issue https://github.com/moby/moby/issues/42275#issuecomment-1115055846. [#41857](https://github.com/ClickHouse/ClickHouse/pull/41857) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Allow building ClickHouse with Musl (small changes after it was already supported but broken). [#41987](https://github.com/ClickHouse/ClickHouse/pull/41987) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* - Add the `$CLICKHOUSE_CRONFILE` file checking to avoid running the `sed` command to get the file not found error. [#42081](https://github.com/ClickHouse/ClickHouse/pull/42081) ([Chun-Sheng, Li](https://github.com/peter279k)). +* Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)). +* Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix power8 support. [#42462](https://github.com/ClickHouse/ClickHouse/pull/42462) ([Boris Kuschel](https://github.com/bkuschel)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Several fixes for DiskWeb. [#41652](https://github.com/ClickHouse/ClickHouse/pull/41652) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixes issue when docker run will fail if "https_port" is not present in config. [#41693](https://github.com/ClickHouse/ClickHouse/pull/41693) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Mutations were not cancelled properly on server shutdown or `SYSTEM STOP MERGES` query and cancellation might take long time, it's fixed. [#41699](https://github.com/ClickHouse/ClickHouse/pull/41699) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix wrong result of queries with `ORDER BY` or `GROUP BY` by columns from prefix of sorting key, wrapped into monotonic functions, with enable "read in order" optimization (settings `optimize_read_in_order` and `optimize_aggregation_in_order`). [#41701](https://github.com/ClickHouse/ClickHouse/pull/41701) ([Anton Popov](https://github.com/CurtizJ)). +* Fix possible crash in `SELECT` from `Merge` table with enabled `optimize_monotonous_functions_in_order_by` setting. Fixes [#41269](https://github.com/ClickHouse/ClickHouse/issues/41269). [#41740](https://github.com/ClickHouse/ClickHouse/pull/41740) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed "Part ... intersects part ..." error that might happen in extremely rare cases if replica was restarted just after detaching some part as broken. [#41741](https://github.com/ClickHouse/ClickHouse/pull/41741) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Don't allow to create or alter merge tree tables with virtual column name _row_exists, which is reserved for lightweight delete. Fixed [#41716](https://github.com/ClickHouse/ClickHouse/issues/41716). [#41763](https://github.com/ClickHouse/ClickHouse/pull/41763) ([Jianmei Zhang](https://github.com/zhangjmruc)). +* Fix a bug that CORS headers are missing in some HTTP responses. [#41792](https://github.com/ClickHouse/ClickHouse/pull/41792) ([Frank Chen](https://github.com/FrankChen021)). +* 22.9 might fail to startup `ReplicatedMergeTree` table if that table was created by 20.3 or older version and was never altered, it's fixed. Fixes [#41742](https://github.com/ClickHouse/ClickHouse/issues/41742). [#41796](https://github.com/ClickHouse/ClickHouse/pull/41796) ([Alexander Tokmakov](https://github.com/tavplubix)). +* When the batch sending fails for some reason, it cannot be automatically recovered, and if it is not processed in time, it will lead to accumulation, and the printed error message will become longer and longer, which will cause the http thread to block. [#41813](https://github.com/ClickHouse/ClickHouse/pull/41813) ([zhongyuankai](https://github.com/zhongyuankai)). +* Fix compact parts with compressed marks setting. Fixes [#41783](https://github.com/ClickHouse/ClickHouse/issues/41783) and [#41746](https://github.com/ClickHouse/ClickHouse/issues/41746). [#41823](https://github.com/ClickHouse/ClickHouse/pull/41823) ([alesapin](https://github.com/alesapin)). +* Old versions of Replicated database doesn't have a special marker in [Zoo]Keeper. We need to check only whether the node contains come obscure data instead of special mark. [#41875](https://github.com/ClickHouse/ClickHouse/pull/41875) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix possible exception in fs cache. [#41884](https://github.com/ClickHouse/ClickHouse/pull/41884) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix use_environment_credentials for s3 table function. [#41970](https://github.com/ClickHouse/ClickHouse/pull/41970) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed "Directory already exists and is not empty" error on detaching broken part that might prevent `ReplicatedMergeTree` table from starting replication. Fixes [#40957](https://github.com/ClickHouse/ClickHouse/issues/40957). [#41981](https://github.com/ClickHouse/ClickHouse/pull/41981) ([Alexander Tokmakov](https://github.com/tavplubix)). +* toDateTime64() now returns the same output with negative integer and float arguments. [#42025](https://github.com/ClickHouse/ClickHouse/pull/42025) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix write into AzureBlobStorage. Partially closes [#41754](https://github.com/ClickHouse/ClickHouse/issues/41754). [#42034](https://github.com/ClickHouse/ClickHouse/pull/42034) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix the bzip2 decoding issue for specific bzip2 files. [#42046](https://github.com/ClickHouse/ClickHouse/pull/42046) ([Nikolay Degterinsky](https://github.com/evillique)). +* - Fix SQL function "toLastDayOfMonth()" with setting "enable_extended_results_for_datetime_functions = 1" at the beginning of the extended range (January 1900). - Fix SQL function "toRelativeWeekNum()" with setting "enable_extended_results_for_datetime_functions = 1" at the end of extended range (December 2299). - Improve the performance of for SQL functions "toISOYear()", "toFirstDayNumOfISOYearIndex()" and "toYearWeekOfNewyearMode()" by avoiding unnecessary index arithmetics. [#42084](https://github.com/ClickHouse/ClickHouse/pull/42084) ([Roman Vasin](https://github.com/rvasin)). +* The maximum size of fetches for each table accidentally was set to 8 while the pool size could be bigger. Now the maximum size of fetches for table is equal to the pool size. [#42090](https://github.com/ClickHouse/ClickHouse/pull/42090) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* A table might be shut down and a dictionary might be detached before checking if can be dropped without breaking dependencies between table, it's fixed. Fixes [#41982](https://github.com/ClickHouse/ClickHouse/issues/41982). [#42106](https://github.com/ClickHouse/ClickHouse/pull/42106) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix bad inefficiency of `remote_filesystem_read_method=read` with filesystem cache. Closes [#42125](https://github.com/ClickHouse/ClickHouse/issues/42125). [#42129](https://github.com/ClickHouse/ClickHouse/pull/42129) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible timeout exception for distributed queries with use_hedged_requests=0. [#42130](https://github.com/ClickHouse/ClickHouse/pull/42130) ([Azat Khuzhin](https://github.com/azat)). +* Fixed a minor bug inside function `runningDifference` in case of using it with `Date32` type. Previously `Date` was used and it may cause some logical errors like `Bad cast from type DB::ColumnVector to DB::ColumnVector'`. [#42143](https://github.com/ClickHouse/ClickHouse/pull/42143) ([Alfred Xu](https://github.com/sperlingxx)). +* Fix reusing of files > 4GB from base backup. [#42146](https://github.com/ClickHouse/ClickHouse/pull/42146) ([Azat Khuzhin](https://github.com/azat)). +* DISTINCT in order fails with LOGICAL_ERROR if first column in sorting key contains function. [#42186](https://github.com/ClickHouse/ClickHouse/pull/42186) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix a bug with projections and the `aggregate_functions_null_for_empty` setting. This bug is very rare and appears only if you enable the `aggregate_functions_null_for_empty` setting in the server's config. This closes [#41647](https://github.com/ClickHouse/ClickHouse/issues/41647). [#42198](https://github.com/ClickHouse/ClickHouse/pull/42198) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* - Fix read from buffer with read in order desc. [#42236](https://github.com/ClickHouse/ClickHouse/pull/42236) ([Duc Canh Le](https://github.com/canhld94)). +* Fix a bug which prevents ClickHouse to start when background_pool_size setting is set on default profile but background_merges_mutations_concurrency_ratio is not. [#42315](https://github.com/ClickHouse/ClickHouse/pull/42315) ([nvartolomei](https://github.com/nvartolomei)). +* `ALTER UPDATE` of attached part (with columns different from table schema) could create an invalid `columns.txt` metadata on disk. Reading from such part could fail with errors or return invalid data. Fixes [#42161](https://github.com/ClickHouse/ClickHouse/issues/42161). [#42319](https://github.com/ClickHouse/ClickHouse/pull/42319) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Setting `additional_table_filters` were not applied to `Distributed` storage. Fixes [#41692](https://github.com/ClickHouse/ClickHouse/issues/41692). [#42322](https://github.com/ClickHouse/ClickHouse/pull/42322) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix a data race in query finish/cancel. This closes [#42346](https://github.com/ClickHouse/ClickHouse/issues/42346). [#42362](https://github.com/ClickHouse/ClickHouse/pull/42362) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* This reverts [#40217](https://github.com/ClickHouse/ClickHouse/issues/40217) which introduced a regression in date/time functions. [#42367](https://github.com/ClickHouse/ClickHouse/pull/42367) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix assert cast in join on falsy condition, Close [#42380](https://github.com/ClickHouse/ClickHouse/issues/42380). [#42407](https://github.com/ClickHouse/ClickHouse/pull/42407) ([Vladimir C](https://github.com/vdimir)). +* Fix buffer overflow in the processing of Decimal data types. This closes [#42451](https://github.com/ClickHouse/ClickHouse/issues/42451). [#42465](https://github.com/ClickHouse/ClickHouse/pull/42465) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* `AggregateFunctionQuantile` now correctly works with UInt128 columns. Previously, the quantile state interpreted `UInt128` columns as `Int128` which could have led to incorrect results. [#42473](https://github.com/ClickHouse/ClickHouse/pull/42473) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix bad_assert during INSERT into Annoy indexes over non-Float32 columns. [#42485](https://github.com/ClickHouse/ClickHouse/pull/42485) ([Robert Schulze](https://github.com/rschu1ze)). +* This closes [#42453](https://github.com/ClickHouse/ClickHouse/issues/42453). [#42573](https://github.com/ClickHouse/ClickHouse/pull/42573) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix function `arrayElement` with type `Map` with `Nullable` values and `Nullable` index. [#42623](https://github.com/ClickHouse/ClickHouse/pull/42623) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Fix unexpected table loading error when partition key contains alias function names during server upgrade. [#36379](https://github.com/ClickHouse/ClickHouse/pull/36379) ([Amos Bird](https://github.com/amosbird)). + +#### Build Improvement + +* Fixed SipHash Endian issue for s390x platform. [#41372](https://github.com/ClickHouse/ClickHouse/pull/41372) ([Harry Lee](https://github.com/HarryLeeIBM)). +* Enable lib base64 for ppc64le platform. [#41974](https://github.com/ClickHouse/ClickHouse/pull/41974) ([Suzy Wang](https://github.com/SuzyWangIBMer)). +* Fixed Endian issue in T64 compression codec on s390x. [#42314](https://github.com/ClickHouse/ClickHouse/pull/42314) ([Harry Lee](https://github.com/HarryLeeIBM)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Disable parallel s3 multipart upload for part moves."'. [#41681](https://github.com/ClickHouse/ClickHouse/pull/41681) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Attempt to fix abort from parallel parsing"'. [#42545](https://github.com/ClickHouse/ClickHouse/pull/42545) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* NO CL ENTRY: 'Revert "Low cardinality cases moved to the function for its corresponding type"'. [#42633](https://github.com/ClickHouse/ClickHouse/pull/42633) ([Anton Popov](https://github.com/CurtizJ)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Test for ignore function in PARTITION KEY [#39875](https://github.com/ClickHouse/ClickHouse/pull/39875) ([UnamedRus](https://github.com/UnamedRus)). +* Add fuzzer for table definitions [#40096](https://github.com/ClickHouse/ClickHouse/pull/40096) ([Anton Popov](https://github.com/CurtizJ)). +* Add missing tests for legacy geobase [#40684](https://github.com/ClickHouse/ClickHouse/pull/40684) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove obsolete comment from the config.xml [#41518](https://github.com/ClickHouse/ClickHouse/pull/41518) ([filimonov](https://github.com/filimonov)). +* Resurrect parallel distributed insert select with s3Cluster [#41535](https://github.com/ClickHouse/ClickHouse/pull/41535) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Update runners to a recent version to install on 22.04 [#41556](https://github.com/ClickHouse/ClickHouse/pull/41556) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Refactor wiping sensitive information from logs. [#41562](https://github.com/ClickHouse/ClickHouse/pull/41562) ([Vitaly Baranov](https://github.com/vitlibar)). +* Better S3 logs [#41587](https://github.com/ClickHouse/ClickHouse/pull/41587) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix typos in JSON formats after [#40910](https://github.com/ClickHouse/ClickHouse/issues/40910) [#41614](https://github.com/ClickHouse/ClickHouse/pull/41614) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix drop for KeeperMap [#41616](https://github.com/ClickHouse/ClickHouse/pull/41616) ([Antonio Andelic](https://github.com/antonio2368)). +* increase default max_suspicious_broken_parts to 100 [#41619](https://github.com/ClickHouse/ClickHouse/pull/41619) ([Denny Crane](https://github.com/den-crane)). +* Release AWS SDK log level + replace one exception [#41649](https://github.com/ClickHouse/ClickHouse/pull/41649) ([alesapin](https://github.com/alesapin)). +* Fix a destruction order for views ThreadStatus [#41650](https://github.com/ClickHouse/ClickHouse/pull/41650) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add very explicit logging on disk choice for fetch [#41653](https://github.com/ClickHouse/ClickHouse/pull/41653) ([alesapin](https://github.com/alesapin)). +* Fix race between ~BackgroundSchedulePool and ~DNSCacheUpdater [#41654](https://github.com/ClickHouse/ClickHouse/pull/41654) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add changelog for 22.9 [#41668](https://github.com/ClickHouse/ClickHouse/pull/41668) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version after release [#41670](https://github.com/ClickHouse/ClickHouse/pull/41670) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix error message [#41680](https://github.com/ClickHouse/ClickHouse/pull/41680) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add test for setting output_format_json_validate_utf8 [#41691](https://github.com/ClickHouse/ClickHouse/pull/41691) ([Kruglov Pavel](https://github.com/Avogar)). +* Resolve findings from clang-tidy [#41702](https://github.com/ClickHouse/ClickHouse/pull/41702) ([ltrk2](https://github.com/ltrk2)). +* Ignore Keeper errors from ReplicatedMergeTreeAttachThread in stress tests [#41717](https://github.com/ClickHouse/ClickHouse/pull/41717) ([Antonio Andelic](https://github.com/antonio2368)). +* Collect logs in Stress test using clickhouse-local [#41721](https://github.com/ClickHouse/ClickHouse/pull/41721) ([Antonio Andelic](https://github.com/antonio2368)). +* Disable flaky `test_merge_tree_azure_blob_storage` [#41722](https://github.com/ClickHouse/ClickHouse/pull/41722) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update version_date.tsv and changelogs after v22.9.2.7-stable [#41724](https://github.com/ClickHouse/ClickHouse/pull/41724) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Fix part removal retries [#41728](https://github.com/ClickHouse/ClickHouse/pull/41728) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Try fix azure tests [#41731](https://github.com/ClickHouse/ClickHouse/pull/41731) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix test build [#41732](https://github.com/ClickHouse/ClickHouse/pull/41732) ([Robert Schulze](https://github.com/rschu1ze)). +* Change logging levels in cache [#41733](https://github.com/ClickHouse/ClickHouse/pull/41733) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Revert of "Revert the revert of "ColumnVector: optimize filter with AVX512 VBMI2 compress store" [#40033](https://github.com/ClickHouse/ClickHouse/issues/40033)" [#41752](https://github.com/ClickHouse/ClickHouse/pull/41752) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix SET query parameters formatting [#41755](https://github.com/ClickHouse/ClickHouse/pull/41755) ([Nikolay Degterinsky](https://github.com/evillique)). +* Support to run testcases on macOS [#41760](https://github.com/ClickHouse/ClickHouse/pull/41760) ([Frank Chen](https://github.com/FrankChen021)). +* Bump LLVM from 12 to 13 [#41762](https://github.com/ClickHouse/ClickHouse/pull/41762) ([Robert Schulze](https://github.com/rschu1ze)). +* ColumnVector: re-enable AVX512_VBMI/AVX512_VBMI2 optimized filter and index [#41765](https://github.com/ClickHouse/ClickHouse/pull/41765) ([Guo Wangyang](https://github.com/guowangy)). +* Update 02354_annoy.sql [#41767](https://github.com/ClickHouse/ClickHouse/pull/41767) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix the typo preventing building latest images [#41769](https://github.com/ClickHouse/ClickHouse/pull/41769) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Make automatic download script choose between ARMv8.0 or ARMv8.2 builds [#41775](https://github.com/ClickHouse/ClickHouse/pull/41775) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix tests for docker-ci [#41777](https://github.com/ClickHouse/ClickHouse/pull/41777) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Possible fix for KeeperMap drop [#41784](https://github.com/ClickHouse/ClickHouse/pull/41784) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix drop of completely dropped table [#41789](https://github.com/ClickHouse/ClickHouse/pull/41789) ([alesapin](https://github.com/alesapin)). +* Log git hash during startup [#41790](https://github.com/ClickHouse/ClickHouse/pull/41790) ([Robert Schulze](https://github.com/rschu1ze)). +* Revert "ColumnVector: optimize UInt8 index with AVX512VBMI ([#41247](https://github.com/ClickHouse/ClickHouse/issues/41247))" [#41797](https://github.com/ClickHouse/ClickHouse/pull/41797) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Small fix in dashboard [#41798](https://github.com/ClickHouse/ClickHouse/pull/41798) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Keep the most important log in stress tests [#41821](https://github.com/ClickHouse/ClickHouse/pull/41821) ([alesapin](https://github.com/alesapin)). +* Use copy for some operations instead of hardlinks [#41832](https://github.com/ClickHouse/ClickHouse/pull/41832) ([alesapin](https://github.com/alesapin)). +* Remove unused variable in registerStorageMergeTree.cpp [#41839](https://github.com/ClickHouse/ClickHouse/pull/41839) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix Jepsen [#41845](https://github.com/ClickHouse/ClickHouse/pull/41845) ([Antonio Andelic](https://github.com/antonio2368)). +* Increase `request_timeout_ms` for s3 tests in CI [#41853](https://github.com/ClickHouse/ClickHouse/pull/41853) ([Kseniia Sumarokova](https://github.com/kssenii)). +* tests: fix debug symbols (and possible crashes) for backward compatiblity check [#41854](https://github.com/ClickHouse/ClickHouse/pull/41854) ([Azat Khuzhin](https://github.com/azat)). +* Remove two redundant lines [#41856](https://github.com/ClickHouse/ClickHouse/pull/41856) ([alesapin](https://github.com/alesapin)). +* Infer Object type only when allow_experimental_object_type is enabled [#41858](https://github.com/ClickHouse/ClickHouse/pull/41858) ([Kruglov Pavel](https://github.com/Avogar)). +* Add default UNION/EXCEPT/INTERSECT to the echo query text [#41862](https://github.com/ClickHouse/ClickHouse/pull/41862) ([Nikolay Degterinsky](https://github.com/evillique)). +* Consolidate CMake-generated config headers [#41873](https://github.com/ClickHouse/ClickHouse/pull/41873) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix 02267_file_globs_schema_inference.sql flakiness [#41877](https://github.com/ClickHouse/ClickHouse/pull/41877) ([Kruglov Pavel](https://github.com/Avogar)). +* Docs: Remove obsolete modelEvaluate() mention [#41878](https://github.com/ClickHouse/ClickHouse/pull/41878) ([Robert Schulze](https://github.com/rschu1ze)). +* Better exception message for duplicate column names in schema inference [#41885](https://github.com/ClickHouse/ClickHouse/pull/41885) ([Kruglov Pavel](https://github.com/Avogar)). +* Docs: Reference external papers as DOIs [#41886](https://github.com/ClickHouse/ClickHouse/pull/41886) ([Robert Schulze](https://github.com/rschu1ze)). +* Make LDAPR a prerequisite for downloading the ARMv8.2 build [#41897](https://github.com/ClickHouse/ClickHouse/pull/41897) ([Robert Schulze](https://github.com/rschu1ze)). +* Another sync replicas in test_recovery_replica [#41898](https://github.com/ClickHouse/ClickHouse/pull/41898) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* remove unused code [#41921](https://github.com/ClickHouse/ClickHouse/pull/41921) ([flynn](https://github.com/ucasfl)). +* Move all queries for MV creation to the end of queue during recovering [#41932](https://github.com/ClickHouse/ClickHouse/pull/41932) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix broken test_disks_app_func [#41933](https://github.com/ClickHouse/ClickHouse/pull/41933) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Temporarily disable ThreadFuzzer with TSan [#41943](https://github.com/ClickHouse/ClickHouse/pull/41943) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Enable some disabled S3 tests [#41945](https://github.com/ClickHouse/ClickHouse/pull/41945) ([alesapin](https://github.com/alesapin)). +* QOL log improvements [#41947](https://github.com/ClickHouse/ClickHouse/pull/41947) ([Raúl Marín](https://github.com/Algunenano)). +* Fix non-deterministic test results [#41948](https://github.com/ClickHouse/ClickHouse/pull/41948) ([Robert Schulze](https://github.com/rschu1ze)). +* Earlier throw exception in PullingAsyncPipelineExecutor. [#41949](https://github.com/ClickHouse/ClickHouse/pull/41949) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix linker error [#41950](https://github.com/ClickHouse/ClickHouse/pull/41950) ([ltrk2](https://github.com/ltrk2)). +* Bump LLVM from 13 to 14 [#41951](https://github.com/ClickHouse/ClickHouse/pull/41951) ([Robert Schulze](https://github.com/rschu1ze)). +* Update version_date.tsv and changelogs after v22.3.13.80-lts [#41953](https://github.com/ClickHouse/ClickHouse/pull/41953) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v22.7.6.74-stable [#41954](https://github.com/ClickHouse/ClickHouse/pull/41954) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v22.8.6.71-lts [#41955](https://github.com/ClickHouse/ClickHouse/pull/41955) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v22.9.3.18-stable [#41956](https://github.com/ClickHouse/ClickHouse/pull/41956) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Add a warning message to release.py script, require release type [#41975](https://github.com/ClickHouse/ClickHouse/pull/41975) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Rename max_temp_data_on_disk -> max_temporary_data_on_disk [#41984](https://github.com/ClickHouse/ClickHouse/pull/41984) ([Vladimir C](https://github.com/vdimir)). +* Add more checkStackSize calls [#41991](https://github.com/ClickHouse/ClickHouse/pull/41991) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix test 02403_big_http_chunk_size [#41996](https://github.com/ClickHouse/ClickHouse/pull/41996) ([Vitaly Baranov](https://github.com/vitlibar)). +* More sane behavior of part number thresholds override in query level settings [#42001](https://github.com/ClickHouse/ClickHouse/pull/42001) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#42004](https://github.com/ClickHouse/ClickHouse/pull/42004) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Refactoring: Uninline some error handling methods [#42010](https://github.com/ClickHouse/ClickHouse/pull/42010) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix warning that ENABLE_REPLXX is unused [#42013](https://github.com/ClickHouse/ClickHouse/pull/42013) ([Robert Schulze](https://github.com/rschu1ze)). +* Drop leftovers of libexecinfo [#42014](https://github.com/ClickHouse/ClickHouse/pull/42014) ([Robert Schulze](https://github.com/rschu1ze)). +* More detailed exception message [#42022](https://github.com/ClickHouse/ClickHouse/pull/42022) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Build against an LLVM version which has clang[-extra-tools], lldb and lld removed [#42023](https://github.com/ClickHouse/ClickHouse/pull/42023) ([Robert Schulze](https://github.com/rschu1ze)). +* Add log message and lower the retry timeout in MergeTreeRestartingThread [#42026](https://github.com/ClickHouse/ClickHouse/pull/42026) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Update amqp-cpp [#42031](https://github.com/ClickHouse/ClickHouse/pull/42031) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix No such key during table drop [#42036](https://github.com/ClickHouse/ClickHouse/pull/42036) ([alesapin](https://github.com/alesapin)). +* Temporarily disable too aggressive tests [#42050](https://github.com/ClickHouse/ClickHouse/pull/42050) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix style check [#42055](https://github.com/ClickHouse/ClickHouse/pull/42055) ([Anton Popov](https://github.com/CurtizJ)). +* Function name normalization fix functions header [#42063](https://github.com/ClickHouse/ClickHouse/pull/42063) ([Maksim Kita](https://github.com/kitaisreal)). +* remove unused virtual keyword [#42065](https://github.com/ClickHouse/ClickHouse/pull/42065) ([flynn](https://github.com/ucasfl)). +* Fix crash in `SummingMergeTree` with `LowCardinality` [#42066](https://github.com/ClickHouse/ClickHouse/pull/42066) ([Anton Popov](https://github.com/CurtizJ)). +* Fix drop of completely dropped table [#42067](https://github.com/ClickHouse/ClickHouse/pull/42067) ([alesapin](https://github.com/alesapin)). +* Fix assertion in bloom filter index [#42072](https://github.com/ClickHouse/ClickHouse/pull/42072) ([Anton Popov](https://github.com/CurtizJ)). +* Ignore core.autocrlf for tests references [#42076](https://github.com/ClickHouse/ClickHouse/pull/42076) ([Azat Khuzhin](https://github.com/azat)). +* Fix progress for INSERT SELECT [#42078](https://github.com/ClickHouse/ClickHouse/pull/42078) ([Azat Khuzhin](https://github.com/azat)). +* Avoid adding extra new line after using fuzzy history search [#42080](https://github.com/ClickHouse/ClickHouse/pull/42080) ([Azat Khuzhin](https://github.com/azat)). +* Add `at` to runner AMI, bump gh runner version [#42082](https://github.com/ClickHouse/ClickHouse/pull/42082) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Use send_metadata instead of send_object_metadata [#42085](https://github.com/ClickHouse/ClickHouse/pull/42085) ([Elena Torró](https://github.com/elenatorro)). +* Docs: Preparations to remove misc statements page [#42086](https://github.com/ClickHouse/ClickHouse/pull/42086) ([Robert Schulze](https://github.com/rschu1ze)). +* Followup for TemporaryDataOnDisk [#42103](https://github.com/ClickHouse/ClickHouse/pull/42103) ([Vladimir C](https://github.com/vdimir)). +* Disable 02122_join_group_by_timeout for debug [#42104](https://github.com/ClickHouse/ClickHouse/pull/42104) ([Vladimir C](https://github.com/vdimir)). +* Update version_date.tsv and changelogs after v22.6.9.11-stable [#42114](https://github.com/ClickHouse/ClickHouse/pull/42114) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* JIT compilation migration to LLVM 15 [#42123](https://github.com/ClickHouse/ClickHouse/pull/42123) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix build without TSA [#42128](https://github.com/ClickHouse/ClickHouse/pull/42128) ([Raúl Marín](https://github.com/Algunenano)). +* Update codespell-ignore-words.list [#42132](https://github.com/ClickHouse/ClickHouse/pull/42132) ([Dan Roscigno](https://github.com/DanRoscigno)). +* Add null pointer checks [#42135](https://github.com/ClickHouse/ClickHouse/pull/42135) ([ltrk2](https://github.com/ltrk2)). +* Revert [#27787](https://github.com/ClickHouse/ClickHouse/issues/27787) [#42136](https://github.com/ClickHouse/ClickHouse/pull/42136) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Follow up for [#42129](https://github.com/ClickHouse/ClickHouse/issues/42129) [#42144](https://github.com/ClickHouse/ClickHouse/pull/42144) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix checking parent for old-format parts [#42147](https://github.com/ClickHouse/ClickHouse/pull/42147) ([alesapin](https://github.com/alesapin)). +* Revert "Resurrect parallel distributed insert select with s3Cluster [#42150](https://github.com/ClickHouse/ClickHouse/pull/42150) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Docs: Add "TABLE" to CHECK/DESCRIBE statements in sidebar [#42152](https://github.com/ClickHouse/ClickHouse/pull/42152) ([Robert Schulze](https://github.com/rschu1ze)). +* Add logging during merge tree startup [#42163](https://github.com/ClickHouse/ClickHouse/pull/42163) ([alesapin](https://github.com/alesapin)). +* Abort instead of `__builtin_unreachable` in debug builds [#42168](https://github.com/ClickHouse/ClickHouse/pull/42168) ([Alexander Tokmakov](https://github.com/tavplubix)). +* [RFC] Enable -Wshorten-64-to-32 [#42190](https://github.com/ClickHouse/ClickHouse/pull/42190) ([Azat Khuzhin](https://github.com/azat)). +* Fix dialect setting description [#42196](https://github.com/ClickHouse/ClickHouse/pull/42196) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Add a test for #658 [#42197](https://github.com/ClickHouse/ClickHouse/pull/42197) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* use alias for MergeMutateSelectedEntry share ptr [#42211](https://github.com/ClickHouse/ClickHouse/pull/42211) ([Tian Xinhui](https://github.com/xinhuitian)). +* Fix LLVM build [#42216](https://github.com/ClickHouse/ClickHouse/pull/42216) ([Raúl Marín](https://github.com/Algunenano)). +* Exclude comments from style-check defined extern [#42217](https://github.com/ClickHouse/ClickHouse/pull/42217) ([Vladimir C](https://github.com/vdimir)). +* Update safeExit.cpp [#42220](https://github.com/ClickHouse/ClickHouse/pull/42220) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable concurrent parts removal [#42222](https://github.com/ClickHouse/ClickHouse/pull/42222) ([alesapin](https://github.com/alesapin)). +* Fail fast on empty URL in HDFS [#42223](https://github.com/ClickHouse/ClickHouse/pull/42223) ([Ilya Yatsishin](https://github.com/qoega)). +* Add a test for [#2389](https://github.com/ClickHouse/ClickHouse/issues/2389) [#42235](https://github.com/ClickHouse/ClickHouse/pull/42235) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use MultiRead where possible [#42243](https://github.com/ClickHouse/ClickHouse/pull/42243) ([Antonio Andelic](https://github.com/antonio2368)). +* Minor cleanups of LLVM integration [#42249](https://github.com/ClickHouse/ClickHouse/pull/42249) ([Robert Schulze](https://github.com/rschu1ze)). +* remove useless code [#42253](https://github.com/ClickHouse/ClickHouse/pull/42253) ([flynn](https://github.com/ucasfl)). +* Early return of corner cases in selectPartsToMutate function [#42254](https://github.com/ClickHouse/ClickHouse/pull/42254) ([Tian Xinhui](https://github.com/xinhuitian)). +* Refactor the implementation of user-defined functions [#42263](https://github.com/ClickHouse/ClickHouse/pull/42263) ([Vitaly Baranov](https://github.com/vitlibar)). +* assert unused value in test_replicated_merge_tree_compatibility [#42266](https://github.com/ClickHouse/ClickHouse/pull/42266) ([nvartolomei](https://github.com/nvartolomei)). +* Fix Date Interval add/minus over DataTypeDate32 [#42279](https://github.com/ClickHouse/ClickHouse/pull/42279) ([Alfred Xu](https://github.com/sperlingxx)). +* Fix log-level in `clickhouse-disks` [#42302](https://github.com/ClickHouse/ClickHouse/pull/42302) ([Nikolay Degterinsky](https://github.com/evillique)). +* Remove forgotten debug logging [#42313](https://github.com/ClickHouse/ClickHouse/pull/42313) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix another trash in zero-copy replication [#42317](https://github.com/ClickHouse/ClickHouse/pull/42317) ([alesapin](https://github.com/alesapin)). +* go update for diagnostics tool [#42325](https://github.com/ClickHouse/ClickHouse/pull/42325) ([Dale McDiarmid](https://github.com/gingerwizard)). +* Better logging for asynchronous inserts [#42345](https://github.com/ClickHouse/ClickHouse/pull/42345) ([Anton Popov](https://github.com/CurtizJ)). +* Use nfpm packager for archlinux packages [#42349](https://github.com/ClickHouse/ClickHouse/pull/42349) ([Azat Khuzhin](https://github.com/azat)). +* Bump llvm/clang to 15.0.2 [#42351](https://github.com/ClickHouse/ClickHouse/pull/42351) ([Azat Khuzhin](https://github.com/azat)). +* Make getResource() independent from the order of the sections [#42353](https://github.com/ClickHouse/ClickHouse/pull/42353) ([Azat Khuzhin](https://github.com/azat)). +* Smaller threshold for multipart upload part size increase [#42392](https://github.com/ClickHouse/ClickHouse/pull/42392) ([alesapin](https://github.com/alesapin)). +* Better error message for unsupported delimiters in custom formats [#42406](https://github.com/ClickHouse/ClickHouse/pull/42406) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix formatting of `ALTER FREEZE` [#42409](https://github.com/ClickHouse/ClickHouse/pull/42409) ([Anton Popov](https://github.com/CurtizJ)). +* Replace table name in ast fuzzer more often [#42413](https://github.com/ClickHouse/ClickHouse/pull/42413) ([Anton Popov](https://github.com/CurtizJ)). +* Add *-15 tools to cmake.tools for GCC build [#42430](https://github.com/ClickHouse/ClickHouse/pull/42430) ([Ilya Yatsishin](https://github.com/qoega)). +* Deactivate tasks in ReplicatedMergeTree until startup [#42441](https://github.com/ClickHouse/ClickHouse/pull/42441) ([alesapin](https://github.com/alesapin)). +* Revert "Revert [#27787](https://github.com/ClickHouse/ClickHouse/issues/27787)" [#42442](https://github.com/ClickHouse/ClickHouse/pull/42442) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update woboq_codebrowser location [#42448](https://github.com/ClickHouse/ClickHouse/pull/42448) ([Azat Khuzhin](https://github.com/azat)). +* add mdx and jsx to list of doc files [#42454](https://github.com/ClickHouse/ClickHouse/pull/42454) ([Dan Roscigno](https://github.com/DanRoscigno)). +* Remove code browser docs [#42455](https://github.com/ClickHouse/ClickHouse/pull/42455) ([Dan Roscigno](https://github.com/DanRoscigno)). +* Better workaround for emitting .debug_aranges section [#42457](https://github.com/ClickHouse/ClickHouse/pull/42457) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky test [#42459](https://github.com/ClickHouse/ClickHouse/pull/42459) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in Julian Day functions [#42464](https://github.com/ClickHouse/ClickHouse/pull/42464) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* rename filesystem_query_cache [#42472](https://github.com/ClickHouse/ClickHouse/pull/42472) ([Han Shukai](https://github.com/KinderRiven)). +* Add convenience typedefs for Date/Date32/DateTime/DateTime64 columns [#42476](https://github.com/ClickHouse/ClickHouse/pull/42476) ([Robert Schulze](https://github.com/rschu1ze)). +* Add error "Destination table is myself" to exception list in BC check [#42479](https://github.com/ClickHouse/ClickHouse/pull/42479) ([Kruglov Pavel](https://github.com/Avogar)). +* Get current clickhouse version without sending query in BC check [#42483](https://github.com/ClickHouse/ClickHouse/pull/42483) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix logical error from welchTTest [#42487](https://github.com/ClickHouse/ClickHouse/pull/42487) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Attempt to fix abort from parallel parsing [#42496](https://github.com/ClickHouse/ClickHouse/pull/42496) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Increase threshold for using physical cores for `max_threads` [#42503](https://github.com/ClickHouse/ClickHouse/pull/42503) ([Nikita Taranov](https://github.com/nickitat)). +* Add a test for [#16827](https://github.com/ClickHouse/ClickHouse/issues/16827) [#42511](https://github.com/ClickHouse/ClickHouse/pull/42511) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#13653](https://github.com/ClickHouse/ClickHouse/issues/13653) [#42512](https://github.com/ClickHouse/ClickHouse/pull/42512) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix aliases [#42514](https://github.com/ClickHouse/ClickHouse/pull/42514) ([Nikolay Degterinsky](https://github.com/evillique)). +* tests: fix 00705_drop_create_merge_tree flakiness [#42522](https://github.com/ClickHouse/ClickHouse/pull/42522) ([Azat Khuzhin](https://github.com/azat)). +* Fix sanitizer reports in integration tests [#42529](https://github.com/ClickHouse/ClickHouse/pull/42529) ([Azat Khuzhin](https://github.com/azat)). +* Fix `KeeperTCPHandler` data race [#42532](https://github.com/ClickHouse/ClickHouse/pull/42532) ([Antonio Andelic](https://github.com/antonio2368)). +* Disable `test_storage_nats`, because it's permanently broken [#42535](https://github.com/ClickHouse/ClickHouse/pull/42535) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Better logs in clickhouse-disks [#42549](https://github.com/ClickHouse/ClickHouse/pull/42549) ([Nikolay Degterinsky](https://github.com/evillique)). +* add lib_fuzzer and lib_fuzzer_no_main to llvm-project build [#42550](https://github.com/ClickHouse/ClickHouse/pull/42550) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Some polishing: replicated merge tree [#42560](https://github.com/ClickHouse/ClickHouse/pull/42560) ([Igor Nikonov](https://github.com/devcrafter)). +* Temporarily disable flaky `test_replicated_merge_tree_hdfs_zero_copy` [#42563](https://github.com/ClickHouse/ClickHouse/pull/42563) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Adapt internal data structures to 512-bit era [#42564](https://github.com/ClickHouse/ClickHouse/pull/42564) ([Nikita Taranov](https://github.com/nickitat)). +* Fix strange code in date monotonicity [#42574](https://github.com/ClickHouse/ClickHouse/pull/42574) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Clear thread::id when ThreadFromGlobalPool exits. [#42577](https://github.com/ClickHouse/ClickHouse/pull/42577) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* ci/stress: fix memory limits overrides for hung check [#42585](https://github.com/ClickHouse/ClickHouse/pull/42585) ([Azat Khuzhin](https://github.com/azat)). +* tests: avoid model overlap for obfuscator [#42586](https://github.com/ClickHouse/ClickHouse/pull/42586) ([Azat Khuzhin](https://github.com/azat)). +* Fix possible segfault in expression parser [#42598](https://github.com/ClickHouse/ClickHouse/pull/42598) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix incorrect trace log line on dict reload [#42609](https://github.com/ClickHouse/ClickHouse/pull/42609) ([filimonov](https://github.com/filimonov)). +* Fix flaky 02458_datediff_date32 test [#42611](https://github.com/ClickHouse/ClickHouse/pull/42611) ([Roman Vasin](https://github.com/rvasin)). +* Revert revert 41268 disable s3 parallel write for part moves to disk s3 [#42617](https://github.com/ClickHouse/ClickHouse/pull/42617) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Try to fix data race on zookeeper vs DDLWorker at server shutdown. [#42620](https://github.com/ClickHouse/ClickHouse/pull/42620) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add a template for installation issues [#42626](https://github.com/ClickHouse/ClickHouse/pull/42626) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix typo in cmake code related to fuzzing [#42627](https://github.com/ClickHouse/ClickHouse/pull/42627) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix build [#42635](https://github.com/ClickHouse/ClickHouse/pull/42635) ([Anton Popov](https://github.com/CurtizJ)). +* Add .rgignore for test data [#42639](https://github.com/ClickHouse/ClickHouse/pull/42639) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 65ec5ddec01..7bbd8547506 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v22.10.1.1875-stable 2022-10-26 v22.9.3.18-stable 2022-09-30 v22.9.2.7-stable 2022-09-23 v22.9.1.2603-stable 2022-09-22 From a8f3b39dcc23ef389cf68cf7c5cfccbe98c3c6ee Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 26 Oct 2022 12:07:42 +0200 Subject: [PATCH 45/50] Fix execution of version_helper.py to use git tweaks --- tests/ci/version_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index 966858c0747..162bab6a50a 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -344,7 +344,7 @@ def main(): update_contributors() return - version = get_version_from_repo(args.version_path) + version = get_version_from_repo(args.version_path, Git(True)) if args.update: version = version.update(args.update) From e0852ba028b59db7e538cf0265c768b66fa04513 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 26 Oct 2022 13:15:32 +0200 Subject: [PATCH 46/50] Always run BuilderReport in all CI types --- .github/workflows/backport_branches.yml | 2 ++ .github/workflows/master.yml | 2 ++ .github/workflows/release_branches.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 4c8d023f2ec..30a77a9b27f 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -466,6 +466,7 @@ jobs: - BuilderDebTsan - BuilderDebDebug runs-on: [self-hosted, style-checker] + if: ${{ success() || failure() }} steps: - name: Set envs run: | @@ -504,6 +505,7 @@ jobs: - BuilderBinDarwin - BuilderBinDarwinAarch64 runs-on: [self-hosted, style-checker] + if: ${{ success() || failure() }} steps: - name: Set envs run: | diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 3d22cb984dd..fba8a975ca6 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -974,6 +974,7 @@ jobs: - BuilderDebTsan - BuilderDebUBsan runs-on: [self-hosted, style-checker] + if: ${{ success() || failure() }} steps: - name: Set envs run: | @@ -1021,6 +1022,7 @@ jobs: - BuilderBinClangTidy - BuilderDebShared runs-on: [self-hosted, style-checker] + if: ${{ success() || failure() }} steps: - name: Set envs run: | diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 8f42ca92646..abe85d3e72d 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -541,6 +541,7 @@ jobs: - BuilderDebMsan - BuilderDebDebug runs-on: [self-hosted, style-checker] + if: ${{ success() || failure() }} steps: - name: Set envs run: | @@ -580,6 +581,7 @@ jobs: - BuilderBinDarwin - BuilderBinDarwinAarch64 runs-on: [self-hosted, style-checker] + if: ${{ success() || failure() }} steps: - name: Set envs run: | From 4e2f56ac34e6a92da3db446be4fe12d3ada63296 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 26 Oct 2022 13:24:51 +0200 Subject: [PATCH 47/50] Try to force StyleCheck running --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 2795dc62d6d..3951f99b16b 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -112,7 +112,7 @@ jobs: StyleCheck: needs: DockerHubPush runs-on: [self-hosted, style-checker] - if: ${{ success() || failure() }} + if: ${{ success() || failure() || always() }} steps: - name: Set envs run: | From d3461abd1f0c43055daaff0def45c32de2b203f1 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 26 Oct 2022 15:27:09 +0200 Subject: [PATCH 48/50] Update version to 22.11.1.1 --- cmake/autogenerated_versions.txt | 10 ++++---- .../StorageSystemContributors.generated.cpp | 24 +++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 9728451f38a..11b37f5a7c8 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54467) +SET(VERSION_REVISION 54468) SET(VERSION_MAJOR 22) -SET(VERSION_MINOR 10) +SET(VERSION_MINOR 11) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH 3030d4c7ff09ec44ab07d0a8069ea923227288a1) -SET(VERSION_DESCRIBE v22.10.1.1-testing) -SET(VERSION_STRING 22.10.1.1) +SET(VERSION_GITHASH 98ab5a3c189232ea2a3dddb9d2be7196ae8b3434) +SET(VERSION_DESCRIBE v22.11.1.1-testing) +SET(VERSION_STRING 22.11.1.1) # end of autochange diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp index 25eae3b83b6..e1f4f7b82bf 100644 --- a/src/Storages/System/StorageSystemContributors.generated.cpp +++ b/src/Storages/System/StorageSystemContributors.generated.cpp @@ -12,6 +12,7 @@ const char * auto_contributors[] { "821008736@qq.com", "ANDREI STAROVEROV", "Aaron Katz", + "Adam Rutkowski", "Adri Fernandez", "Ahmed Dardery", "Aimiyoo", @@ -76,11 +77,15 @@ const char * auto_contributors[] { "Alexey Elymanov", "Alexey Gusev", "Alexey Ilyukhov", + "Alexey Ivanov", "Alexey Milovidov", "Alexey Tronov", "Alexey Vasiliev", "Alexey Zatelepin", "Alexsey Shestakov", + "AlfVII", + "Alfonso Martinez", + "Alfred Xu", "Ali Demirci", "Aliaksandr Pliutau", "Aliaksandr Shylau", @@ -196,6 +201,7 @@ const char * auto_contributors[] { "Brian Hunter", "Bulat Gaifullin", "Carbyn", + "Carlos Rodríguez Hernández", "Caspian", "Chao Ma", "Chao Wang", @@ -222,6 +228,7 @@ const char * auto_contributors[] { "DIAOZHAFENG", "Dale McDiarmid", "Dale Mcdiarmid", + "Dalitso Banda", "Dan Roscigno", "DanRoscigno", "Daniel Bershatsky", @@ -267,6 +274,7 @@ const char * auto_contributors[] { "Dmitry S..ky / skype: dvska-at-skype", "Dmitry Ukolov", "Doge", + "Dom Del Nano", "Dongdong Yang", "DoomzD", "Dr. Strange Looker", @@ -276,6 +284,7 @@ const char * auto_contributors[] { "Egor Savin", "Ekaterina", "Eldar Zaitov", + "Elena", "Elena Baskakova", "Elghazal Ahmed", "Elizaveta Mironyuk", @@ -342,6 +351,7 @@ const char * auto_contributors[] { "Grigory Pervakov", "GruffGemini", "Guillaume Tassery", + "Guo Wangyang", "Guo Wei (William)", "Haavard Kvaalen", "Habibullah Oladepo", @@ -349,6 +359,7 @@ const char * auto_contributors[] { "Hakob Saghatelyan", "Hamoon", "Han Fei", + "Han Shukai", "Harry Lee", "Harry-Lee", "HarryLeeIBM", @@ -404,6 +415,7 @@ const char * auto_contributors[] { "Jack Song", "JackyWoo", "Jacob Hayes", + "Jacob Herrington", "Jake Liu", "Jakub Kuklis", "James Maidment", @@ -419,6 +431,7 @@ const char * auto_contributors[] { "Jiading Guo", "Jiang Tao", "Jianmei Zhang", + "Jiebin Sun", "Jochen Schalanda", "John", "John Hummel", @@ -432,6 +445,7 @@ const char * auto_contributors[] { "Julian Gilyadov", "Julian Zhou", "Julio Jimenez", + "Jus", "Justin Hilliard", "Kang Liu", "Karl Pietrzak", @@ -652,6 +666,7 @@ const char * auto_contributors[] { "OuO", "PHO", "Pablo Alegre", + "Pablo Marcos", "Paramtamtam", "Patrick Zippenfenig", "Paul Loyd", @@ -681,6 +696,7 @@ const char * auto_contributors[] { "Prashant Shahi", "Pxl", "Pysaoke", + "Quanfa Fu", "Quid37", "Rafael Acevedo", "Rafael David Tinoco", @@ -693,6 +709,7 @@ const char * auto_contributors[] { "RedClusive", "RegulusZ", "Reilee", + "Reinaldy Rafli", "Reto Kromer", "Ri", "Rich Raposa", @@ -726,6 +743,7 @@ const char * auto_contributors[] { "Sachin", "Safronov Michail", "SaltTan", + "Salvatore Mesoraca", "Sami Kerola", "Samuel Chou", "San", @@ -927,6 +945,7 @@ const char * auto_contributors[] { "ZhiYong Wang", "Zhichang Yu", "Zhichun Wu", + "Zhiguo Zhou", "Zhipeng", "Zijie Lu", "Zoran Pandovski", @@ -950,6 +969,7 @@ const char * auto_contributors[] { "alexander goryanets", "alexander kozhikhov", "alexey-milovidov", + "alexeyerm", "alexeypavlenko", "alfredlu", "amesaru", @@ -1131,6 +1151,7 @@ const char * auto_contributors[] { "jennyma", "jetgm", "jewisliu", + "jferroal", "jiahui-97", "jianmei zhang", "jinjunzh", @@ -1236,6 +1257,7 @@ const char * auto_contributors[] { "mo-avatar", "morty", "moscas", + "mosinnik", "mreddy017", "msaf1980", "msirm", @@ -1321,6 +1343,7 @@ const char * auto_contributors[] { "simon-says", "snyk-bot", "songenjie", + "sperlingxx", "spff", "spongedc", "spume", @@ -1422,6 +1445,7 @@ const char * auto_contributors[] { "zhongyuankai", "zhoubintao", "zhukai", + "zimv", "zkun", "zlx19950903", "zombee0", From 42f5a3b2f886711459771526e012710d068c2fd0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 26 Oct 2022 13:34:29 +0000 Subject: [PATCH 49/50] Update version_date.tsv and changelogs after v22.10.1.1877-stable --- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- .../{v22.10.1.1875-stable.md => v22.10.1.1877-stable.md} | 3 ++- utils/list-versions/version_date.tsv | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) rename docs/changelogs/{v22.10.1.1875-stable.md => v22.10.1.1877-stable.md} (99%) diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 9b633b66188..cf4eb3fe645 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="22.10.1.1875" +ARG VERSION="22.10.1.1877" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 35c78763a31..d26657a7979 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -21,7 +21,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="22.10.1.1875" +ARG VERSION="22.10.1.1877" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v22.10.1.1875-stable.md b/docs/changelogs/v22.10.1.1877-stable.md similarity index 99% rename from docs/changelogs/v22.10.1.1875-stable.md rename to docs/changelogs/v22.10.1.1877-stable.md index 49f93168a00..77e540ce928 100644 --- a/docs/changelogs/v22.10.1.1875-stable.md +++ b/docs/changelogs/v22.10.1.1877-stable.md @@ -5,7 +5,7 @@ sidebar_label: 2022 # 2022 Changelog -### ClickHouse release v22.10.1.1875-stable (011ae8675a2) FIXME as compared to v22.9.1.2603-stable (3030d4c7ff0) +### ClickHouse release v22.10.1.1877-stable (98ab5a3c189) FIXME as compared to v22.9.1.2603-stable (3030d4c7ff0) #### Backward Incompatible Change * Rename cache commands: `show caches` -> `show filesystem caches`, `describe cache` -> `describe filesystem cache`. [#41508](https://github.com/ClickHouse/ClickHouse/pull/41508) ([Kseniia Sumarokova](https://github.com/kssenii)). @@ -348,4 +348,5 @@ sidebar_label: 2022 * Fix typo in cmake code related to fuzzing [#42627](https://github.com/ClickHouse/ClickHouse/pull/42627) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Fix build [#42635](https://github.com/ClickHouse/ClickHouse/pull/42635) ([Anton Popov](https://github.com/CurtizJ)). * Add .rgignore for test data [#42639](https://github.com/ClickHouse/ClickHouse/pull/42639) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky 02457_datediff_via_unix_epoch test [#42655](https://github.com/ClickHouse/ClickHouse/pull/42655) ([Roman Vasin](https://github.com/rvasin)). diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 7bbd8547506..e72fce63fda 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,4 +1,4 @@ -v22.10.1.1875-stable 2022-10-26 +v22.10.1.1877-stable 2022-10-26 v22.9.3.18-stable 2022-09-30 v22.9.2.7-stable 2022-09-23 v22.9.1.2603-stable 2022-09-22 From 9b4da8c89ae79f0f6e4da4c8c20d862ef52a48e9 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 26 Oct 2022 15:41:12 +0200 Subject: [PATCH 50/50] Update SECURITY.md --- SECURITY.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index fb6caa92cb8..0fb333c8ea3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -10,9 +10,11 @@ The following versions of ClickHouse server are currently being supported with s | Version | Supported | |:-|:-| +| 22.10 | ✔️ | +| 22.9 | ✔️ | | 22.8 | ✔️ | -| 22.7 | ✔️ | -| 22.6 | ✔️ | +| 22.7 | ❌ | +| 22.6 | ❌ | | 22.5 | ❌ | | 22.4 | ❌ | | 22.3 | ✔️ |