From 3ab8ba0d4ab0e4446883bf57ba3d859d1ee49a52 Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Mon, 27 May 2024 21:33:54 +0000 Subject: [PATCH 001/129] Update zlib-ng from 2.0.2 to 2.1.6 --- contrib/zlib-ng | 2 +- contrib/zlib-ng-cmake/CMakeLists.txt | 84 +++++++++++++++------------- 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/contrib/zlib-ng b/contrib/zlib-ng index 50f0eae1a41..bfb184bb0fb 160000 --- a/contrib/zlib-ng +++ b/contrib/zlib-ng @@ -1 +1 @@ -Subproject commit 50f0eae1a411764cd6d1e85b3ce471438acd3c1c +Subproject commit bfb184bb0fbdabe82f3a36d209e56c3e2c33866a diff --git a/contrib/zlib-ng-cmake/CMakeLists.txt b/contrib/zlib-ng-cmake/CMakeLists.txt index 79f343bfc75..78a5f1ae3a8 100644 --- a/contrib/zlib-ng-cmake/CMakeLists.txt +++ b/contrib/zlib-ng-cmake/CMakeLists.txt @@ -14,6 +14,7 @@ add_definitions(-DHAVE_VISIBILITY_HIDDEN) add_definitions(-DHAVE_VISIBILITY_INTERNAL) add_definitions(-DHAVE_BUILTIN_CTZ) add_definitions(-DHAVE_BUILTIN_CTZLL) +add_definitions(-DHAVE_ATTRIBUTE_ALIGNED) set(ZLIB_ARCH_SRCS) set(ZLIB_ARCH_HDRS) @@ -26,14 +27,15 @@ if(ARCH_AARCH64) add_definitions(-DARM_FEATURES) add_definitions(-DARM_AUXV_HAS_CRC32 -DARM_ASM_HWCAP) add_definitions(-DARM_AUXV_HAS_NEON) - add_definitions(-DARM_ACLE_CRC_HASH) - add_definitions(-DARM_NEON_ADLER32 -DARM_NEON_CHUNKSET -DARM_NEON_SLIDEHASH) + add_definitions(-DARM_ACLE) + add_definitions(-DARM_NEON) - list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/arm.h) - list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/armfeature.c) + list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/arm_features.h) + list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/arm_features.c) set(ACLE_SRCS ${ARCHDIR}/crc32_acle.c ${ARCHDIR}/insert_string_acle.c) list(APPEND ZLIB_ARCH_SRCS ${ACLE_SRCS}) - set(NEON_SRCS ${ARCHDIR}/adler32_neon.c ${ARCHDIR}/chunkset_neon.c ${ARCHDIR}/slide_neon.c) + set(NEON_SRCS ${ARCHDIR}/adler32_neon.c ${ARCHDIR}/chunkset_neon.c + ${ARCHDIR}/compare256_neon.c ${ARCHDIR}/slide_hash_neon.c) list(APPEND ZLIB_ARCH_SRCS ${NEON_SRCS}) elseif(ARCH_PPC64LE) @@ -41,50 +43,47 @@ elseif(ARCH_PPC64LE) add_definitions(-DPOWER8) add_definitions(-DPOWER_FEATURES) - add_definitions(-DPOWER8_VSX_ADLER32) - add_definitions(-DPOWER8_VSX_SLIDEHASH) + add_definitions(-DPOWER8_VSX) + add_definitions(-DPOWER8_VSX_CRC32) - list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/power.h) - list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/power.c) - set(POWER8_SRCS ${ARCHDIR}/adler32_power8.c ${ARCHDIR}/slide_hash_power8.c) + list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/power_features.h) + list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/power_features.c) + set(POWER8_SRCS ${ARCHDIR}/adler32_power8.c ${ARCHDIR}/chunkset_power8.c ${ARCHDIR}/slide_hash_power8.c) + list(APPEND POWER8_SRCS ${ARCHDIR}/crc32_power8.c) list(APPEND ZLIB_ARCH_SRCS ${POWER8_SRCS}) elseif(ARCH_AMD64) set(ARCHDIR "${SOURCE_DIR}/arch/x86") add_definitions(-DX86_FEATURES) - list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/x86.h) - list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/x86.c) + list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/x86_features.h) + list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/x86_features.c) if(ENABLE_AVX2) - add_definitions(-DX86_AVX2 -DX86_AVX2_ADLER32 -DX86_AVX_CHUNKSET) - set(AVX2_SRCS ${ARCHDIR}/slide_avx.c) - list(APPEND AVX2_SRCS ${ARCHDIR}/chunkset_avx.c) - list(APPEND AVX2_SRCS ${ARCHDIR}/compare258_avx.c) - list(APPEND AVX2_SRCS ${ARCHDIR}/adler32_avx.c) + add_definitions(-DX86_AVX2) + set(AVX2_SRCS ${ARCHDIR}/slide_hash_avx2.c) + list(APPEND AVX2_SRCS ${ARCHDIR}/chunkset_avx2.c) + list(APPEND AVX2_SRCS ${ARCHDIR}/compare256_avx2.c) + list(APPEND AVX2_SRCS ${ARCHDIR}/adler32_avx2.c) list(APPEND ZLIB_ARCH_SRCS ${AVX2_SRCS}) endif() if(ENABLE_SSE42) - add_definitions(-DX86_SSE42_CRC_HASH) - set(SSE42_SRCS ${ARCHDIR}/insert_string_sse.c) - list(APPEND ZLIB_ARCH_SRCS ${SSE42_SRCS}) - add_definitions(-DX86_SSE42_CRC_INTRIN) - add_definitions(-DX86_SSE42_CMP_STR) - set(SSE42_SRCS ${ARCHDIR}/compare258_sse.c) + add_definitions(-DX86_SSE42) + set(SSE42_SRCS ${ARCHDIR}/adler32_sse42.c ${ARCHDIR}/insert_string_sse42.c) list(APPEND ZLIB_ARCH_SRCS ${SSE42_SRCS}) endif() if(ENABLE_SSSE3) - add_definitions(-DX86_SSSE3 -DX86_SSSE3_ADLER32) - set(SSSE3_SRCS ${ARCHDIR}/adler32_ssse3.c) + add_definitions(-DX86_SSSE3) + set(SSSE3_SRCS ${ARCHDIR}/adler32_ssse3.c ${ARCHDIR}/chunkset_ssse3.c) list(APPEND ZLIB_ARCH_SRCS ${SSSE3_SRCS}) endif() if(ENABLE_PCLMULQDQ) add_definitions(-DX86_PCLMULQDQ_CRC) - set(PCLMULQDQ_SRCS ${ARCHDIR}/crc_folding.c) + set(PCLMULQDQ_SRCS ${ARCHDIR}/crc32_pclmulqdq.c) list(APPEND ZLIB_ARCH_SRCS ${PCLMULQDQ_SRCS}) endif() - add_definitions(-DX86_SSE2 -DX86_SSE2_CHUNKSET -DX86_SSE2_SLIDEHASH) - set(SSE2_SRCS ${ARCHDIR}/chunkset_sse.c ${ARCHDIR}/slide_sse.c) + add_definitions(-DX86_SSE2) + set(SSE2_SRCS ${ARCHDIR}/chunkset_sse2.c ${ARCHDIR}/compare256_sse2.c ${ARCHDIR}/slide_hash_sse2.c) list(APPEND ZLIB_ARCH_SRCS ${SSE2_SRCS}) add_definitions(-DX86_NOCHECK_SSE2) endif () @@ -106,39 +105,45 @@ generate_cmakein(${SOURCE_DIR}/zconf.h.in ${CMAKE_CURRENT_BINARY_DIR}/zconf.h.cm set(ZLIB_SRCS ${SOURCE_DIR}/adler32.c + ${SOURCE_DIR}/adler32_fold.c ${SOURCE_DIR}/chunkset.c - ${SOURCE_DIR}/compare258.c + ${SOURCE_DIR}/compare256.c ${SOURCE_DIR}/compress.c - ${SOURCE_DIR}/crc32.c - ${SOURCE_DIR}/crc32_comb.c + ${SOURCE_DIR}/cpu_features.c + ${SOURCE_DIR}/crc32_braid.c + ${SOURCE_DIR}/crc32_braid_comb.c + ${SOURCE_DIR}/crc32_fold.c ${SOURCE_DIR}/deflate.c ${SOURCE_DIR}/deflate_fast.c + ${SOURCE_DIR}/deflate_huff.c ${SOURCE_DIR}/deflate_medium.c ${SOURCE_DIR}/deflate_quick.c + ${SOURCE_DIR}/deflate_rle.c ${SOURCE_DIR}/deflate_slow.c + ${SOURCE_DIR}/deflate_stored.c ${SOURCE_DIR}/functable.c ${SOURCE_DIR}/infback.c - ${SOURCE_DIR}/inffast.c ${SOURCE_DIR}/inflate.c ${SOURCE_DIR}/inftrees.c ${SOURCE_DIR}/insert_string.c + ${SOURCE_DIR}/insert_string_roll.c + ${SOURCE_DIR}/slide_hash.c ${SOURCE_DIR}/trees.c ${SOURCE_DIR}/uncompr.c ${SOURCE_DIR}/zutil.c +) + +set(ZLIB_GZFILE_SRCS ${SOURCE_DIR}/gzlib.c - ${SOURCE_DIR}/gzread.c + ${CMAKE_CURRENT_BINARY_DIR}/gzread.c ${SOURCE_DIR}/gzwrite.c ) -set(ZLIB_ALL_SRCS ${ZLIB_SRCS} ${ZLIB_ARCH_SRCS}) +set(ZLIB_ALL_SRCS ${ZLIB_SRCS} ${ZLIB_ARCH_SRCS} ${ZLIB_GZFILE_SRCS}) add_library(_zlib ${ZLIB_ALL_SRCS}) add_library(ch_contrib::zlib ALIAS _zlib) -# https://github.com/zlib-ng/zlib-ng/pull/733 -# This is disabed by default -add_compile_definitions(Z_TLS=__thread) - if(HAVE_UNISTD_H) SET(ZCONF_UNISTD_LINE "#if 1 /* was set to #if 1 by configure/cmake/etc */") else() @@ -153,6 +158,9 @@ endif() set(ZLIB_PC ${CMAKE_CURRENT_BINARY_DIR}/zlib.pc) configure_file(${SOURCE_DIR}/zlib.pc.cmakein ${ZLIB_PC} @ONLY) configure_file(${CMAKE_CURRENT_BINARY_DIR}/zconf.h.cmakein ${CMAKE_CURRENT_BINARY_DIR}/zconf.h @ONLY) +configure_file(${SOURCE_DIR}/zlib.h.in ${CMAKE_CURRENT_BINARY_DIR}/zlib.h @ONLY) +configure_file(${SOURCE_DIR}/zlib_name_mangling.h.in ${CMAKE_CURRENT_BINARY_DIR}/zlib_name_mangling.h @ONLY) +configure_file(${SOURCE_DIR}/gzread.c.in ${CMAKE_CURRENT_BINARY_DIR}/gzread.c @ONLY) # We should use same defines when including zlib.h as used when zlib compiled target_compile_definitions (_zlib PUBLIC ZLIB_COMPAT WITH_GZFILEOP) From 62d6e3d3396fbcbcc6f666ac9b7bc2b60fe828b2 Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Wed, 29 May 2024 00:19:25 +0000 Subject: [PATCH 002/129] Fix ARM --- contrib/zlib-ng-cmake/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/zlib-ng-cmake/CMakeLists.txt b/contrib/zlib-ng-cmake/CMakeLists.txt index 78a5f1ae3a8..40397e14f20 100644 --- a/contrib/zlib-ng-cmake/CMakeLists.txt +++ b/contrib/zlib-ng-cmake/CMakeLists.txt @@ -28,7 +28,9 @@ if(ARCH_AARCH64) add_definitions(-DARM_AUXV_HAS_CRC32 -DARM_ASM_HWCAP) add_definitions(-DARM_AUXV_HAS_NEON) add_definitions(-DARM_ACLE) + add_definitions(-DHAVE_ARM_ACLE_H) add_definitions(-DARM_NEON) + add_definitions(-DARM_NEON_HASLD4) list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/arm_features.h) list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/arm_features.c) From 8115926eb61779d669509f16863730557882b3c8 Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Wed, 29 May 2024 04:49:19 +0000 Subject: [PATCH 003/129] Fix ARM some more --- contrib/zlib-ng-cmake/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/zlib-ng-cmake/CMakeLists.txt b/contrib/zlib-ng-cmake/CMakeLists.txt index 40397e14f20..1ed1d98ed66 100644 --- a/contrib/zlib-ng-cmake/CMakeLists.txt +++ b/contrib/zlib-ng-cmake/CMakeLists.txt @@ -25,6 +25,7 @@ if(ARCH_AARCH64) set(ARCHDIR "${SOURCE_DIR}/arch/arm") add_definitions(-DARM_FEATURES) + add_definitions(-DHAVE_SYS_AUXV_H) add_definitions(-DARM_AUXV_HAS_CRC32 -DARM_ASM_HWCAP) add_definitions(-DARM_AUXV_HAS_NEON) add_definitions(-DARM_ACLE) From 06781efcb785a0504d9599e6a1446e235caef13d Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Thu, 20 Jun 2024 22:59:36 +0000 Subject: [PATCH 004/129] Switch to a cleaner update, hopefully fix builds --- contrib/zlib-ng | 2 +- contrib/zlib-ng-cmake/CMakeLists.txt | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/contrib/zlib-ng b/contrib/zlib-ng index bfb184bb0fb..c19ba056b7c 160000 --- a/contrib/zlib-ng +++ b/contrib/zlib-ng @@ -1 +1 @@ -Subproject commit bfb184bb0fbdabe82f3a36d209e56c3e2c33866a +Subproject commit c19ba056b7cc8029bb80f509956090c7ded58032 diff --git a/contrib/zlib-ng-cmake/CMakeLists.txt b/contrib/zlib-ng-cmake/CMakeLists.txt index 1ed1d98ed66..91a8eb2e0a3 100644 --- a/contrib/zlib-ng-cmake/CMakeLists.txt +++ b/contrib/zlib-ng-cmake/CMakeLists.txt @@ -15,6 +15,7 @@ add_definitions(-DHAVE_VISIBILITY_INTERNAL) add_definitions(-DHAVE_BUILTIN_CTZ) add_definitions(-DHAVE_BUILTIN_CTZLL) add_definitions(-DHAVE_ATTRIBUTE_ALIGNED) +add_definitions(-DHAVE_POSIX_MEMALIGN) set(ZLIB_ARCH_SRCS) set(ZLIB_ARCH_HDRS) @@ -44,10 +45,16 @@ if(ARCH_AARCH64) elseif(ARCH_PPC64LE) set(ARCHDIR "${SOURCE_DIR}/arch/power") - add_definitions(-DPOWER8) add_definitions(-DPOWER_FEATURES) - add_definitions(-DPOWER8_VSX) - add_definitions(-DPOWER8_VSX_CRC32) + add_definitions(-DHAVE_SYS_AUXV_H) + + if(POWER9) + add_definitions(-DPOWER9) + else() + add_definitions(-DPOWER8) + add_definitions(-DPOWER8_VSX) + add_definitions(-DPOWER8_VSX_CRC32) + endif() list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/power_features.h) list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/power_features.c) From a2b17b01f9561fd1853851932a2ae77513c49e26 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 8 Jul 2024 17:47:56 +0000 Subject: [PATCH 005/129] fix filling of missed subcolumns --- src/Interpreters/inplaceBlockConversions.cpp | 38 ++++++++---- src/Storages/MergeTree/IMergeTreeReader.cpp | 48 +++++++++++---- .../MergeTree/MergeTreeReaderCompact.cpp | 60 ++++++++++++------- .../MergeTree/MergeTreeReaderCompact.h | 1 + .../03203_fill_missed_subcolumns.reference | 31 ++++++++++ .../03203_fill_missed_subcolumns.sql | 47 +++++++++++++++ 6 files changed, 181 insertions(+), 44 deletions(-) create mode 100644 tests/queries/0_stateless/03203_fill_missed_subcolumns.reference create mode 100644 tests/queries/0_stateless/03203_fill_missed_subcolumns.sql diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp index 239cce5b427..f7d8a2a2daf 100644 --- a/src/Interpreters/inplaceBlockConversions.cpp +++ b/src/Interpreters/inplaceBlockConversions.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -35,8 +36,13 @@ namespace /// Add all required expressions for missing columns calculation void addDefaultRequiredExpressionsRecursively( - const Block & block, const String & required_column_name, DataTypePtr required_column_type, - const ColumnsDescription & columns, ASTPtr default_expr_list_accum, NameSet & added_columns, bool null_as_default) + const Block & block, + const String & required_column_name, + DataTypePtr required_column_type, + const ColumnsDescription & columns, + ASTPtr default_expr_list_accum, + NameSet & added_columns, + bool null_as_default) { checkStackSize(); @@ -273,6 +279,20 @@ static std::unordered_map collectOffsetsColumns( return offsets_columns; } +static ColumnPtr createColumnWithDefaultValue(const IDataType & data_type, const String & subcolumn_name, size_t num_rows) +{ + auto column = data_type.createColumnConstWithDefaultValue(num_rows); + + if (subcolumn_name.empty()) + return column->convertToFullColumnIfConst(); + + /// Firstly get subcolumn from const column and then replicate. + column = assert_cast(*column).getDataColumnPtr(); + column = data_type.getSubcolumn(subcolumn_name, column); + + return ColumnConst::create(std::move(column), num_rows)->convertToFullColumnIfConst(); +} + void fillMissingColumns( Columns & res_columns, size_t num_rows, @@ -298,21 +318,19 @@ void fillMissingColumns( auto requested_column = requested_columns.begin(); for (size_t i = 0; i < num_columns; ++i, ++requested_column) { - const auto & [name, type] = *requested_column; - - if (res_columns[i] && partially_read_columns.contains(name)) + if (res_columns[i] && partially_read_columns.contains(requested_column->name)) res_columns[i] = nullptr; if (res_columns[i]) continue; - if (metadata_snapshot && metadata_snapshot->getColumns().hasDefault(name)) + if (metadata_snapshot && metadata_snapshot->getColumns().hasDefault(requested_column->getNameInStorage())) continue; std::vector current_offsets; size_t num_dimensions = 0; - const auto * array_type = typeid_cast(type.get()); + const auto * array_type = typeid_cast(requested_column->type.get()); if (array_type && !offsets_columns.empty()) { num_dimensions = getNumberOfDimensions(*array_type); @@ -348,10 +366,10 @@ void fillMissingColumns( if (!current_offsets.empty()) { size_t num_empty_dimensions = num_dimensions - current_offsets.size(); - auto scalar_type = createArrayOfType(getBaseTypeOfArray(type), num_empty_dimensions); + auto scalar_type = createArrayOfType(getBaseTypeOfArray(requested_column->getTypeInStorage()), num_empty_dimensions); size_t data_size = assert_cast(*current_offsets.back()).getData().back(); - res_columns[i] = scalar_type->createColumnConstWithDefaultValue(data_size)->convertToFullColumnIfConst(); + res_columns[i] = createColumnWithDefaultValue(*scalar_type, requested_column->getSubcolumnName(), data_size); for (auto it = current_offsets.rbegin(); it != current_offsets.rend(); ++it) res_columns[i] = ColumnArray::create(res_columns[i], *it); @@ -360,7 +378,7 @@ void fillMissingColumns( { /// We must turn a constant column into a full column because the interpreter could infer /// that it is constant everywhere but in some blocks (from other parts) it can be a full column. - res_columns[i] = type->createColumnConstWithDefaultValue(num_rows)->convertToFullColumnIfConst(); + res_columns[i] = createColumnWithDefaultValue(*requested_column->getTypeInStorage(), requested_column->getSubcolumnName(), num_rows); } } } diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 4ad7f6ef991..aff1001163e 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -144,19 +145,26 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns throw Exception(ErrorCodes::LOGICAL_ERROR, "invalid number of columns passed to MergeTreeReader::fillMissingColumns. " "Expected {}, got {}", num_columns, res_columns.size()); - /// Convert columns list to block. - /// TODO: rewrite with columns interface. It will be possible after changes in ExpressionActions. - auto name_and_type = requested_columns.begin(); - for (size_t pos = 0; pos < num_columns; ++pos, ++name_and_type) - { - if (res_columns[pos] == nullptr) - continue; + NameSet full_requested_columns_set; + NamesAndTypesList full_requested_columns; - additional_columns.insert({res_columns[pos], name_and_type->type, name_and_type->name}); + /// Convert columns list to block. And convert subcolumns to full columns. + /// TODO: rewrite with columns interface. It will be possible after changes in ExpressionActions. + + auto it = requested_columns.begin(); + for (size_t pos = 0; pos < num_columns; ++pos, ++it) + { + auto name_in_storage = it->getNameInStorage(); + + if (full_requested_columns_set.emplace(name_in_storage).second) + full_requested_columns.emplace_back(name_in_storage, it->getTypeInStorage()); + + if (res_columns[pos]) + additional_columns.insert({res_columns[pos], it->type, it->name}); } auto dag = DB::evaluateMissingDefaults( - additional_columns, requested_columns, + additional_columns, full_requested_columns, storage_snapshot->metadata->getColumns(), data_part_info_for_read->getContext()); @@ -170,9 +178,18 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns } /// Move columns from block. - name_and_type = requested_columns.begin(); - for (size_t pos = 0; pos < num_columns; ++pos, ++name_and_type) - res_columns[pos] = std::move(additional_columns.getByName(name_and_type->name).column); + it = requested_columns.begin(); + for (size_t pos = 0; pos < num_columns; ++pos, ++it) + { + auto name_in_storage = it->getNameInStorage(); + res_columns[pos] = additional_columns.getByName(name_in_storage).column; + + if (it->isSubcolumn()) + { + const auto & type_in_storage = it->getTypeInStorage(); + res_columns[pos] = type_in_storage->getSubcolumn(it->getSubcolumnName(), res_columns[pos]); + } + } } catch (Exception & e) { @@ -192,7 +209,12 @@ bool IMergeTreeReader::isSubcolumnOffsetsOfNested(const String & name_in_storage if (!data_part_info_for_read->isWidePart() || subcolumn_name != "size0") return false; - return Nested::isSubcolumnOfNested(name_in_storage, part_columns); + auto split = Nested::splitName(name_in_storage); + if (split.second.empty()) + return false; + + auto nested_column = part_columns.tryGetColumn(GetColumnsOptions::All, split.first); + return nested_column && isNested(nested_column->type); } String IMergeTreeReader::getColumnNameInPart(const NameAndTypePair & required_column) const diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index a2b8f0ad96f..fde9dafffb8 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -60,39 +60,25 @@ void MergeTreeReaderCompact::fillColumnPositions() for (size_t i = 0; i < columns_num; ++i) { - const auto & column_to_read = columns_to_read[i]; - + auto & column_to_read = columns_to_read[i]; auto position = data_part_info_for_read->getColumnPosition(column_to_read.getNameInStorage()); - bool is_array = isArray(column_to_read.type); if (column_to_read.isSubcolumn()) { - auto storage_column_from_part = getColumnInPart( - {column_to_read.getNameInStorage(), column_to_read.getTypeInStorage()}); + NameAndTypePair column_in_storage{column_to_read.getNameInStorage(), column_to_read.getTypeInStorage()}; + auto storage_column_from_part = getColumnInPart(column_in_storage); auto subcolumn_name = column_to_read.getSubcolumnName(); if (!storage_column_from_part.type->hasSubcolumn(subcolumn_name)) position.reset(); } + column_positions[i] = std::move(position); + /// If array of Nested column is missing in part, /// we have to read its offsets if they exist. - if (!position && is_array) - { - auto column_to_read_with_subcolumns = getColumnConvertedToSubcolumnOfNested(column_to_read); - auto name_level_for_offsets = findColumnForOffsets(column_to_read_with_subcolumns); - - if (name_level_for_offsets.has_value()) - { - column_positions[i] = data_part_info_for_read->getColumnPosition(name_level_for_offsets->first); - columns_for_offsets[i] = name_level_for_offsets; - partially_read_columns.insert(column_to_read.name); - } - } - else - { - column_positions[i] = std::move(position); - } + if (!column_positions[i]) + findPositionForMissedNested(i); } } @@ -125,6 +111,38 @@ NameAndTypePair MergeTreeReaderCompact::getColumnConvertedToSubcolumnOfNested(co Nested::concatenateName(name_in_storage, subcolumn_name)); } +void MergeTreeReaderCompact::findPositionForMissedNested(size_t pos) +{ + auto & column = columns_to_read[pos]; + + bool is_array = isArray(column.type); + bool is_offsets_subcolumn = isArray(column.getTypeInStorage()) && column.getSubcolumnName() == "size0"; + + if (!is_array && !is_offsets_subcolumn) + return; + + NameAndTypePair column_in_storage{column.getNameInStorage(), column.getTypeInStorage()}; + + auto column_to_read_with_subcolumns = getColumnConvertedToSubcolumnOfNested(column_in_storage); + auto name_level_for_offsets = findColumnForOffsets(column_to_read_with_subcolumns); + + if (!name_level_for_offsets) + return; + + column_positions[pos] = data_part_info_for_read->getColumnPosition(name_level_for_offsets->first); + + if (is_offsets_subcolumn) + { + /// Read offsets from antoher array from the same Nested column. + column = {name_level_for_offsets->first, column.getSubcolumnName(), column.getTypeInStorage(), column.type}; + } + else + { + columns_for_offsets[pos] = std::move(name_level_for_offsets); + partially_read_columns.insert(column.name); + } +} + void MergeTreeReaderCompact::readData( const NameAndTypePair & name_and_type, ColumnPtr & column, diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h index a783e595af5..22eabd47930 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -36,6 +36,7 @@ public: protected: void fillColumnPositions(); NameAndTypePair getColumnConvertedToSubcolumnOfNested(const NameAndTypePair & column); + void findPositionForMissedNested(size_t pos); using InputStreamGetter = ISerialization::InputStreamGetter; diff --git a/tests/queries/0_stateless/03203_fill_missed_subcolumns.reference b/tests/queries/0_stateless/03203_fill_missed_subcolumns.reference new file mode 100644 index 00000000000..872eb7da3c8 --- /dev/null +++ b/tests/queries/0_stateless/03203_fill_missed_subcolumns.reference @@ -0,0 +1,31 @@ +0 +2 +4 +6 +8 +0 +2 +4 +6 +8 +1 ['aaa',NULL] [NULL,NULL] +2 ['ccc'] [NULL] +3 [NULL] [NULL] +4 [NULL,'bbb'] ['ddd',NULL] +5 [NULL] [NULL] +1 2 2 +2 1 1 +3 1 1 +4 2 2 +5 1 1 +1 [0,1] [1,1] +2 [0] [1] +3 [1] [1] +4 [1,0] [0,1] +5 [1] [1] +1 ('foo','bar') [1,NULL,3] +2 ('aaa','bbb') [1,NULL,3] +3 ('ccc','ddd') [4,5,6] +1 foo bar 3 [0,1,0] +2 foo bar 3 [0,1,0] +3 ccc ddd 3 [0,0,0] diff --git a/tests/queries/0_stateless/03203_fill_missed_subcolumns.sql b/tests/queries/0_stateless/03203_fill_missed_subcolumns.sql new file mode 100644 index 00000000000..2789c9de35c --- /dev/null +++ b/tests/queries/0_stateless/03203_fill_missed_subcolumns.sql @@ -0,0 +1,47 @@ +DROP TABLE IF EXISTS t_missed_subcolumns; + +-- .null subcolumn + +CREATE TABLE t_missed_subcolumns (x UInt32) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_missed_subcolumns SELECT * FROM numbers(10); + +ALTER TABLE t_missed_subcolumns ADD COLUMN `y` Nullable(UInt32); + +INSERT INTO t_missed_subcolumns SELECT number, if(number % 2, NULL, number) FROM numbers(10); + +SELECT x FROM t_missed_subcolumns WHERE y IS NOT NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT x FROM t_missed_subcolumns WHERE y IS NOT NULL SETTINGS optimize_functions_to_subcolumns = 0; + +DROP TABLE IF EXISTS t_missed_subcolumns; + +-- .null and .size0 subcolumn in array + +CREATE TABLE t_missed_subcolumns (id UInt64, `n.a` Array(Nullable(String))) ENGINE = MergeTree ORDER BY id; + +INSERT INTO t_missed_subcolumns VALUES (1, ['aaa', NULL]) (2, ['ccc']) (3, [NULL]); +ALTER TABLE t_missed_subcolumns ADD COLUMN `n.b` Array(Nullable(String)); +INSERT INTO t_missed_subcolumns VALUES (4, [NULL, 'bbb'], ['ddd', NULL]), (5, [NULL], [NULL]); + +SELECT id, n.a, n.b FROM t_missed_subcolumns ORDER BY id; +SELECT id, n.a.size0, n.b.size0 FROM t_missed_subcolumns ORDER BY id; +SELECT id, n.a.null, n.b.null FROM t_missed_subcolumns ORDER BY id; + +DROP TABLE IF EXISTS t_missed_subcolumns; + +-- subcolumns and custom defaults + +CREATE TABLE t_missed_subcolumns (id UInt64) ENGINE = MergeTree ORDER BY id; +SYSTEM STOP MERGES t_missed_subcolumns; + +INSERT INTO t_missed_subcolumns VALUES (1); + +ALTER TABLE t_missed_subcolumns ADD COLUMN t Tuple(a String, b String) DEFAULT ('foo', 'bar'); +INSERT INTO t_missed_subcolumns VALUES (2, ('aaa', 'bbb')); + +ALTER TABLE t_missed_subcolumns ADD COLUMN arr Array(Nullable(UInt64)) DEFAULT [1, NULL, 3]; +INSERT INTO t_missed_subcolumns VALUES (3, ('ccc', 'ddd'), [4, 5, 6]); + +SELECT id, t, arr FROM t_missed_subcolumns ORDER BY id; +SELECT id, t.a, t.b, arr.size0, arr.null FROM t_missed_subcolumns ORDER BY id; + +DROP TABLE t_missed_subcolumns; From 246f421f2402799fd11b22a608b4d0d497cb8438 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Tue, 9 Jul 2024 16:33:57 +0200 Subject: [PATCH 006/129] merge tree sink cancel delayed_chunk --- .../Transforms/ExceptionKeepingTransform.h | 2 +- src/Storages/MergeTree/MergeTreeSink.cpp | 9 +++++++++ .../MergeTree/ReplicatedMergeTreeSink.cpp | 15 +++++++++++++++ src/Storages/MergeTree/ReplicatedMergeTreeSink.h | 2 ++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/Processors/Transforms/ExceptionKeepingTransform.h b/src/Processors/Transforms/ExceptionKeepingTransform.h index 000b5da798a..9aa33a8cbe5 100644 --- a/src/Processors/Transforms/ExceptionKeepingTransform.h +++ b/src/Processors/Transforms/ExceptionKeepingTransform.h @@ -52,7 +52,7 @@ protected: virtual void onConsume(Chunk chunk) = 0; virtual GenerateResult onGenerate() = 0; virtual void onFinish() {} - virtual void onException(std::exception_ptr /* exception */) {} + virtual void onException(std::exception_ptr /* exception */) { } public: ExceptionKeepingTransform(const Block & in_header, const Block & out_header, bool ignore_on_start_and_finish_ = true); diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 05751e0fa6f..2e8f6db6868 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -56,6 +56,15 @@ void MergeTreeSink::onFinish() void MergeTreeSink::onCancel() { + if (!delayed_chunk) + return; + + for (auto & partition : delayed_chunk->partitions) + { + partition.temp_part.cancel(); + } + + delayed_chunk.reset(); } void MergeTreeSink::consume(Chunk chunk) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 4b4f4c33e7d..93f82a5a789 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -1155,6 +1155,21 @@ void ReplicatedMergeTreeSinkImpl::onFinish() finishDelayedChunk(std::make_shared(zookeeper)); } + +template +void ReplicatedMergeTreeSinkImpl::onCancel() +{ + if (!delayed_chunk) + return; + + for (auto & partition : delayed_chunk->partitions) + { + partition.temp_part.cancel(); + } + + delayed_chunk.reset(); +} + template void ReplicatedMergeTreeSinkImpl::waitForQuorum( const ZooKeeperWithFaultInjectionPtr & zookeeper, diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h index 39623c20584..b1796a35ed2 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h @@ -54,6 +54,8 @@ public: void consume(Chunk chunk) override; void onFinish() override; + void onCancel() override; + String getName() const override { return "ReplicatedMergeTreeSink"; } /// For ATTACHing existing data on filesystem. From 614e899e8d286b5b25f34f0cd94163905dceffeb Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 10 Jul 2024 18:52:09 +0200 Subject: [PATCH 007/129] onCancel is concurrent call --- .../Executors/CompletedPipelineExecutor.cpp | 7 +++ .../Formats/Impl/ArrowBlockInputFormat.h | 2 +- .../Formats/Impl/DWARFBlockInputFormat.h | 2 +- .../Formats/Impl/NativeORCBlockInputFormat.h | 2 +- .../Formats/Impl/ORCBlockInputFormat.h | 2 +- .../Impl/ParallelFormattingOutputFormat.h | 2 +- .../Impl/ParallelParsingInputFormat.cpp | 2 +- .../Formats/Impl/ParallelParsingInputFormat.h | 2 +- .../Formats/Impl/ParquetBlockInputFormat.h | 2 +- .../Formats/Impl/ParquetBlockOutputFormat.cpp | 2 +- .../Formats/Impl/ParquetBlockOutputFormat.h | 2 +- .../Formats/Impl/ParquetMetadataInputFormat.h | 2 +- src/Processors/Formats/LazyOutputFormat.h | 2 +- src/Processors/IProcessor.cpp | 2 +- src/Processors/IProcessor.h | 2 +- src/Processors/Sources/RemoteSource.cpp | 2 +- src/Processors/Sources/RemoteSource.h | 2 +- .../Transforms/AggregatingTransform.cpp | 2 +- src/Server/TCPHandler.cpp | 3 ++ src/Storages/Distributed/DistributedSink.cpp | 2 +- src/Storages/Distributed/DistributedSink.h | 2 +- src/Storages/LiveView/LiveViewEventsSource.h | 2 +- src/Storages/LiveView/LiveViewSource.h | 2 +- .../MergeTree/MergeTreeDataWriter.cpp | 4 ++ src/Storages/MergeTree/MergeTreeSink.cpp | 32 +++++++------ src/Storages/MergeTree/MergeTreeSink.h | 1 - src/Storages/MergeTree/MergeTreeSource.cpp | 2 +- src/Storages/MergeTree/MergeTreeSource.h | 2 +- .../MergeTree/ReplicatedMergeTreeSink.cpp | 48 ++++++++++++------- .../MergeTree/ReplicatedMergeTreeSink.h | 2 - src/Storages/MessageQueueSink.h | 6 ++- .../StorageObjectStorageSink.cpp | 28 ++++------- .../ObjectStorage/StorageObjectStorageSink.h | 8 +--- src/Storages/StorageFile.cpp | 37 ++++---------- src/Storages/StorageURL.cpp | 26 ++++------ src/Storages/StorageURL.h | 6 +-- 36 files changed, 123 insertions(+), 131 deletions(-) diff --git a/src/Processors/Executors/CompletedPipelineExecutor.cpp b/src/Processors/Executors/CompletedPipelineExecutor.cpp index 598a51bf0c7..909e742ffbf 100644 --- a/src/Processors/Executors/CompletedPipelineExecutor.cpp +++ b/src/Processors/Executors/CompletedPipelineExecutor.cpp @@ -3,6 +3,7 @@ #include #include #include +#include "Common/Logger.h" #include #include #include @@ -97,7 +98,10 @@ void CompletedPipelineExecutor::execute() break; if (is_cancelled_callback()) + { + LOG_INFO(getLogger("CompletedPipelineExecutor"), "execute CancelCallback FULLY_CANCELLED"); data->executor->cancel(); + } } if (data->has_exception) @@ -116,7 +120,10 @@ CompletedPipelineExecutor::~CompletedPipelineExecutor() try { if (data && data->executor) + { + LOG_INFO(getLogger("CompletedPipelineExecutor"), "~CompletedPipelineExecutor"); data->executor->cancel(); + } } catch (...) { diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h index cdbc5e57e4e..4fe01d0be12 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h @@ -32,7 +32,7 @@ public: private: Chunk read() override; - void onCancel() override + void onCancelX() override { is_stopped = 1; } diff --git a/src/Processors/Formats/Impl/DWARFBlockInputFormat.h b/src/Processors/Formats/Impl/DWARFBlockInputFormat.h index d8f5fc3d896..6cab5d34994 100644 --- a/src/Processors/Formats/Impl/DWARFBlockInputFormat.h +++ b/src/Processors/Formats/Impl/DWARFBlockInputFormat.h @@ -32,7 +32,7 @@ public: protected: Chunk read() override; - void onCancel() override + void onCancelX() override { is_stopped = 1; } diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h index a3ef9ed4b8f..de9925e3737 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h @@ -64,7 +64,7 @@ public: protected: Chunk read() override; - void onCancel() override { is_stopped = 1; } + void onCancelX() override { is_stopped = 1; } private: void prepareFileReader(); diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.h b/src/Processors/Formats/Impl/ORCBlockInputFormat.h index 34630345849..167436ad4b9 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.h @@ -34,7 +34,7 @@ public: protected: Chunk read() override; - void onCancel() override + void onCancelX() override { is_stopped = 1; } diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 341141dd633..40774fcfbfa 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -122,7 +122,7 @@ public: started_prefix = true; } - void onCancel() override + void onCancelX() override { finishAndWait(); } diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 447adb1ed48..d38a299cb6e 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -176,7 +176,7 @@ Chunk ParallelParsingInputFormat::read() if (background_exception) { lock.unlock(); - onCancel(); + onCancelX(); std::rethrow_exception(background_exception); } diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index 963ccd88def..eed40dc43e5 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -137,7 +137,7 @@ private: Chunk read() final; - void onCancel() final + void onCancelX() final { /* * The format parsers themselves are not being cancelled here, so we'll diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h index 24735ee4371..0123329f026 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h @@ -68,7 +68,7 @@ public: private: Chunk read() override; - void onCancel() override + void onCancelX() override { is_stopped = 1; } diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index a5d334f4f1d..d08c91d286b 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -268,7 +268,7 @@ void ParquetBlockOutputFormat::resetFormatterImpl() staging_bytes = 0; } -void ParquetBlockOutputFormat::onCancel() +void ParquetBlockOutputFormat::onCancelX() { is_stopped = true; } diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h index 422bae5c315..0704ba0ed90 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h @@ -112,7 +112,7 @@ private: void consume(Chunk) override; void finalizeImpl() override; void resetFormatterImpl() override; - void onCancel() override; + void onCancelX() override; void writeRowGroup(std::vector chunks); void writeUsingArrow(std::vector chunks); diff --git a/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h b/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h index ff63d78fa44..35180d202d8 100644 --- a/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h @@ -65,7 +65,7 @@ public: private: Chunk read() override; - void onCancel() override + void onCancelX() override { is_stopped = 1; } diff --git a/src/Processors/Formats/LazyOutputFormat.h b/src/Processors/Formats/LazyOutputFormat.h index 9cf609ed2d7..a245df8234d 100644 --- a/src/Processors/Formats/LazyOutputFormat.h +++ b/src/Processors/Formats/LazyOutputFormat.h @@ -29,7 +29,7 @@ public: void setRowsBeforeLimit(size_t rows_before_limit) override; - void onCancel() override + void onCancelX() override { queue.clearAndFinish(); } diff --git a/src/Processors/IProcessor.cpp b/src/Processors/IProcessor.cpp index f403aca2280..4d95bb5f3e0 100644 --- a/src/Processors/IProcessor.cpp +++ b/src/Processors/IProcessor.cpp @@ -16,7 +16,7 @@ void IProcessor::cancel() if (already_cancelled) return; - onCancel(); + onCancelX(); } String IProcessor::debug() const diff --git a/src/Processors/IProcessor.h b/src/Processors/IProcessor.h index 6f779e7a8d4..9ef7d83eefa 100644 --- a/src/Processors/IProcessor.h +++ b/src/Processors/IProcessor.h @@ -363,7 +363,7 @@ public: virtual void setRowsBeforeLimitCounter(RowsBeforeLimitCounterPtr /* counter */) {} protected: - virtual void onCancel() {} + virtual void onCancelX() {} std::atomic is_cancelled{false}; diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp index 3d7dd3f76b8..44cf26e0b01 100644 --- a/src/Processors/Sources/RemoteSource.cpp +++ b/src/Processors/Sources/RemoteSource.cpp @@ -182,7 +182,7 @@ std::optional RemoteSource::tryGenerate() return chunk; } -void RemoteSource::onCancel() +void RemoteSource::onCancelX() { query_executor->cancel(); } diff --git a/src/Processors/Sources/RemoteSource.h b/src/Processors/Sources/RemoteSource.h index 052567bc261..880eb234bfb 100644 --- a/src/Processors/Sources/RemoteSource.h +++ b/src/Processors/Sources/RemoteSource.h @@ -36,7 +36,7 @@ public: protected: std::optional tryGenerate() override; - void onCancel() override; + void onCancelX() override; private: bool was_query_sent = false; diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index cdbe194cfac..e42c1fd3a8d 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -375,7 +375,7 @@ public: return prepareTwoLevel(); } - void onCancel() override + void onCancelX() override { shared_data->is_cancelled.store(true, std::memory_order_seq_cst); } diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index ac1423f87c1..9fffad26a72 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -556,7 +556,10 @@ void TCPHandler::runImpl() std::scoped_lock lock(out_mutex, task_callback_mutex); if (getQueryCancellationStatus() == CancellationStatus::FULLY_CANCELLED) + { + LOG_INFO(log, "CancelCallback FULLY_CANCELLED"); return true; + } sendProgress(); sendSelectProfileEvents(); diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index e556bda2561..6283594e0d2 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -596,7 +596,7 @@ void DistributedSink::onFinish() } } -void DistributedSink::onCancel() +void DistributedSink::onCancelX() { std::lock_guard lock(execution_mutex); if (pool && !pool->finished()) diff --git a/src/Storages/Distributed/DistributedSink.h b/src/Storages/Distributed/DistributedSink.h index a4c95633595..1dac3eeba6d 100644 --- a/src/Storages/Distributed/DistributedSink.h +++ b/src/Storages/Distributed/DistributedSink.h @@ -53,7 +53,7 @@ public: void onFinish() override; private: - void onCancel() override; + void onCancelX() override; IColumn::Selector createSelector(const Block & source_block) const; diff --git a/src/Storages/LiveView/LiveViewEventsSource.h b/src/Storages/LiveView/LiveViewEventsSource.h index de10a98e1a2..d1ed222c185 100644 --- a/src/Storages/LiveView/LiveViewEventsSource.h +++ b/src/Storages/LiveView/LiveViewEventsSource.h @@ -54,7 +54,7 @@ public: String getName() const override { return "LiveViewEventsSource"; } - void onCancel() override + void onCancelX() override { if (storage->shutdown_called) return; diff --git a/src/Storages/LiveView/LiveViewSource.h b/src/Storages/LiveView/LiveViewSource.h index f8b428fc04d..83589067cf5 100644 --- a/src/Storages/LiveView/LiveViewSource.h +++ b/src/Storages/LiveView/LiveViewSource.h @@ -36,7 +36,7 @@ public: String getName() const override { return "LiveViewSource"; } - void onCancel() override + void onCancelX() override { if (storage->shutdown_called) return; diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 5c8aa32949d..0aaa7909a0f 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -13,6 +13,8 @@ #include #include #include +#include "Common/Logger.h" +#include "Common/logger_useful.h" #include #include #include @@ -182,6 +184,8 @@ void updateTTL( void MergeTreeDataWriter::TemporaryPart::cancel() { + LOG_INFO(getLogger("MergeTreeDataWriter"), "TemporaryPart cancel"); + try { /// An exception context is needed to proper delete write buffers without finalization diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 2e8f6db6868..d2e34665962 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -26,7 +26,23 @@ struct MergeTreeSink::DelayedChunk }; -MergeTreeSink::~MergeTreeSink() = default; +MergeTreeSink::~MergeTreeSink() +{ + size_t addr = delayed_chunk ? size_t(delayed_chunk.get()) : 0; + LOG_INFO(storage.log, "~ReplicatedMergeTreeSinkImpl, delayed_chunk {}, called from {}", addr, StackTrace().toString()); + + if (!delayed_chunk) + return; + + for (auto & partition : delayed_chunk->partitions) + { + partition.temp_part.cancel(); + } + + delayed_chunk.reset(); + + LOG_INFO(storage.log, "~ReplicatedMergeTreeSinkImpl end"); +} MergeTreeSink::MergeTreeSink( StorageMergeTree & storage_, @@ -51,22 +67,10 @@ void MergeTreeSink::onStart() void MergeTreeSink::onFinish() { + chassert(!isCancelled()); finishDelayedChunk(); } -void MergeTreeSink::onCancel() -{ - if (!delayed_chunk) - return; - - for (auto & partition : delayed_chunk->partitions) - { - partition.temp_part.cancel(); - } - - delayed_chunk.reset(); -} - void MergeTreeSink::consume(Chunk chunk) { if (num_blocks_processed > 0) diff --git a/src/Storages/MergeTree/MergeTreeSink.h b/src/Storages/MergeTree/MergeTreeSink.h index cf6715a3415..07ab3850df2 100644 --- a/src/Storages/MergeTree/MergeTreeSink.h +++ b/src/Storages/MergeTree/MergeTreeSink.h @@ -28,7 +28,6 @@ public: void consume(Chunk chunk) override; void onStart() override; void onFinish() override; - void onCancel() override; private: StorageMergeTree & storage; diff --git a/src/Storages/MergeTree/MergeTreeSource.cpp b/src/Storages/MergeTree/MergeTreeSource.cpp index e323b9f9ee7..4070ccf4433 100644 --- a/src/Storages/MergeTree/MergeTreeSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSource.cpp @@ -149,7 +149,7 @@ std::string MergeTreeSource::getName() const return processor->getName(); } -void MergeTreeSource::onCancel() +void MergeTreeSource::onCancelX() { processor->cancel(); } diff --git a/src/Storages/MergeTree/MergeTreeSource.h b/src/Storages/MergeTree/MergeTreeSource.h index fc39b4f9b09..c7092aa26b1 100644 --- a/src/Storages/MergeTree/MergeTreeSource.h +++ b/src/Storages/MergeTree/MergeTreeSource.h @@ -26,7 +26,7 @@ public: protected: std::optional tryGenerate() override; - void onCancel() override; + void onCancelX() override; private: MergeTreeSelectProcessorPtr processor; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 93f82a5a789..6c7ed9bdae0 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -4,11 +4,13 @@ #include #include #include "Common/Exception.h" +#include "Common/StackTrace.h" #include #include #include #include #include +#include "base/defines.h" #include #include #include @@ -151,7 +153,23 @@ ReplicatedMergeTreeSinkImpl::ReplicatedMergeTreeSinkImpl( } template -ReplicatedMergeTreeSinkImpl::~ReplicatedMergeTreeSinkImpl() = default; +ReplicatedMergeTreeSinkImpl::~ReplicatedMergeTreeSinkImpl() +{ + size_t addr = delayed_chunk ? size_t(delayed_chunk.get()) : 0; + LOG_INFO(log, "~ReplicatedMergeTreeSinkImpl, delayed_chunk {}, called from {}", addr, StackTrace().toString()); + + if (!delayed_chunk) + return; + + for (auto & partition : delayed_chunk->partitions) + { + partition.temp_part.cancel(); + } + + delayed_chunk.reset(); + + LOG_INFO(log, "~ReplicatedMergeTreeSinkImpl end"); +} template size_t ReplicatedMergeTreeSinkImpl::checkQuorumPrecondition(const ZooKeeperWithFaultInjectionPtr & zookeeper) @@ -255,6 +273,8 @@ size_t ReplicatedMergeTreeSinkImpl::checkQuorumPrecondition(const template void ReplicatedMergeTreeSinkImpl::consume(Chunk chunk) { + LOG_INFO(log, "consume"); + if (num_blocks_processed > 0) storage.delayInsertOrThrowIfNeeded(&storage.partial_shutdown_event, context, false); @@ -428,6 +448,9 @@ void ReplicatedMergeTreeSinkImpl::consume(Chunk chunk) template<> void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithFaultInjectionPtr & zookeeper) { + size_t addr = delayed_chunk ? size_t(delayed_chunk.get()) : 0; + LOG_INFO(log, "finishDelayedChunk {}", addr); + if (!delayed_chunk) return; @@ -457,16 +480,22 @@ void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithF { auto counters_snapshot = std::make_shared(partition.part_counters.getPartiallyAtomicSnapshot()); PartLog::addNewPart(storage.getContext(), PartLog::PartLogEntry(part, partition.elapsed_ns, counters_snapshot), ExecutionStatus::fromCurrentException("", true)); + + size_t addr1 = delayed_chunk ? size_t(delayed_chunk.get()) : 0; + LOG_INFO(log, "finishDelayedChunk exception, delayed_chunk {}", addr1); throw; } } delayed_chunk.reset(); + + LOG_INFO(log, "finishDelayedChunk end, delayed_chunk {}", bool(delayed_chunk)); } template<> void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithFaultInjectionPtr & zookeeper) { + if (!delayed_chunk) return; @@ -1151,25 +1180,12 @@ void ReplicatedMergeTreeSinkImpl::onStart() template void ReplicatedMergeTreeSinkImpl::onFinish() { + chassert(!isCancelled()); + auto zookeeper = storage.getZooKeeper(); finishDelayedChunk(std::make_shared(zookeeper)); } - -template -void ReplicatedMergeTreeSinkImpl::onCancel() -{ - if (!delayed_chunk) - return; - - for (auto & partition : delayed_chunk->partitions) - { - partition.temp_part.cancel(); - } - - delayed_chunk.reset(); -} - template void ReplicatedMergeTreeSinkImpl::waitForQuorum( const ZooKeeperWithFaultInjectionPtr & zookeeper, diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h index b1796a35ed2..39623c20584 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h @@ -54,8 +54,6 @@ public: void consume(Chunk chunk) override; void onFinish() override; - void onCancel() override; - String getName() const override { return "ReplicatedMergeTreeSink"; } /// For ATTACHing existing data on filesystem. diff --git a/src/Storages/MessageQueueSink.h b/src/Storages/MessageQueueSink.h index b3c1e61734f..38754e9475e 100644 --- a/src/Storages/MessageQueueSink.h +++ b/src/Storages/MessageQueueSink.h @@ -33,13 +33,17 @@ public: const String & storage_name_, const ContextPtr & context_); + ~MessageQueueSink() override + { + onFinish(); + } + String getName() const override { return storage_name + "Sink"; } void consume(Chunk chunk) override; void onStart() override; void onFinish() override; - void onCancel() override { onFinish(); } void onException(std::exception_ptr /* exception */) override { onFinish(); } protected: diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp index f2f6eac333c..3bd0e88ecdb 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp @@ -2,6 +2,7 @@ #include #include #include +#include "base/defines.h" #include namespace DB @@ -41,31 +42,16 @@ StorageObjectStorageSink::StorageObjectStorageSink( void StorageObjectStorageSink::consume(Chunk chunk) { - std::lock_guard lock(cancel_mutex); - if (cancelled) + if (isCancelled()) return; writer->write(getHeader().cloneWithColumns(chunk.detachColumns())); } -void StorageObjectStorageSink::onCancel() -{ - std::lock_guard lock(cancel_mutex); - cancelBuffers(); - releaseBuffers(); - cancelled = true; -} - -void StorageObjectStorageSink::onException(std::exception_ptr) -{ - std::lock_guard lock(cancel_mutex); - cancelBuffers(); - releaseBuffers(); -} - void StorageObjectStorageSink::onFinish() { - std::lock_guard lock(cancel_mutex); + chassert(!isCancelled()); finalizeBuffers(); + releaseBuffers(); } void StorageObjectStorageSink::finalizeBuffers() @@ -119,6 +105,12 @@ PartitionedStorageObjectStorageSink::PartitionedStorageObjectStorageSink( { } +StorageObjectStorageSink::~StorageObjectStorageSink() +{ + if (isCancelled()) + cancelBuffers(); +} + SinkPtr PartitionedStorageObjectStorageSink::createSinkForPartition(const String & partition_id) { auto partition_bucket = replaceWildcards(configuration->getNamespace(), partition_id); diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSink.h b/src/Storages/ObjectStorage/StorageObjectStorageSink.h index e0081193686..578290a92a5 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSink.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageSink.h @@ -18,22 +18,18 @@ public: ContextPtr context, const std::string & blob_path = ""); + ~StorageObjectStorageSink() override; + String getName() const override { return "StorageObjectStorageSink"; } void consume(Chunk chunk) override; - void onCancel() override; - - void onException(std::exception_ptr exception) override; - void onFinish() override; private: const Block sample_block; std::unique_ptr write_buf; OutputFormatPtr writer; - bool cancelled = false; - std::mutex cancel_mutex; void finalizeBuffers(); void releaseBuffers(); diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 7f39ff615f0..855667b1cc6 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -52,6 +52,7 @@ #include #include #include +#include "base/defines.h" #include #include @@ -1747,6 +1748,12 @@ public: initialize(); } + ~StorageFileSink() override + { + if (isCancelled()) + cancelBuffers(); + } + void initialize() { std::unique_ptr naked_buffer; @@ -1780,37 +1787,14 @@ public: void consume(Chunk chunk) override { - std::lock_guard cancel_lock(cancel_mutex); - if (cancelled) + if (isCancelled()) return; writer->write(getHeader().cloneWithColumns(chunk.detachColumns())); } - void onCancel() override - { - std::lock_guard cancel_lock(cancel_mutex); - cancelBuffers(); - releaseBuffers(); - cancelled = true; - } - - void onException(std::exception_ptr exception) override - { - std::lock_guard cancel_lock(cancel_mutex); - try - { - std::rethrow_exception(exception); - } - catch (...) - { - /// An exception context is needed to proper delete write buffers without finalization - releaseBuffers(); - } - } - void onFinish() override { - std::lock_guard cancel_lock(cancel_mutex); + chassert(!isCancelled()); finalizeBuffers(); } @@ -1865,9 +1849,6 @@ private: int flags; std::unique_lock lock; - - std::mutex cancel_mutex; - bool cancelled = false; }; class PartitionedStorageFileSink : public PartitionedSink diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 895da028fc2..6f600393263 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -567,31 +567,15 @@ StorageURLSink::StorageURLSink( void StorageURLSink::consume(Chunk chunk) { - std::lock_guard lock(cancel_mutex); - if (cancelled) + if (isCancelled()) return; writer->write(getHeader().cloneWithColumns(chunk.detachColumns())); } -void StorageURLSink::onCancel() -{ - std::lock_guard lock(cancel_mutex); - cancelBuffers(); - releaseBuffers(); - cancelled = true; -} - -void StorageURLSink::onException(std::exception_ptr) -{ - std::lock_guard lock(cancel_mutex); - cancelBuffers(); - releaseBuffers(); -} - void StorageURLSink::onFinish() { - std::lock_guard lock(cancel_mutex); finalizeBuffers(); + releaseBuffers(); } void StorageURLSink::finalizeBuffers() @@ -1396,6 +1380,11 @@ StorageURLWithFailover::StorageURLWithFailover( } } +StorageURLSink::~StorageURLSink() +{ + if (isCancelled()) + cancelBuffers(); +} FormatSettings StorageURL::getFormatSettingsFromArgs(const StorageFactory::Arguments & args) { @@ -1586,4 +1575,5 @@ void registerStorageURL(StorageFactory & factory) .source_access_type = AccessType::URL, }); } + } diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index fa7cc6eeeef..12a49d3dff5 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -250,10 +250,10 @@ public: const HTTPHeaderEntries & headers = {}, const String & method = Poco::Net::HTTPRequest::HTTP_POST); + ~StorageURLSink() override; + std::string getName() const override { return "StorageURLSink"; } void consume(Chunk chunk) override; - void onCancel() override; - void onException(std::exception_ptr exception) override; void onFinish() override; private: @@ -263,8 +263,6 @@ private: std::unique_ptr write_buf; OutputFormatPtr writer; - std::mutex cancel_mutex; - bool cancelled = false; }; class StorageURL : public IStorageURLBase From 34e54fd089b5c4b4892e80bd426a9409b31f29c9 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Fri, 12 Jul 2024 13:58:41 +0200 Subject: [PATCH 008/129] fix test test_upload_s3_fail_upload_part_when_multi_part_upload --- tests/integration/test_checking_s3_blobs_paranoid/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py index 1ed70e20b79..22a5cd525f3 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/test.py +++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py @@ -197,7 +197,7 @@ def test_upload_s3_fail_upload_part_when_multi_part_upload( ) assert create_multipart == 1 assert upload_parts >= 2 - assert s3_errors >= 2 + assert s3_errors == 1 @pytest.mark.parametrize( From 344e5b716d49eda59783b6fff85757e6a5b6e98f Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Mon, 15 Jul 2024 17:41:32 +0200 Subject: [PATCH 009/129] cancel method is noexcept --- src/Common/ConcurrentBoundedQueue.h | 2 +- .../Executors/CompletedPipelineExecutor.cpp | 2 -- .../Formats/Impl/ArrowBlockInputFormat.h | 2 +- .../Formats/Impl/DWARFBlockInputFormat.h | 2 +- .../Formats/Impl/NativeORCBlockInputFormat.h | 2 +- .../Formats/Impl/ORCBlockInputFormat.h | 2 +- .../Impl/ParallelFormattingOutputFormat.cpp | 2 +- .../Impl/ParallelFormattingOutputFormat.h | 4 ++-- .../Impl/ParallelParsingInputFormat.cpp | 2 +- .../Formats/Impl/ParallelParsingInputFormat.h | 4 ++-- .../Formats/Impl/ParquetBlockInputFormat.h | 2 +- .../Formats/Impl/ParquetBlockOutputFormat.cpp | 2 +- .../Formats/Impl/ParquetBlockOutputFormat.h | 2 +- .../Formats/Impl/ParquetMetadataInputFormat.h | 2 +- src/Processors/Formats/LazyOutputFormat.h | 2 +- src/Processors/IProcessor.cpp | 4 ++-- src/Processors/IProcessor.h | 4 ++-- src/Processors/Sources/RemoteSource.cpp | 13 ++++++++++-- src/Processors/Sources/RemoteSource.h | 2 +- .../Transforms/AggregatingTransform.cpp | 2 +- src/Server/TCPHandler.cpp | 1 - src/Storages/Distributed/DistributedSink.cpp | 20 +++++++++++++++---- src/Storages/Distributed/DistributedSink.h | 2 +- src/Storages/LiveView/LiveViewEventsSource.h | 2 +- src/Storages/LiveView/LiveViewSource.h | 2 +- .../MergeTree/MergeTreeSelectProcessor.h | 2 +- src/Storages/MergeTree/MergeTreeSink.cpp | 5 ----- src/Storages/MergeTree/MergeTreeSource.cpp | 2 +- src/Storages/MergeTree/MergeTreeSource.h | 2 +- .../MergeTree/ReplicatedMergeTreeSink.cpp | 16 --------------- src/Storages/MessageQueueSink.cpp | 13 ++++++++++++ src/Storages/MessageQueueSink.h | 6 +----- .../StorageObjectStorageSink.cpp | 2 +- 33 files changed, 70 insertions(+), 64 deletions(-) diff --git a/src/Common/ConcurrentBoundedQueue.h b/src/Common/ConcurrentBoundedQueue.h index 16b9488c98d..a830ae157a5 100644 --- a/src/Common/ConcurrentBoundedQueue.h +++ b/src/Common/ConcurrentBoundedQueue.h @@ -243,7 +243,7 @@ public: } /// Clear and finish queue - void clearAndFinish() + void clearAndFinish() noexcept { { std::lock_guard lock(queue_mutex); diff --git a/src/Processors/Executors/CompletedPipelineExecutor.cpp b/src/Processors/Executors/CompletedPipelineExecutor.cpp index 909e742ffbf..1eeee896ede 100644 --- a/src/Processors/Executors/CompletedPipelineExecutor.cpp +++ b/src/Processors/Executors/CompletedPipelineExecutor.cpp @@ -99,7 +99,6 @@ void CompletedPipelineExecutor::execute() if (is_cancelled_callback()) { - LOG_INFO(getLogger("CompletedPipelineExecutor"), "execute CancelCallback FULLY_CANCELLED"); data->executor->cancel(); } } @@ -121,7 +120,6 @@ CompletedPipelineExecutor::~CompletedPipelineExecutor() { if (data && data->executor) { - LOG_INFO(getLogger("CompletedPipelineExecutor"), "~CompletedPipelineExecutor"); data->executor->cancel(); } } diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h index 4fe01d0be12..cb74a9dd93e 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h @@ -32,7 +32,7 @@ public: private: Chunk read() override; - void onCancelX() override + void onCancel() noexcept override { is_stopped = 1; } diff --git a/src/Processors/Formats/Impl/DWARFBlockInputFormat.h b/src/Processors/Formats/Impl/DWARFBlockInputFormat.h index 6cab5d34994..2d94d166708 100644 --- a/src/Processors/Formats/Impl/DWARFBlockInputFormat.h +++ b/src/Processors/Formats/Impl/DWARFBlockInputFormat.h @@ -32,7 +32,7 @@ public: protected: Chunk read() override; - void onCancelX() override + void onCancel() noexcept override { is_stopped = 1; } diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h index de9925e3737..e4f2ef9ebe3 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h @@ -64,7 +64,7 @@ public: protected: Chunk read() override; - void onCancelX() override { is_stopped = 1; } + void onCancel() noexcept override { is_stopped = 1; } private: void prepareFileReader(); diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.h b/src/Processors/Formats/Impl/ORCBlockInputFormat.h index 167436ad4b9..85f1636d3dc 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.h @@ -34,7 +34,7 @@ public: protected: Chunk read() override; - void onCancelX() override + void onCancel() noexcept override { is_stopped = 1; } diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp index b2871310be5..5d404d493a6 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp @@ -96,7 +96,7 @@ namespace DB } - void ParallelFormattingOutputFormat::finishAndWait() + void ParallelFormattingOutputFormat::finishAndWait() noexcept { emergency_stop = true; diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 40774fcfbfa..54617c77f86 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -122,7 +122,7 @@ public: started_prefix = true; } - void onCancelX() override + void onCancel() noexcept override { finishAndWait(); } @@ -268,7 +268,7 @@ private: bool collected_suffix = false; bool collected_finalize = false; - void finishAndWait(); + void finishAndWait() noexcept; void onBackgroundException() { diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index d38a299cb6e..447adb1ed48 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -176,7 +176,7 @@ Chunk ParallelParsingInputFormat::read() if (background_exception) { lock.unlock(); - onCancelX(); + onCancel(); std::rethrow_exception(background_exception); } diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index eed40dc43e5..b97bf5213e6 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -137,7 +137,7 @@ private: Chunk read() final; - void onCancelX() final + void onCancel() noexcept final { /* * The format parsers themselves are not being cancelled here, so we'll @@ -292,7 +292,7 @@ private: first_parser_finished.wait(); } - void finishAndWait() + void finishAndWait() noexcept { /// Defending concurrent segmentator thread join std::lock_guard finish_and_wait_lock(finish_and_wait_mutex); diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h index 0123329f026..ed528cc077c 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h @@ -68,7 +68,7 @@ public: private: Chunk read() override; - void onCancelX() override + void onCancel() noexcept override { is_stopped = 1; } diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index d08c91d286b..01fb97223f1 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -268,7 +268,7 @@ void ParquetBlockOutputFormat::resetFormatterImpl() staging_bytes = 0; } -void ParquetBlockOutputFormat::onCancelX() +void ParquetBlockOutputFormat::onCancel() noexcept { is_stopped = true; } diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h index 0704ba0ed90..f8f5d2556a5 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h @@ -112,7 +112,7 @@ private: void consume(Chunk) override; void finalizeImpl() override; void resetFormatterImpl() override; - void onCancelX() override; + void onCancel() noexcept override; void writeRowGroup(std::vector chunks); void writeUsingArrow(std::vector chunks); diff --git a/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h b/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h index 35180d202d8..5d2d8989859 100644 --- a/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h @@ -65,7 +65,7 @@ public: private: Chunk read() override; - void onCancelX() override + void onCancel() noexcept override { is_stopped = 1; } diff --git a/src/Processors/Formats/LazyOutputFormat.h b/src/Processors/Formats/LazyOutputFormat.h index a245df8234d..c803ed5dc61 100644 --- a/src/Processors/Formats/LazyOutputFormat.h +++ b/src/Processors/Formats/LazyOutputFormat.h @@ -29,7 +29,7 @@ public: void setRowsBeforeLimit(size_t rows_before_limit) override; - void onCancelX() override + void onCancel() noexcept override { queue.clearAndFinish(); } diff --git a/src/Processors/IProcessor.cpp b/src/Processors/IProcessor.cpp index 4d95bb5f3e0..edb4d662d8b 100644 --- a/src/Processors/IProcessor.cpp +++ b/src/Processors/IProcessor.cpp @@ -9,14 +9,14 @@ namespace DB { -void IProcessor::cancel() +void IProcessor::cancel() noexcept { bool already_cancelled = is_cancelled.exchange(true, std::memory_order_acq_rel); if (already_cancelled) return; - onCancelX(); + onCancel(); } String IProcessor::debug() const diff --git a/src/Processors/IProcessor.h b/src/Processors/IProcessor.h index 5f4d71fbf54..680a446173e 100644 --- a/src/Processors/IProcessor.h +++ b/src/Processors/IProcessor.h @@ -238,7 +238,7 @@ public: /// In case if query was cancelled executor will wait till all processors finish their jobs. /// Generally, there is no reason to check this flag. However, it may be reasonable for long operations (e.g. i/o). bool isCancelled() const { return is_cancelled.load(std::memory_order_acquire); } - void cancel(); + void cancel() noexcept; /// Additional method which is called in case if ports were updated while work() method. /// May be used to stop execution in rare cases. @@ -363,7 +363,7 @@ public: virtual void setRowsBeforeLimitCounter(RowsBeforeLimitCounterPtr /* counter */) {} protected: - virtual void onCancelX() {} + virtual void onCancel() noexcept {} std::atomic is_cancelled{false}; diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp index 44cf26e0b01..42696f9c3ce 100644 --- a/src/Processors/Sources/RemoteSource.cpp +++ b/src/Processors/Sources/RemoteSource.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include namespace DB { @@ -182,9 +184,16 @@ std::optional RemoteSource::tryGenerate() return chunk; } -void RemoteSource::onCancelX() +void RemoteSource::onCancel() noexcept { - query_executor->cancel(); + try + { + query_executor->cancel(); + } + catch (...) + { + tryLogCurrentException(getLogger("RemoteSource"), "Error occurs on cancelation."); + } } void RemoteSource::onUpdatePorts() diff --git a/src/Processors/Sources/RemoteSource.h b/src/Processors/Sources/RemoteSource.h index 880eb234bfb..adf1b8e9fac 100644 --- a/src/Processors/Sources/RemoteSource.h +++ b/src/Processors/Sources/RemoteSource.h @@ -36,7 +36,7 @@ public: protected: std::optional tryGenerate() override; - void onCancelX() override; + void onCancel() noexcept override; private: bool was_query_sent = false; diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index e42c1fd3a8d..870ba84722d 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -375,7 +375,7 @@ public: return prepareTwoLevel(); } - void onCancelX() override + void onCancel() noexcept override { shared_data->is_cancelled.store(true, std::memory_order_seq_cst); } diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index ba12cad7771..e7342ff9b6f 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -557,7 +557,6 @@ void TCPHandler::runImpl() if (getQueryCancellationStatus() == CancellationStatus::FULLY_CANCELLED) { - LOG_INFO(log, "CancelCallback FULLY_CANCELLED"); return true; } diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index 6283594e0d2..d67910a141f 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -596,7 +596,7 @@ void DistributedSink::onFinish() } } -void DistributedSink::onCancelX() +void DistributedSink::onCancel() noexcept { std::lock_guard lock(execution_mutex); if (pool && !pool->finished()) @@ -607,14 +607,26 @@ void DistributedSink::onCancelX() } catch (...) { - tryLogCurrentException(storage.log); + tryLogCurrentException(storage.log, "Error occurs on cancelation."); } } for (auto & shard_jobs : per_shard_jobs) + { for (JobReplica & job : shard_jobs.replicas_jobs) - if (job.executor) - job.executor->cancel(); + { + try + { + if (job.executor) + job.executor->cancel(); + } + catch (...) + { + tryLogCurrentException(storage.log, "Error occurs on cancelation."); + } + } + } + } diff --git a/src/Storages/Distributed/DistributedSink.h b/src/Storages/Distributed/DistributedSink.h index 1dac3eeba6d..65a5eb52787 100644 --- a/src/Storages/Distributed/DistributedSink.h +++ b/src/Storages/Distributed/DistributedSink.h @@ -53,7 +53,7 @@ public: void onFinish() override; private: - void onCancelX() override; + void onCancel() noexcept override; IColumn::Selector createSelector(const Block & source_block) const; diff --git a/src/Storages/LiveView/LiveViewEventsSource.h b/src/Storages/LiveView/LiveViewEventsSource.h index d1ed222c185..4210acbc5bc 100644 --- a/src/Storages/LiveView/LiveViewEventsSource.h +++ b/src/Storages/LiveView/LiveViewEventsSource.h @@ -54,7 +54,7 @@ public: String getName() const override { return "LiveViewEventsSource"; } - void onCancelX() override + void onCancel() noexcept override { if (storage->shutdown_called) return; diff --git a/src/Storages/LiveView/LiveViewSource.h b/src/Storages/LiveView/LiveViewSource.h index 83589067cf5..81dd5620e57 100644 --- a/src/Storages/LiveView/LiveViewSource.h +++ b/src/Storages/LiveView/LiveViewSource.h @@ -36,7 +36,7 @@ public: String getName() const override { return "LiveViewSource"; } - void onCancelX() override + void onCancel() noexcept override { if (storage->shutdown_called) return; diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 03ca30dd5b3..7a9cebbcb2e 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -48,7 +48,7 @@ public: ChunkAndProgress read(); - void cancel() { is_cancelled = true; } + void cancel() noexcept { is_cancelled = true; } const MergeTreeReaderSettings & getSettings() const { return reader_settings; } diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index d2e34665962..210a7057f94 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -28,9 +28,6 @@ struct MergeTreeSink::DelayedChunk MergeTreeSink::~MergeTreeSink() { - size_t addr = delayed_chunk ? size_t(delayed_chunk.get()) : 0; - LOG_INFO(storage.log, "~ReplicatedMergeTreeSinkImpl, delayed_chunk {}, called from {}", addr, StackTrace().toString()); - if (!delayed_chunk) return; @@ -40,8 +37,6 @@ MergeTreeSink::~MergeTreeSink() } delayed_chunk.reset(); - - LOG_INFO(storage.log, "~ReplicatedMergeTreeSinkImpl end"); } MergeTreeSink::MergeTreeSink( diff --git a/src/Storages/MergeTree/MergeTreeSource.cpp b/src/Storages/MergeTree/MergeTreeSource.cpp index 4070ccf4433..380c47723bc 100644 --- a/src/Storages/MergeTree/MergeTreeSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSource.cpp @@ -149,7 +149,7 @@ std::string MergeTreeSource::getName() const return processor->getName(); } -void MergeTreeSource::onCancelX() +void MergeTreeSource::onCancel() noexcept { processor->cancel(); } diff --git a/src/Storages/MergeTree/MergeTreeSource.h b/src/Storages/MergeTree/MergeTreeSource.h index c7092aa26b1..7506af4f9b8 100644 --- a/src/Storages/MergeTree/MergeTreeSource.h +++ b/src/Storages/MergeTree/MergeTreeSource.h @@ -26,7 +26,7 @@ public: protected: std::optional tryGenerate() override; - void onCancelX() override; + void onCancel() noexcept override; private: MergeTreeSelectProcessorPtr processor; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 6c7ed9bdae0..7bfe647fa7f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -155,9 +155,6 @@ ReplicatedMergeTreeSinkImpl::ReplicatedMergeTreeSinkImpl( template ReplicatedMergeTreeSinkImpl::~ReplicatedMergeTreeSinkImpl() { - size_t addr = delayed_chunk ? size_t(delayed_chunk.get()) : 0; - LOG_INFO(log, "~ReplicatedMergeTreeSinkImpl, delayed_chunk {}, called from {}", addr, StackTrace().toString()); - if (!delayed_chunk) return; @@ -167,8 +164,6 @@ ReplicatedMergeTreeSinkImpl::~ReplicatedMergeTreeSinkImpl() } delayed_chunk.reset(); - - LOG_INFO(log, "~ReplicatedMergeTreeSinkImpl end"); } template @@ -273,8 +268,6 @@ size_t ReplicatedMergeTreeSinkImpl::checkQuorumPrecondition(const template void ReplicatedMergeTreeSinkImpl::consume(Chunk chunk) { - LOG_INFO(log, "consume"); - if (num_blocks_processed > 0) storage.delayInsertOrThrowIfNeeded(&storage.partial_shutdown_event, context, false); @@ -448,9 +441,6 @@ void ReplicatedMergeTreeSinkImpl::consume(Chunk chunk) template<> void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithFaultInjectionPtr & zookeeper) { - size_t addr = delayed_chunk ? size_t(delayed_chunk.get()) : 0; - LOG_INFO(log, "finishDelayedChunk {}", addr); - if (!delayed_chunk) return; @@ -480,22 +470,16 @@ void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithF { auto counters_snapshot = std::make_shared(partition.part_counters.getPartiallyAtomicSnapshot()); PartLog::addNewPart(storage.getContext(), PartLog::PartLogEntry(part, partition.elapsed_ns, counters_snapshot), ExecutionStatus::fromCurrentException("", true)); - - size_t addr1 = delayed_chunk ? size_t(delayed_chunk.get()) : 0; - LOG_INFO(log, "finishDelayedChunk exception, delayed_chunk {}", addr1); throw; } } delayed_chunk.reset(); - - LOG_INFO(log, "finishDelayedChunk end, delayed_chunk {}", bool(delayed_chunk)); } template<> void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithFaultInjectionPtr & zookeeper) { - if (!delayed_chunk) return; diff --git a/src/Storages/MessageQueueSink.cpp b/src/Storages/MessageQueueSink.cpp index 4fb81d69070..10617422f40 100644 --- a/src/Storages/MessageQueueSink.cpp +++ b/src/Storages/MessageQueueSink.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include namespace DB @@ -79,4 +80,16 @@ void MessageQueueSink::consume(Chunk chunk) } +void MessageQueueSink::onCancel() noexcept +{ + try + { + onFinish(); + } + catch (...) + { + tryLogCurrentException(getLogger("MessageQueueSink"), "Error occurs on cancelation."); + } +} + } diff --git a/src/Storages/MessageQueueSink.h b/src/Storages/MessageQueueSink.h index 38754e9475e..6964af8cb4d 100644 --- a/src/Storages/MessageQueueSink.h +++ b/src/Storages/MessageQueueSink.h @@ -33,17 +33,13 @@ public: const String & storage_name_, const ContextPtr & context_); - ~MessageQueueSink() override - { - onFinish(); - } - String getName() const override { return storage_name + "Sink"; } void consume(Chunk chunk) override; void onStart() override; void onFinish() override; + void onCancel() noexcept override; void onException(std::exception_ptr /* exception */) override { onFinish(); } protected: diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp index 3bd0e88ecdb..d5813015e13 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp @@ -2,8 +2,8 @@ #include #include #include -#include "base/defines.h" #include +#include namespace DB { From 12101f455b06a3df1bd00b8b070b4c7862d087bf Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Mon, 15 Jul 2024 17:55:02 +0200 Subject: [PATCH 010/129] fix typo --- src/Processors/Sources/RemoteSource.cpp | 2 +- src/Storages/Distributed/DistributedSink.cpp | 4 ++-- src/Storages/MessageQueueSink.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp index 42696f9c3ce..48a6804de9a 100644 --- a/src/Processors/Sources/RemoteSource.cpp +++ b/src/Processors/Sources/RemoteSource.cpp @@ -192,7 +192,7 @@ void RemoteSource::onCancel() noexcept } catch (...) { - tryLogCurrentException(getLogger("RemoteSource"), "Error occurs on cancelation."); + tryLogCurrentException(getLogger("RemoteSource"), "Error occurs on cancellation."); } } diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index fa4ba01a37c..197905c0849 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -608,7 +608,7 @@ void DistributedSink::onCancel() noexcept } catch (...) { - tryLogCurrentException(storage.log, "Error occurs on cancelation."); + tryLogCurrentException(storage.log, "Error occurs on cancellation."); } } @@ -623,7 +623,7 @@ void DistributedSink::onCancel() noexcept } catch (...) { - tryLogCurrentException(storage.log, "Error occurs on cancelation."); + tryLogCurrentException(storage.log, "Error occurs on cancellation."); } } } diff --git a/src/Storages/MessageQueueSink.cpp b/src/Storages/MessageQueueSink.cpp index 10617422f40..9cddb2e7ce8 100644 --- a/src/Storages/MessageQueueSink.cpp +++ b/src/Storages/MessageQueueSink.cpp @@ -88,7 +88,7 @@ void MessageQueueSink::onCancel() noexcept } catch (...) { - tryLogCurrentException(getLogger("MessageQueueSink"), "Error occurs on cancelation."); + tryLogCurrentException(getLogger("MessageQueueSink"), "Error occurs on cancellation."); } } From 47573f1ae158faabd6ac0a62598c907c85fbd4c1 Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Wed, 17 Jul 2024 20:44:44 +0000 Subject: [PATCH 011/129] add options --- src/Core/SettingsEnums.h | 9 +++++++++ src/Storages/MergeTree/MergeTreeSettings.h | 1 + 2 files changed, 10 insertions(+) diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index f6d9593ca56..ac3264fe041 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -315,6 +315,15 @@ enum class LightweightMutationProjectionMode : uint8_t DECLARE_SETTING_ENUM(LightweightMutationProjectionMode) +enum class DeduplicateMergeProjectionMode : uint8_t +{ + THROW, + DROP, + REBUILD, +}; + +DECLARE_SETTING_ENUM(DeduplicateMergeProjectionMode) + DECLARE_SETTING_ENUM(LocalFSReadMethod) enum class ObjectStorageQueueMode : uint8_t diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index f5ada81cf55..166a18a7bab 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -215,6 +215,7 @@ struct Settings; M(Float, primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns, 0.9f, "If the value of a column of the primary key in data part changes at least in this ratio of times, skip loading next columns in memory. This allows to save memory usage by not loading useless columns of the primary key.", 0) \ /** Projection settings. */ \ M(UInt64, max_projections, 25, "The maximum number of merge tree projections.", 0) \ + M(DeduplicateMergeProjectionMode, deduplicate_merge_projection_mode, DeduplicateMergeProjectionMode::THROW, "Whether to allow create projection for the table with non-classic MergeTree, if allowed, what is the next action.", 0) \ #define MAKE_OBSOLETE_MERGE_TREE_SETTING(M, TYPE, NAME, DEFAULT) \ M(TYPE, NAME, DEFAULT, "Obsolete setting, does nothing.", BaseSettingsHelpers::Flags::OBSOLETE) From 76904d4ae645aaf9a4cfce938be23a0c60a03fb6 Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Thu, 18 Jul 2024 21:08:35 +0000 Subject: [PATCH 012/129] support throw option --- src/Core/SettingsEnums.cpp | 5 +++ src/Interpreters/InterpreterCreateQuery.cpp | 21 ++++++++++++ ...ojection_merge_special_mergetree.reference | 0 ...206_projection_merge_special_mergetree.sql | 32 +++++++++++++++++++ 4 files changed, 58 insertions(+) create mode 100644 tests/queries/0_stateless/03206_projection_merge_special_mergetree.reference create mode 100644 tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index 82e7d6db410..74b6c793849 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -177,6 +177,11 @@ IMPLEMENT_SETTING_ENUM(LightweightMutationProjectionMode, ErrorCodes::BAD_ARGUME {{"throw", LightweightMutationProjectionMode::THROW}, {"drop", LightweightMutationProjectionMode::DROP}}) +IMPLEMENT_SETTING_ENUM(DeduplicateMergeProjectionMode, ErrorCodes::BAD_ARGUMENTS, + {{"throw", DeduplicateMergeProjectionMode::THROW}, + {"drop", DeduplicateMergeProjectionMode::DROP}, + {"rebuild", DeduplicateMergeProjectionMode::THROW}}) + IMPLEMENT_SETTING_AUTO_ENUM(LocalFSReadMethod, ErrorCodes::BAD_ARGUMENTS) IMPLEMENT_SETTING_ENUM(ParquetVersion, ErrorCodes::BAD_ARGUMENTS, diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index ea10ad59db4..df5ec4525eb 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1281,6 +1281,27 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) /// Set and retrieve list of columns, indices and constraints. Set table engine if needed. Rewrite query in canonical way. TableProperties properties = getTablePropertiesAndNormalizeCreateQuery(create, mode); + /// Projection is only supported in (Replictaed)MergeTree. + if (std::string_view engine_name(create.storage->engine->name); + !properties.projections.empty() && engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree") + { + bool projection_support = false; + if (auto * setting = create.storage->settings; setting != nullptr) + { + for (const auto & change : setting->changes) + { + if (change.name == "deduplicate_merge_projection_mode" && change.value != Field("throw")) + { + projection_support = true; + break; + } + } + } + if (!projection_support) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "Projection is only supported in (Replictaed)MergeTree. Consider drop or rebuild option of deduplicate_merge_projection_mode."); + } + /// Check type compatible for materialized dest table and select columns if (create.select && create.is_materialized_view && create.to_table_id && mode <= LoadingStrictnessLevel::CREATE) { diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.reference b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql new file mode 100644 index 00000000000..6b5e516ad21 --- /dev/null +++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql @@ -0,0 +1,32 @@ +DROP TABLE IF EXISTS tp; + +CREATE TABLE tp ( + type Int32, + eventcnt UInt64, + PROJECTION p (select sum(eventcnt), type group by type) +) engine = ReplacingMergeTree order by type; -- { serverError SUPPORT_IS_DISABLED } + +CREATE TABLE tp ( + type Int32, + eventcnt UInt64, + PROJECTION p (select sum(eventcnt), type group by type) +) engine = ReplacingMergeTree order by type +SETTINGS deduplicate_merge_projection_mode = 'throw'; -- { serverError SUPPORT_IS_DISABLED } + +CREATE TABLE tp ( + type Int32, + eventcnt UInt64, + PROJECTION p (select sum(eventcnt), type group by type) +) engine = ReplacingMergeTree order by type +SETTINGS deduplicate_merge_projection_mode = 'drop'; + +DROP TABLE tp; + +CREATE TABLE tp ( + type Int32, + eventcnt UInt64, + PROJECTION p (select sum(eventcnt), type group by type) +) engine = ReplacingMergeTree order by type +SETTINGS deduplicate_merge_projection_mode = 'rebuild'; + +DROP TABLE tp; From 06594935f0e2dd9fc61882c7e643677de474fd7c Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Fri, 19 Jul 2024 01:34:03 +0000 Subject: [PATCH 013/129] forbid optimize deduplicate --- src/Interpreters/InterpreterOptimizeQuery.cpp | 5 +++++ .../03206_projection_merge_special_mergetree.sql | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/src/Interpreters/InterpreterOptimizeQuery.cpp b/src/Interpreters/InterpreterOptimizeQuery.cpp index 907a01b0432..8d1ac3455b7 100644 --- a/src/Interpreters/InterpreterOptimizeQuery.cpp +++ b/src/Interpreters/InterpreterOptimizeQuery.cpp @@ -20,6 +20,7 @@ namespace DB namespace ErrorCodes { extern const int THERE_IS_NO_COLUMN; + extern const int NOT_IMPLEMENTED; } @@ -42,6 +43,10 @@ BlockIO InterpreterOptimizeQuery::execute() auto metadata_snapshot = table->getInMemoryMetadataPtr(); auto storage_snapshot = table->getStorageSnapshot(metadata_snapshot, getContext()); + /// Don't allow OPTIMIZE DEDUPLICATE for all engines with projections. + if (ast.deduplicate && !metadata_snapshot->projections.empty()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "DEDUPLICATE with projections are not supported yet"); + // Empty list of names means we deduplicate by all columns, but user can explicitly state which columns to use. Names column_names; if (ast.deduplicate_by_columns) diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql index 6b5e516ad21..06fb9a30aca 100644 --- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql +++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql @@ -30,3 +30,12 @@ CREATE TABLE tp ( SETTINGS deduplicate_merge_projection_mode = 'rebuild'; DROP TABLE tp; + + +-- don't allow OPTIMIZE DEDUPLICATE for all engines with projections +CREATE TABLE test ( + a INT PRIMARY KEY, + PROJECTION p (SELECT * ORDER BY a) +) engine = MergeTree; + +OPTIMIZE TABLE test DEDUPLICATE; -- { serverError NOT_IMPLEMENTED } From 212b81da533d18a6b9f02c66b34a2161ff1e5d71 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Fri, 19 Jul 2024 14:33:20 +0200 Subject: [PATCH 014/129] rm debug prints, fix headers --- src/Processors/Executors/CompletedPipelineExecutor.cpp | 1 - src/Storages/MergeTree/MergeTreeDataWriter.cpp | 4 ---- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 1 - 3 files changed, 6 deletions(-) diff --git a/src/Processors/Executors/CompletedPipelineExecutor.cpp b/src/Processors/Executors/CompletedPipelineExecutor.cpp index 1eeee896ede..888835c9beb 100644 --- a/src/Processors/Executors/CompletedPipelineExecutor.cpp +++ b/src/Processors/Executors/CompletedPipelineExecutor.cpp @@ -3,7 +3,6 @@ #include #include #include -#include "Common/Logger.h" #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 97335b601f9..73244b714bf 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -14,8 +14,6 @@ #include #include #include -#include "Common/Logger.h" -#include "Common/logger_useful.h" #include #include #include @@ -186,8 +184,6 @@ void updateTTL( void MergeTreeDataWriter::TemporaryPart::cancel() { - LOG_INFO(getLogger("MergeTreeDataWriter"), "TemporaryPart cancel"); - try { /// An exception context is needed to proper delete write buffers without finalization diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index f6d6b8cb7a3..17662f92035 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include From 57c1d7a1011f96cea21ca66a3064b7481f8ce40b Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 19 Jul 2024 12:36:57 +0000 Subject: [PATCH 015/129] fix filling of empty Nested --- src/DataTypes/IDataType.cpp | 4 +- src/DataTypes/ObjectUtils.cpp | 31 ++++++++++ src/DataTypes/ObjectUtils.h | 3 + src/DataTypes/Serializations/ISerialization.h | 3 +- src/Interpreters/inplaceBlockConversions.cpp | 57 ++++++++++++++++--- src/Storages/MergeTree/IMergeTreeReader.cpp | 7 ++- src/Storages/MergeTree/IMergeTreeReader.h | 3 + 7 files changed, 95 insertions(+), 13 deletions(-) diff --git a/src/DataTypes/IDataType.cpp b/src/DataTypes/IDataType.cpp index 1cb64b65d3a..824bc6e33b0 100644 --- a/src/DataTypes/IDataType.cpp +++ b/src/DataTypes/IDataType.cpp @@ -90,7 +90,9 @@ void IDataType::forEachSubcolumn( { auto name = ISerialization::getSubcolumnNameForStream(subpath, prefix_len); auto subdata = ISerialization::createFromPath(subpath, prefix_len); - callback(subpath, name, subdata); + auto path_copy = subpath; + path_copy.resize(prefix_len); + callback(path_copy, name, subdata); } subpath[i].visited = true; } diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index 1d525e5987f..356e609e77a 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,36 @@ DataTypePtr getBaseTypeOfArray(const DataTypePtr & type) return last_array ? last_array->getNestedType() : type; } +DataTypePtr getBaseTypeOfArray(DataTypePtr type, const Names & tuple_elements) +{ + auto it = tuple_elements.begin(); + while (true) + { + if (const auto * type_array = typeid_cast(type.get())) + { + type = type_array->getNestedType(); + } + else if (const auto * type_tuple = typeid_cast(type.get())) + { + if (it == tuple_elements.end()) + break; + + auto pos = type_tuple->tryGetPositionByName(*it); + if (!pos) + break; + + ++it; + type = type_tuple->getElement(*pos); + } + else + { + break; + } + } + + return type; +} + ColumnPtr getBaseColumnOfArray(const ColumnPtr & column) { /// Get raw pointers to avoid extra copying of column pointers. diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index 6599d8adef1..21e5c3b2f59 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -27,6 +27,9 @@ size_t getNumberOfDimensions(const IColumn & column); /// Returns type of scalars of Array of arbitrary dimensions. DataTypePtr getBaseTypeOfArray(const DataTypePtr & type); +/// The same as above but takes into account Tuples of Nested. +DataTypePtr getBaseTypeOfArray(DataTypePtr type, const Names & tuple_elements); + /// Returns Array type with requested scalar type and number of dimensions. DataTypePtr createArrayOfType(DataTypePtr type, size_t num_dimensions); diff --git a/src/DataTypes/Serializations/ISerialization.h b/src/DataTypes/Serializations/ISerialization.h index 255dbbfadd2..5d0bf60c59f 100644 --- a/src/DataTypes/Serializations/ISerialization.h +++ b/src/DataTypes/Serializations/ISerialization.h @@ -195,7 +195,7 @@ public: /// Types of substreams that can have arbitrary name. static const std::set named_types; - Type type; + Type type = Type::Regular; /// The name of a variant element type. String variant_element_name; @@ -212,6 +212,7 @@ public: /// Flag, that may help to traverse substream paths. mutable bool visited = false; + Substream() = default; Substream(Type type_) : type(type_) {} /// NOLINT String toString() const; }; diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp index f7d8a2a2daf..ce3f25d16f8 100644 --- a/src/Interpreters/inplaceBlockConversions.cpp +++ b/src/Interpreters/inplaceBlockConversions.cpp @@ -283,6 +283,9 @@ static ColumnPtr createColumnWithDefaultValue(const IDataType & data_type, const { auto column = data_type.createColumnConstWithDefaultValue(num_rows); + /// We must turn a constant column into a full column because the interpreter could infer + /// that it is constant everywhere but in some blocks (from other parts) it can be a full column. + if (subcolumn_name.empty()) return column->convertToFullColumnIfConst(); @@ -293,6 +296,35 @@ static ColumnPtr createColumnWithDefaultValue(const IDataType & data_type, const return ColumnConst::create(std::move(column), num_rows)->convertToFullColumnIfConst(); } +static bool hasDefault(const StorageMetadataPtr & metadata_snapshot, const NameAndTypePair & column) +{ + if (!metadata_snapshot) + return false; + + const auto & columns = metadata_snapshot->getColumns(); + if (columns.has(column.name)) + return columns.hasDefault(column.name); + + auto name_in_storage = column.getNameInStorage(); + return columns.hasDefault(name_in_storage); +} + +static String removeTupleElementsFromSubcolumn(String subcolumn_name, const Names & tuple_elements) +{ + subcolumn_name += "."; + for (const auto & elem : tuple_elements) + { + auto pos = subcolumn_name.find(elem + "."); + if (pos != std::string::npos) + subcolumn_name.erase(pos, elem.size()); + } + + if (subcolumn_name.ends_with(".")) + subcolumn_name.pop_back(); + + return subcolumn_name; +} + void fillMissingColumns( Columns & res_columns, size_t num_rows, @@ -321,10 +353,8 @@ void fillMissingColumns( if (res_columns[i] && partially_read_columns.contains(requested_column->name)) res_columns[i] = nullptr; - if (res_columns[i]) - continue; - - if (metadata_snapshot && metadata_snapshot->getColumns().hasDefault(requested_column->getNameInStorage())) + /// Nothing to fill or default should be filled in evaluateMissingDefaults + if (res_columns[i] || hasDefault(metadata_snapshot, *requested_column)) continue; std::vector current_offsets; @@ -365,19 +395,30 @@ void fillMissingColumns( if (!current_offsets.empty()) { + + Names tuple_elements; + auto serialization = IDataType::getSerialization(*requested_column); + + IDataType::forEachSubcolumn([&](const auto & path, const auto &, const auto &) + { + if (path.back().type == ISerialization::Substream::TupleElement) + tuple_elements.push_back(path.back().name_of_substream); + }, ISerialization::SubstreamData(serialization)); + size_t num_empty_dimensions = num_dimensions - current_offsets.size(); - auto scalar_type = createArrayOfType(getBaseTypeOfArray(requested_column->getTypeInStorage()), num_empty_dimensions); + auto base_type = getBaseTypeOfArray(requested_column->getTypeInStorage(), tuple_elements); + auto scalar_type = createArrayOfType(base_type, num_empty_dimensions); size_t data_size = assert_cast(*current_offsets.back()).getData().back(); - res_columns[i] = createColumnWithDefaultValue(*scalar_type, requested_column->getSubcolumnName(), data_size); + auto subcolumn_name = removeTupleElementsFromSubcolumn(requested_column->getSubcolumnName(), tuple_elements); + + res_columns[i] = createColumnWithDefaultValue(*scalar_type, subcolumn_name, data_size); for (auto it = current_offsets.rbegin(); it != current_offsets.rend(); ++it) res_columns[i] = ColumnArray::create(res_columns[i], *it); } else { - /// We must turn a constant column into a full column because the interpreter could infer - /// that it is constant everywhere but in some blocks (from other parts) it can be a full column. res_columns[i] = createColumnWithDefaultValue(*requested_column->getTypeInStorage(), requested_column->getSubcolumnName(), num_rows); } } diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index aff1001163e..5f36e4c7c13 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -44,6 +44,7 @@ IMergeTreeReader::IMergeTreeReader( , alter_conversions(data_part_info_for_read->getAlterConversions()) /// For wide parts convert plain arrays of Nested to subcolumns /// to allow to use shared offset column from cache. + , original_requested_columns(columns_) , requested_columns(data_part_info_for_read->isWidePart() ? Nested::convertToSubcolumns(columns_) : columns_) @@ -139,7 +140,7 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns { try { - size_t num_columns = requested_columns.size(); + size_t num_columns = original_requested_columns.size(); if (res_columns.size() != num_columns) throw Exception(ErrorCodes::LOGICAL_ERROR, "invalid number of columns passed to MergeTreeReader::fillMissingColumns. " @@ -151,7 +152,7 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns /// Convert columns list to block. And convert subcolumns to full columns. /// TODO: rewrite with columns interface. It will be possible after changes in ExpressionActions. - auto it = requested_columns.begin(); + auto it = original_requested_columns.begin(); for (size_t pos = 0; pos < num_columns; ++pos, ++it) { auto name_in_storage = it->getNameInStorage(); @@ -178,7 +179,7 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns } /// Move columns from block. - it = requested_columns.begin(); + it = original_requested_columns.begin(); for (size_t pos = 0; pos < num_columns; ++pos, ++it) { auto name_in_storage = it->getNameInStorage(); diff --git a/src/Storages/MergeTree/IMergeTreeReader.h b/src/Storages/MergeTree/IMergeTreeReader.h index a1ec0339fd6..d799ce57b40 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.h +++ b/src/Storages/MergeTree/IMergeTreeReader.h @@ -112,6 +112,9 @@ protected: private: /// Columns that are requested to read. + NamesAndTypesList original_requested_columns; + + /// The same as above but with converted Arrays to subcolumns of Nested. NamesAndTypesList requested_columns; /// Actual columns description in part. From 189b3d306fc0e488010564384b193412acd0358b Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 19 Jul 2024 13:17:17 +0000 Subject: [PATCH 016/129] fix tests --- src/Storages/MergeTree/MergeTreeReaderCompact.cpp | 2 +- .../0_stateless/02026_describe_include_subcolumns.reference | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index fde9dafffb8..ff0311dc1ca 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -101,7 +101,7 @@ NameAndTypePair MergeTreeReaderCompact::getColumnConvertedToSubcolumnOfNested(co if (!storage_columns_with_collected_nested) { - auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects(); + auto options = GetColumnsOptions(GetColumnsOptions::All).withExtendedObjects(); auto storage_columns_list = Nested::collect(storage_snapshot->getColumns(options)); storage_columns_with_collected_nested = ColumnsDescription(std::move(storage_columns_list)); } diff --git a/tests/queries/0_stateless/02026_describe_include_subcolumns.reference b/tests/queries/0_stateless/02026_describe_include_subcolumns.reference index dec65f62748..62efafceaad 100644 --- a/tests/queries/0_stateless/02026_describe_include_subcolumns.reference +++ b/tests/queries/0_stateless/02026_describe_include_subcolumns.reference @@ -26,7 +26,7 @@ 10. │ t.s │ String │ │ │ │ ZSTD(1) │ │ 1 │ 11. │ t.a │ Array(Tuple( a UInt32, - b UInt32)) │ │ │ │ │ │ 1 │ + b UInt32)) │ │ │ │ ZSTD(1) │ │ 1 │ 12. │ t.a.size0 │ UInt64 │ │ │ │ │ │ 1 │ 13. │ t.a.a │ Array(UInt32) │ │ │ │ ZSTD(1) │ │ 1 │ 14. │ t.a.b │ Array(UInt32) │ │ │ │ ZSTD(1) │ │ 1 │ From 277dbfa0574b567241d169494a459e6f2b04d5e6 Mon Sep 17 00:00:00 2001 From: "Zhukova, Maria" Date: Fri, 19 Jul 2024 13:13:42 -0700 Subject: [PATCH 017/129] update QPL to 1.6.0 + missing header fix --- contrib/qpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/qpl b/contrib/qpl index d4715e0e798..c2ced94c53c 160000 --- a/contrib/qpl +++ b/contrib/qpl @@ -1 +1 @@ -Subproject commit d4715e0e79896b85612158e135ee1a85f3b3e04d +Subproject commit c2ced94c53c1ee22191201a59878e9280bc9b9b8 From 1347bc32187b694148657459c0b745c45cb92a8d Mon Sep 17 00:00:00 2001 From: "Zhukova, Maria" Date: Fri, 19 Jul 2024 13:32:18 -0700 Subject: [PATCH 018/129] update qpl-cmake to reflect changes in QPL 1.5.0-1.6.0 --- contrib/qpl-cmake/CMakeLists.txt | 92 ++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 35 deletions(-) diff --git a/contrib/qpl-cmake/CMakeLists.txt b/contrib/qpl-cmake/CMakeLists.txt index 7a84048e16b..b2f263252c2 100644 --- a/contrib/qpl-cmake/CMakeLists.txt +++ b/contrib/qpl-cmake/CMakeLists.txt @@ -4,7 +4,6 @@ set (QPL_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl") set (QPL_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl/sources") set (QPL_BINARY_DIR "${ClickHouse_BINARY_DIR}/build/contrib/qpl") set (EFFICIENT_WAIT OFF) -set (BLOCK_ON_FAULT ON) set (LOG_HW_INIT OFF) set (SANITIZE_MEMORY OFF) set (SANITIZE_THREADS OFF) @@ -16,16 +15,18 @@ function(GetLibraryVersion _content _outputVar) SET(${_outputVar} ${CMAKE_MATCH_1} PARENT_SCOPE) endfunction() -set (QPL_VERSION 1.2.0) +set (QPL_VERSION 1.6.0) message(STATUS "Intel QPL version: ${QPL_VERSION}") -# There are 5 source subdirectories under $QPL_SRC_DIR: isal, c_api, core-sw, middle-layer, c_api. -# Generate 8 library targets: middle_layer_lib, isal, isal_asm, qplcore_px, qplcore_avx512, qplcore_sw_dispatcher, core_iaa, middle_layer_lib. +# There are 5 source subdirectories under $QPL_SRC_DIR: c_api, core-iaa, core-sw, middle-layer and isal. +# Generate 8 library targets: qpl_c_api, core_iaa, qplcore_px, qplcore_avx512, qplcore_sw_dispatcher, middle_layer_lib, isal and isal_asm, +# which are then combined into static or shared qpl. # Output ch_contrib::qpl by linking with 8 library targets. -# The qpl submodule comes with its own version of isal. It contains code which does not exist in upstream isal. It would be nice to link -# only upstream isal (ch_contrib::isal) but at this point we can't. +# Note, qpl submodule comes with its own version of isal that is not compatible with upstream isal (e.g., ch_contrib::isal). + +## cmake/CompileOptions.cmake and automatic wrappers generation # ========================================================================== # Copyright (C) 2022 Intel Corporation @@ -442,6 +443,7 @@ function(generate_unpack_kernel_arrays current_directory PLATFORMS_LIST) endforeach() endfunction() +# [SUBDIR]isal enable_language(ASM_NASM) @@ -479,7 +481,6 @@ set(ISAL_ASM_SRC ${QPL_SRC_DIR}/isal/igzip/igzip_body.asm ${QPL_SRC_DIR}/isal/igzip/igzip_set_long_icf_fg_04.asm ${QPL_SRC_DIR}/isal/igzip/igzip_set_long_icf_fg_06.asm ${QPL_SRC_DIR}/isal/igzip/igzip_multibinary.asm - ${QPL_SRC_DIR}/isal/igzip/stdmac.asm ${QPL_SRC_DIR}/isal/crc/crc_multibinary.asm ${QPL_SRC_DIR}/isal/crc/crc32_gzip_refl_by8.asm ${QPL_SRC_DIR}/isal/crc/crc32_gzip_refl_by8_02.asm @@ -505,7 +506,6 @@ set_property(GLOBAL APPEND PROPERTY QPL_LIB_DEPS # Setting external and internal interfaces for ISA-L library target_include_directories(isal PUBLIC $ - PRIVATE ${QPL_SRC_DIR}/isal/include PUBLIC ${QPL_SRC_DIR}/isal/igzip) set_target_properties(isal PROPERTIES @@ -617,12 +617,9 @@ target_compile_options(qplcore_sw_dispatcher # [SUBDIR]core-iaa file(GLOB HW_PATH_SRC ${QPL_SRC_DIR}/core-iaa/sources/aecs/*.c - ${QPL_SRC_DIR}/core-iaa/sources/aecs/*.cpp ${QPL_SRC_DIR}/core-iaa/sources/driver_loader/*.c - ${QPL_SRC_DIR}/core-iaa/sources/driver_loader/*.cpp ${QPL_SRC_DIR}/core-iaa/sources/descriptors/*.c - ${QPL_SRC_DIR}/core-iaa/sources/descriptors/*.cpp - ${QPL_SRC_DIR}/core-iaa/sources/bit_rev.c) + ${QPL_SRC_DIR}/core-iaa/sources/*.c) # Create library add_library(core_iaa OBJECT ${HW_PATH_SRC}) @@ -634,31 +631,27 @@ target_include_directories(core_iaa PRIVATE ${UUID_DIR} PUBLIC $ PUBLIC $ - PRIVATE $ # status.h in own_checkers.h - PRIVATE $ # own_checkers.h + PRIVATE $ # status.h in own_checkers.h + PRIVATE $ # for own_checkers.h PRIVATE $) target_compile_features(core_iaa PRIVATE c_std_11) target_compile_definitions(core_iaa PRIVATE QPL_BADARG_CHECK - PRIVATE $<$: BLOCK_ON_FAULT_ENABLED> PRIVATE $<$:LOG_HW_INIT> PRIVATE $<$:DYNAMIC_LOADING_LIBACCEL_CONFIG>) # [SUBDIR]middle-layer file(GLOB MIDDLE_LAYER_SRC - ${QPL_SRC_DIR}/middle-layer/analytics/*.cpp - ${QPL_SRC_DIR}/middle-layer/c_wrapper/*.cpp - ${QPL_SRC_DIR}/middle-layer/checksum/*.cpp + ${QPL_SRC_DIR}/middle-layer/accelerator/*.cpp + ${QPL_SRC_DIR}/middle-layer/analytics/*.cpp ${QPL_SRC_DIR}/middle-layer/common/*.cpp ${QPL_SRC_DIR}/middle-layer/compression/*.cpp ${QPL_SRC_DIR}/middle-layer/compression/*/*.cpp ${QPL_SRC_DIR}/middle-layer/compression/*/*/*.cpp ${QPL_SRC_DIR}/middle-layer/dispatcher/*.cpp ${QPL_SRC_DIR}/middle-layer/other/*.cpp - ${QPL_SRC_DIR}/middle-layer/util/*.cpp - ${QPL_SRC_DIR}/middle-layer/inflate/*.cpp - ${QPL_SRC_DIR}/core-iaa/sources/accelerator/*.cpp) # todo + ${QPL_SRC_DIR}/middle-layer/util/*.cpp) add_library(middle_layer_lib OBJECT ${MIDDLE_LAYER_SRC}) @@ -667,6 +660,7 @@ set_property(GLOBAL APPEND PROPERTY QPL_LIB_DEPS $) target_compile_options(middle_layer_lib + PRIVATE $<$:$<$:-O3;-U_FORTIFY_SOURCE;-D_FORTIFY_SOURCE=2>> PRIVATE ${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}) target_compile_definitions(middle_layer_lib @@ -682,6 +676,7 @@ target_include_directories(middle_layer_lib PRIVATE ${UUID_DIR} PUBLIC $ PUBLIC $ + PRIVATE $ PUBLIC $ PUBLIC $ PUBLIC $) @@ -689,31 +684,58 @@ target_include_directories(middle_layer_lib target_compile_definitions(middle_layer_lib PUBLIC -DQPL_LIB) # [SUBDIR]c_api -file(GLOB_RECURSE QPL_C_API_SRC - ${QPL_SRC_DIR}/c_api/*.c - ${QPL_SRC_DIR}/c_api/*.cpp) +file(GLOB QPL_C_API_SRC + ${QPL_SRC_DIR}/c_api/compression_operations/*.c + ${QPL_SRC_DIR}/c_api/compression_operations/*.cpp + ${QPL_SRC_DIR}/c_api/filter_operations/*.cpp + ${QPL_SRC_DIR}/c_api/legacy_hw_path/*.c + ${QPL_SRC_DIR}/c_api/legacy_hw_path/*.cpp + ${QPL_SRC_DIR}/c_api/other_operations/*.cpp + ${QPL_SRC_DIR}/c_api/serialization/*.cpp + ${QPL_SRC_DIR}/c_api/*.cpp) + +add_library(qpl_c_api OBJECT ${QPL_C_API_SRC}) + +target_include_directories(qpl_c_api + PUBLIC $ + PUBLIC $ $ + PRIVATE $) + +set_target_properties(qpl_c_api PROPERTIES + $<$:C_STANDARD 17 + CXX_STANDARD 17) + +target_compile_options(qpl_c_api + PRIVATE $<$:$<$:-O3;-U_FORTIFY_SOURCE;-D_FORTIFY_SOURCE=2>> + PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}>) + +target_compile_definitions(qpl_c_api + PUBLIC -DQPL_BADARG_CHECK # own_checkers.h + PUBLIC -DQPL_LIB # needed for middle_layer_lib + PUBLIC $<$:LOG_HW_INIT>) # needed for middle_layer_lib + +set_property(GLOBAL APPEND PROPERTY QPL_LIB_DEPS + $) + +# Final _qpl target get_property(LIB_DEPS GLOBAL PROPERTY QPL_LIB_DEPS) -add_library(_qpl STATIC ${QPL_C_API_SRC} ${LIB_DEPS}) +add_library(_qpl STATIC ${LIB_DEPS}) target_include_directories(_qpl - PUBLIC $ $ - PRIVATE $ - PRIVATE $) + PUBLIC $ $) -target_compile_options(_qpl - PRIVATE ${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}) target_compile_definitions(_qpl - PRIVATE -DQPL_LIB - PRIVATE -DQPL_BADARG_CHECK - PRIVATE $<$:DYNAMIC_LOADING_LIBACCEL_CONFIG> PUBLIC -DENABLE_QPL_COMPRESSION) target_link_libraries(_qpl - PRIVATE ch_contrib::accel-config - PRIVATE ch_contrib::isal) + PRIVATE ch_contrib::accel-config) + +# C++ filesystem library requires additional linking for older GNU/Clang +target_link_libraries(_qpl PRIVATE $<$,$,9.1>>:stdc++fs>) +target_link_libraries(_qpl PRIVATE $<$,$,9.0>>:c++fs>) target_include_directories(_qpl SYSTEM BEFORE PUBLIC "${QPL_PROJECT_DIR}/include" From 7caa7e20601b2500f513476dfe10819f328be3d3 Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Sat, 20 Jul 2024 02:28:13 +0000 Subject: [PATCH 019/129] block deduplicate only in throw mode --- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- src/Interpreters/InterpreterOptimizeQuery.cpp | 4 ---- src/Storages/StorageMergeTree.cpp | 6 ++++-- src/Storages/StorageReplicatedMergeTree.cpp | 6 ++++-- ...206_projection_merge_special_mergetree.sql | 19 +++++++++---------- 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index df5ec4525eb..05df26b0d31 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1298,7 +1298,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) } } if (!projection_support) - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Projection is only supported in (Replictaed)MergeTree. Consider drop or rebuild option of deduplicate_merge_projection_mode."); } diff --git a/src/Interpreters/InterpreterOptimizeQuery.cpp b/src/Interpreters/InterpreterOptimizeQuery.cpp index 8d1ac3455b7..3bee235185d 100644 --- a/src/Interpreters/InterpreterOptimizeQuery.cpp +++ b/src/Interpreters/InterpreterOptimizeQuery.cpp @@ -43,10 +43,6 @@ BlockIO InterpreterOptimizeQuery::execute() auto metadata_snapshot = table->getInMemoryMetadataPtr(); auto storage_snapshot = table->getStorageSnapshot(metadata_snapshot, getContext()); - /// Don't allow OPTIMIZE DEDUPLICATE for all engines with projections. - if (ast.deduplicate && !metadata_snapshot->projections.empty()) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "DEDUPLICATE with projections are not supported yet"); - // Empty list of names means we deduplicate by all columns, but user can explicitly state which columns to use. Names column_names; if (ast.deduplicate_by_columns) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 40b3a12297b..a5d434796ba 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1580,10 +1580,12 @@ bool StorageMergeTree::optimize( { assertNotReadonly(); - if (deduplicate && getInMemoryMetadataPtr()->hasProjections()) + if (deduplicate && getInMemoryMetadataPtr()->hasProjections() + && getSettings()->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "OPTIMIZE DEDUPLICATE query is not supported for table {} as it has projections. " - "User should drop all the projections manually before running the query", + "User should drop all the projections manually before running the query, " + "or consider drop or rebuild option of deduplicate_merge_projection_mode", getStorageID().getTableName()); if (deduplicate) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 72f725965e0..3751883df24 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5794,10 +5794,12 @@ bool StorageReplicatedMergeTree::optimize( if (!is_leader) throw Exception(ErrorCodes::NOT_A_LEADER, "OPTIMIZE cannot be done on this replica because it is not a leader"); - if (deduplicate && getInMemoryMetadataPtr()->hasProjections()) + if (deduplicate && getInMemoryMetadataPtr()->hasProjections() + && getSettings()->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "OPTIMIZE DEDUPLICATE query is not supported for table {} as it has projections. " - "User should drop all the projections manually before running the query", + "User should drop all the projections manually before running the query, " + "or consider drop or rebuild option of deduplicate_merge_projection_mode", getStorageID().getTableName()); if (cleanup) diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql index 06fb9a30aca..c8945fd784c 100644 --- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql +++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql @@ -4,14 +4,14 @@ CREATE TABLE tp ( type Int32, eventcnt UInt64, PROJECTION p (select sum(eventcnt), type group by type) -) engine = ReplacingMergeTree order by type; -- { serverError SUPPORT_IS_DISABLED } +) engine = ReplacingMergeTree order by type; -- { serverError NOT_IMPLEMENTED } CREATE TABLE tp ( type Int32, eventcnt UInt64, PROJECTION p (select sum(eventcnt), type group by type) ) engine = ReplacingMergeTree order by type -SETTINGS deduplicate_merge_projection_mode = 'throw'; -- { serverError SUPPORT_IS_DISABLED } +SETTINGS deduplicate_merge_projection_mode = 'throw'; -- { serverError NOT_IMPLEMENTED } CREATE TABLE tp ( type Int32, @@ -20,6 +20,10 @@ CREATE TABLE tp ( ) engine = ReplacingMergeTree order by type SETTINGS deduplicate_merge_projection_mode = 'drop'; +ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw'; + +OPTIMIZE TABLE tp DEDUPLICATE; -- { serverError NOT_IMPLEMENTED } + DROP TABLE tp; CREATE TABLE tp ( @@ -29,13 +33,8 @@ CREATE TABLE tp ( ) engine = ReplacingMergeTree order by type SETTINGS deduplicate_merge_projection_mode = 'rebuild'; -DROP TABLE tp; +ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw'; +OPTIMIZE TABLE tp DEDUPLICATE; -- { serverError NOT_IMPLEMENTED } --- don't allow OPTIMIZE DEDUPLICATE for all engines with projections -CREATE TABLE test ( - a INT PRIMARY KEY, - PROJECTION p (SELECT * ORDER BY a) -) engine = MergeTree; - -OPTIMIZE TABLE test DEDUPLICATE; -- { serverError NOT_IMPLEMENTED } +DROP TABLE tp; \ No newline at end of file From 09e4faf2dbadfdf1eaedc0eec127098c6b9540f1 Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Sun, 21 Jul 2024 15:05:26 +0000 Subject: [PATCH 020/129] fix --- src/Interpreters/InterpreterOptimizeQuery.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/InterpreterOptimizeQuery.cpp b/src/Interpreters/InterpreterOptimizeQuery.cpp index 3bee235185d..907a01b0432 100644 --- a/src/Interpreters/InterpreterOptimizeQuery.cpp +++ b/src/Interpreters/InterpreterOptimizeQuery.cpp @@ -20,7 +20,6 @@ namespace DB namespace ErrorCodes { extern const int THERE_IS_NO_COLUMN; - extern const int NOT_IMPLEMENTED; } From a9d7abbb928d43664f057dde88058c449a9a521c Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Tue, 23 Jul 2024 17:27:05 +0000 Subject: [PATCH 021/129] add drop option --- src/Core/SettingsEnums.cpp | 2 +- src/Storages/MergeTree/MergeTask.cpp | 9 +++++++++ .../03206_projection_merge_special_mergetree.sql | 12 ++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index 74b6c793849..b53a882de4e 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -180,7 +180,7 @@ IMPLEMENT_SETTING_ENUM(LightweightMutationProjectionMode, ErrorCodes::BAD_ARGUME IMPLEMENT_SETTING_ENUM(DeduplicateMergeProjectionMode, ErrorCodes::BAD_ARGUMENTS, {{"throw", DeduplicateMergeProjectionMode::THROW}, {"drop", DeduplicateMergeProjectionMode::DROP}, - {"rebuild", DeduplicateMergeProjectionMode::THROW}}) + {"rebuild", DeduplicateMergeProjectionMode::REBUILD}}) IMPLEMENT_SETTING_AUTO_ENUM(LocalFSReadMethod, ErrorCodes::BAD_ARGUMENTS) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index fc64fae9a58..0b358c0fd7c 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -797,6 +797,15 @@ bool MergeTask::MergeProjectionsStage::mergeMinMaxIndexAndPrepareProjections() c } + const auto mode = global_ctx->data->getSettings()->deduplicate_merge_projection_mode; + /// Under throw mode, we still choose to drop projections due to backward compatibility since some + /// users might have projections before this change. + if (mode == DeduplicateMergeProjectionMode::THROW || mode == DeduplicateMergeProjectionMode::DROP) + { + ctx->projections_iterator = ctx->tasks_for_projections.begin(); + return false; + } + const auto & projections = global_ctx->metadata_snapshot->getProjections(); for (const auto & projection : projections) diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql index c8945fd784c..749f906569e 100644 --- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql +++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql @@ -20,6 +20,18 @@ CREATE TABLE tp ( ) engine = ReplacingMergeTree order by type SETTINGS deduplicate_merge_projection_mode = 'drop'; +INSERT INTO tp SELECT number%3, 1 FROM numbers(3); + +OPTIMIZE TABLE tp FINAL; + +-- expecting no projection +SYSTEM FLUSH LOGS; +SELECT + name, + part_name +FROM system.projection_parts +WHERE (database = currentDatabase()) AND (`table` = 'tp') AND (active = 1); + ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw'; OPTIMIZE TABLE tp DEDUPLICATE; -- { serverError NOT_IMPLEMENTED } From c847d2f63fdacf1ce5d636a9af8812d543547cfe Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Wed, 24 Jul 2024 20:52:35 +0000 Subject: [PATCH 022/129] fix --- src/Interpreters/InterpreterCreateQuery.cpp | 25 ++++++++++++--------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 05df26b0d31..a5f374ba71c 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1282,24 +1282,27 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) TableProperties properties = getTablePropertiesAndNormalizeCreateQuery(create, mode); /// Projection is only supported in (Replictaed)MergeTree. - if (std::string_view engine_name(create.storage->engine->name); - !properties.projections.empty() && engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree") + if (create.storage && create.storage->engine) { - bool projection_support = false; - if (auto * setting = create.storage->settings; setting != nullptr) + if (std::string_view engine_name(create.storage->engine->name); + !properties.projections.empty() && engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree") { - for (const auto & change : setting->changes) + bool projection_support = false; + if (auto * setting = create.storage->settings; setting != nullptr) { - if (change.name == "deduplicate_merge_projection_mode" && change.value != Field("throw")) + for (const auto & change : setting->changes) { - projection_support = true; - break; + if (change.name == "deduplicate_merge_projection_mode" && change.value != Field("throw")) + { + projection_support = true; + break; + } } } + if (!projection_support) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "Projection is only supported in (Replictaed)MergeTree. Consider drop or rebuild option of deduplicate_merge_projection_mode."); } - if (!projection_support) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, - "Projection is only supported in (Replictaed)MergeTree. Consider drop or rebuild option of deduplicate_merge_projection_mode."); } /// Check type compatible for materialized dest table and select columns From 23c3fa73266cb9f32c7f30a2c83437815de89291 Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Thu, 25 Jul 2024 01:03:59 +0000 Subject: [PATCH 023/129] fix --- .../01710_aggregate_projection_with_normalized_states.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01710_aggregate_projection_with_normalized_states.sql b/tests/queries/0_stateless/01710_aggregate_projection_with_normalized_states.sql index e023c0991b3..5375823aa8e 100644 --- a/tests/queries/0_stateless/01710_aggregate_projection_with_normalized_states.sql +++ b/tests/queries/0_stateless/01710_aggregate_projection_with_normalized_states.sql @@ -9,7 +9,8 @@ CREATE TABLE r ( s Int64, PROJECTION p (SELECT a, quantilesTimingMerge(0.5, 0.95, 0.99)(q), sum(s) GROUP BY a) -) Engine=SummingMergeTree order by (x, a); +) Engine=SummingMergeTree order by (x, a) +SETTINGS deduplicate_merge_projection_mode = 'drop'; -- should set it to rebuild once projection is supported with SummingMergeTree insert into r select number%100 x, From 3f483e547572c47d137d0f2664bd82c0b978ac7b Mon Sep 17 00:00:00 2001 From: heguangnan Date: Fri, 26 Jul 2024 12:09:03 +0800 Subject: [PATCH 024/129] fix memory leak when exception happend during count distinct for null key --- src/Interpreters/Aggregator.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index e073b7a49b6..543fd8a0bf2 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -3300,6 +3300,17 @@ void NO_INLINE Aggregator::destroyImpl(Table & table) const data = nullptr; }); + + if constexpr (Method::low_cardinality_optimization || Method::one_key_nullable_optimization) + { + if (table.getNullKeyData() != nullptr) + { + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_functions[i]->destroy(table.getNullKeyData() + offsets_of_aggregate_states[i]); + + table.getNullKeyData() = nullptr; + } + } } From eeb012357196db988b01a70b33798fce99bb5deb Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 26 Jul 2024 14:48:52 +0000 Subject: [PATCH 025/129] fix filling of multilevel Nested --- src/Interpreters/inplaceBlockConversions.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp index ce3f25d16f8..945cc62754d 100644 --- a/src/Interpreters/inplaceBlockConversions.cpp +++ b/src/Interpreters/inplaceBlockConversions.cpp @@ -316,7 +316,7 @@ static String removeTupleElementsFromSubcolumn(String subcolumn_name, const Name { auto pos = subcolumn_name.find(elem + "."); if (pos != std::string::npos) - subcolumn_name.erase(pos, elem.size()); + subcolumn_name.erase(pos, elem.size() + 1); } if (subcolumn_name.ends_with(".")) @@ -395,7 +395,6 @@ void fillMissingColumns( if (!current_offsets.empty()) { - Names tuple_elements; auto serialization = IDataType::getSerialization(*requested_column); From 7b4951990edc08a1230c3569339c44ac22036eed Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Fri, 26 Jul 2024 15:19:47 +0000 Subject: [PATCH 026/129] better place to check and add more test --- src/Interpreters/InterpreterCreateQuery.cpp | 24 -------------- src/Storages/StorageFactory.cpp | 28 ++++++++++++++++ ...206_projection_merge_special_mergetree.sql | 32 +++++++++++++++++++ 3 files changed, 60 insertions(+), 24 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index a5f374ba71c..ea10ad59db4 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1281,30 +1281,6 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) /// Set and retrieve list of columns, indices and constraints. Set table engine if needed. Rewrite query in canonical way. TableProperties properties = getTablePropertiesAndNormalizeCreateQuery(create, mode); - /// Projection is only supported in (Replictaed)MergeTree. - if (create.storage && create.storage->engine) - { - if (std::string_view engine_name(create.storage->engine->name); - !properties.projections.empty() && engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree") - { - bool projection_support = false; - if (auto * setting = create.storage->settings; setting != nullptr) - { - for (const auto & change : setting->changes) - { - if (change.name == "deduplicate_merge_projection_mode" && change.value != Field("throw")) - { - projection_support = true; - break; - } - } - } - if (!projection_support) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, - "Projection is only supported in (Replictaed)MergeTree. Consider drop or rebuild option of deduplicate_merge_projection_mode."); - } - } - /// Check type compatible for materialized dest table and select columns if (create.select && create.is_materialized_view && create.to_table_id && mode <= LoadingStrictnessLevel::CREATE) { diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 060b271d8f4..71f70a807a8 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes extern const int FUNCTION_CANNOT_HAVE_PARAMETERS; extern const int BAD_ARGUMENTS; extern const int DATA_TYPE_CANNOT_BE_USED_IN_TABLES; + extern const int NOT_IMPLEMENTED; } @@ -196,9 +197,36 @@ StoragePtr StorageFactory::get( [](StorageFeatures features) { return features.supports_skipping_indices; }); if (query.columns_list && query.columns_list->projections && !query.columns_list->projections->children.empty()) + { check_feature( "projections", [](StorageFeatures features) { return features.supports_projections; }); + + /// Now let's handle the merge tree family, projection is fully supported in (Replictaed)MergeTree, + /// but also allowed in non-throw mode with other mergetree family members. + chassert(query.storage->engine); + if (std::string_view engine_name(query.storage->engine->name); + engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree") + { + /// default throw mode in deduplicate_merge_projection_mode + bool projection_allowed = false; + if (auto * setting = query.storage->settings; setting != nullptr) + { + for (const auto & change : setting->changes) + { + if (change.name == "deduplicate_merge_projection_mode" && change.value != Field("throw")) + { + projection_allowed = true; + break; + } + } + } + if (!projection_allowed) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other" + " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode."); + } + } } } diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql index 749f906569e..25517fbba30 100644 --- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql +++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql @@ -1,5 +1,37 @@ DROP TABLE IF EXISTS tp; +-- test regular merge tree +CREATE TABLE tp ( + type Int32, + eventcnt UInt64, + PROJECTION p (select sum(eventcnt), type group by type) +) engine = MergeTree order by type; + +INSERT INTO tp SELECT number%3, 1 FROM numbers(3); + +OPTIMIZE TABLE tp DEDUPLICATE; -- { serverError NOT_IMPLEMENTED } + +DROP TABLE tp; + +CREATE TABLE tp ( + type Int32, + eventcnt UInt64, + PROJECTION p (select sum(eventcnt), type group by type) +) engine = MergeTree order by type +SETTINGS deduplicate_merge_projection_mode = 'drop'; + +INSERT INTO tp SELECT number%3, 1 FROM numbers(3); + +OPTIMIZE TABLE tp DEDUPLICATE; + +ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw'; + +OPTIMIZE TABLE tp DEDUPLICATE; -- { serverError NOT_IMPLEMENTED } + +DROP TABLE tp; + + +-- test irregular merge tree CREATE TABLE tp ( type Int32, eventcnt UInt64, From c78aff235c4f21ceaeae6e94be06ddd5e20db0fa Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Sat, 27 Jul 2024 01:32:09 +0000 Subject: [PATCH 027/129] ignore setting for regular mergetree --- src/Storages/MergeTree/MergeTask.cpp | 3 ++- src/Storages/MergeTree/MergeTreeSettings.h | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 0b358c0fd7c..ce06adf110c 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -800,7 +800,8 @@ bool MergeTask::MergeProjectionsStage::mergeMinMaxIndexAndPrepareProjections() c const auto mode = global_ctx->data->getSettings()->deduplicate_merge_projection_mode; /// Under throw mode, we still choose to drop projections due to backward compatibility since some /// users might have projections before this change. - if (mode == DeduplicateMergeProjectionMode::THROW || mode == DeduplicateMergeProjectionMode::DROP) + if (global_ctx->data->merging_params.mode != MergeTreeData::MergingParams::Ordinary + && (mode == DeduplicateMergeProjectionMode::THROW || mode == DeduplicateMergeProjectionMode::DROP)) { ctx->projections_iterator = ctx->tasks_for_projections.begin(); return false; diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 166a18a7bab..5ba1988cc5d 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -215,7 +215,7 @@ struct Settings; M(Float, primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns, 0.9f, "If the value of a column of the primary key in data part changes at least in this ratio of times, skip loading next columns in memory. This allows to save memory usage by not loading useless columns of the primary key.", 0) \ /** Projection settings. */ \ M(UInt64, max_projections, 25, "The maximum number of merge tree projections.", 0) \ - M(DeduplicateMergeProjectionMode, deduplicate_merge_projection_mode, DeduplicateMergeProjectionMode::THROW, "Whether to allow create projection for the table with non-classic MergeTree, if allowed, what is the next action.", 0) \ + M(DeduplicateMergeProjectionMode, deduplicate_merge_projection_mode, DeduplicateMergeProjectionMode::THROW, "Whether to allow create projection for the table with non-classic MergeTree, if allowed, what is the action when merge, drop or rebuild.", 0) \ #define MAKE_OBSOLETE_MERGE_TREE_SETTING(M, TYPE, NAME, DEFAULT) \ M(TYPE, NAME, DEFAULT, "Obsolete setting, does nothing.", BaseSettingsHelpers::Flags::OBSOLETE) From 985e8ec15ef3cefb7063934ebd9590442faf5045 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sat, 13 Jul 2024 09:25:52 +0800 Subject: [PATCH 028/129] Clean up projection inside storage snapshot --- .../Passes/ShardNumColumnToFunctionPass.cpp | 2 +- src/Interpreters/TreeRewriter.cpp | 3 +- src/Planner/PlannerJoinTree.cpp | 6 +-- .../optimizeUseAggregateProjection.cpp | 4 +- .../optimizeUseNormalProjection.cpp | 4 +- .../QueryPlan/ReadFromMergeTree.cpp | 49 +++++++++---------- src/Processors/QueryPlan/ReadFromMergeTree.h | 4 +- src/Storages/MergeTree/IMergeTreeReader.cpp | 2 +- .../MergeTree/MergeTreeBlockReadUtils.cpp | 3 +- src/Storages/MergeTree/MergeTreeData.cpp | 7 +-- src/Storages/StorageMerge.cpp | 2 +- src/Storages/StorageSnapshot.cpp | 11 ++--- src/Storages/StorageSnapshot.h | 8 --- 13 files changed, 40 insertions(+), 65 deletions(-) diff --git a/src/Analyzer/Passes/ShardNumColumnToFunctionPass.cpp b/src/Analyzer/Passes/ShardNumColumnToFunctionPass.cpp index 82e3281121c..c58504064ce 100644 --- a/src/Analyzer/Passes/ShardNumColumnToFunctionPass.cpp +++ b/src/Analyzer/Passes/ShardNumColumnToFunctionPass.cpp @@ -46,7 +46,7 @@ public: return; const auto & storage_snapshot = table_node ? table_node->getStorageSnapshot() : table_function_node->getStorageSnapshot(); - if (!storage->isVirtualColumn(column.name, storage_snapshot->getMetadataForQuery())) + if (!storage->isVirtualColumn(column.name, storage_snapshot->metadata)) return; auto function_node = std::make_shared("shardNum"); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 6ce6f5e454e..14094c3cccf 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -1158,7 +1158,8 @@ bool TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select } } - has_virtual_shard_num = is_remote_storage && storage->isVirtualColumn("_shard_num", storage_snapshot->getMetadataForQuery()) && virtuals->has("_shard_num"); + has_virtual_shard_num + = is_remote_storage && storage->isVirtualColumn("_shard_num", storage_snapshot->metadata) && virtuals->has("_shard_num"); } /// Collect missed object subcolumns diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index a3db0395ccc..bc31af32a20 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -417,20 +417,20 @@ void updatePrewhereOutputsIfNeeded(SelectQueryInfo & table_expression_query_info /// We evaluate sampling for Merge lazily so we need to get all the columns if (storage_snapshot->storage.getName() == "Merge") { - const auto columns = storage_snapshot->getMetadataForQuery()->getColumns().getAll(); + const auto columns = storage_snapshot->metadata->getColumns().getAll(); for (const auto & column : columns) required_columns.insert(column.name); } else { - auto columns_required_for_sampling = storage_snapshot->getMetadataForQuery()->getColumnsRequiredForSampling(); + auto columns_required_for_sampling = storage_snapshot->metadata->getColumnsRequiredForSampling(); required_columns.insert(columns_required_for_sampling.begin(), columns_required_for_sampling.end()); } } if (table_expression_modifiers->hasFinal()) { - auto columns_required_for_final = storage_snapshot->getMetadataForQuery()->getColumnsRequiredForFinal(); + auto columns_required_for_final = storage_snapshot->metadata->getColumnsRequiredForFinal(); required_columns.insert(columns_required_for_final.begin(), columns_required_for_final.end()); } } diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp index ad89cec5f79..52d1931c51e 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp @@ -757,9 +757,7 @@ std::optional optimizeUseAggregateProjections(QueryPlan::Node & node, Qu else { auto storage_snapshot = reading->getStorageSnapshot(); - auto proj_snapshot = std::make_shared(storage_snapshot->storage, storage_snapshot->metadata); - proj_snapshot->addProjection(best_candidate->projection); - + auto proj_snapshot = std::make_shared(storage_snapshot->storage, best_candidate->projection->metadata); auto projection_query_info = query_info; projection_query_info.prewhere_info = nullptr; projection_query_info.filter_actions_dag = nullptr; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp index e9aee65675f..b15f913fc19 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp @@ -193,9 +193,7 @@ std::optional optimizeUseNormalProjections(Stack & stack, QueryPlan::Nod } auto storage_snapshot = reading->getStorageSnapshot(); - auto proj_snapshot = std::make_shared(storage_snapshot->storage, storage_snapshot->metadata); - proj_snapshot->addProjection(best_candidate->projection); - + auto proj_snapshot = std::make_shared(storage_snapshot->storage, best_candidate->projection->metadata); auto query_info_copy = query_info; query_info_copy.prewhere_info = nullptr; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 483876dd293..fed94632739 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -283,7 +283,6 @@ ReadFromMergeTree::ReadFromMergeTree( , all_column_names(std::move(all_column_names_)) , data(data_) , actions_settings(ExpressionActionsSettings::fromContext(context_)) - , metadata_for_reading(storage_snapshot->getMetadataForQuery()) , block_size{ .max_block_size_rows = max_block_size_, .preferred_block_size_bytes = context->getSettingsRef().preferred_block_size_bytes, @@ -325,7 +324,7 @@ ReadFromMergeTree::ReadFromMergeTree( updateSortDescriptionForOutputStream( *output_stream, - storage_snapshot->getMetadataForQuery()->getSortingKeyColumns(), + storage_snapshot->metadata->getSortingKeyColumns(), getSortDirection(), query_info.input_order_info, prewhere_info, @@ -780,7 +779,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_ Names in_order_column_names_to_read(column_names); /// Add columns needed to calculate the sorting expression - for (const auto & column_name : metadata_for_reading->getColumnsRequiredForSortingKey()) + for (const auto & column_name : storage_snapshot->metadata->getColumnsRequiredForSortingKey()) { if (column_names_set.contains(column_name)) continue; @@ -800,10 +799,10 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_ info.use_uncompressed_cache); }; - auto sorting_expr = metadata_for_reading->getSortingKey().expression; + auto sorting_expr = storage_snapshot->metadata->getSortingKey().expression; SplitPartsWithRangesByPrimaryKeyResult split_ranges_result = splitPartsWithRangesByPrimaryKey( - metadata_for_reading->getPrimaryKey(), + storage_snapshot->metadata->getPrimaryKey(), std::move(sorting_expr), std::move(parts_with_ranges), num_streams, @@ -881,7 +880,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( if (prewhere_info) { NameSet sorting_columns; - for (const auto & column : metadata_for_reading->getSortingKey().expression->getRequiredColumnsWithTypes()) + for (const auto & column : storage_snapshot->metadata->getSortingKey().expression->getRequiredColumnsWithTypes()) sorting_columns.insert(column.name); have_input_columns_removed_after_prewhere = restorePrewhereInputs(*prewhere_info, sorting_columns); @@ -1036,12 +1035,12 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( if (need_preliminary_merge || output_each_partition_through_separate_port) { size_t prefix_size = input_order_info->used_prefix_of_sorting_key_size; - auto order_key_prefix_ast = metadata_for_reading->getSortingKey().expression_list_ast->clone(); + auto order_key_prefix_ast = storage_snapshot->metadata->getSortingKey().expression_list_ast->clone(); order_key_prefix_ast->children.resize(prefix_size); - auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_for_reading->getColumns().getAllPhysical()); + auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, storage_snapshot->metadata->getColumns().getAllPhysical()); auto sorting_key_prefix_expr = ExpressionAnalyzer(order_key_prefix_ast, syntax_result, context).getActionsDAG(false); - const auto & sorting_columns = metadata_for_reading->getSortingKey().column_names; + const auto & sorting_columns = storage_snapshot->metadata->getSortingKey().column_names; SortDescription sort_description; sort_description.compile_sort_description = settings.compile_sort_description; @@ -1148,7 +1147,7 @@ bool ReadFromMergeTree::doNotMergePartsAcrossPartitionsFinal() const if (settings.do_not_merge_across_partitions_select_final.changed) return settings.do_not_merge_across_partitions_select_final; - if (!metadata_for_reading->hasPrimaryKey() || !metadata_for_reading->hasPartitionKey()) + if (!storage_snapshot->metadata->hasPrimaryKey() || !storage_snapshot->metadata->hasPartitionKey()) return false; /** To avoid merging parts across partitions we want result of partition key expression for @@ -1158,11 +1157,11 @@ bool ReadFromMergeTree::doNotMergePartsAcrossPartitionsFinal() const * in primary key, then for same primary key column values, result of partition key expression * will be the same. */ - const auto & partition_key_expression = metadata_for_reading->getPartitionKey().expression; + const auto & partition_key_expression = storage_snapshot->metadata->getPartitionKey().expression; if (partition_key_expression->getActionsDAG().hasNonDeterministic()) return false; - const auto & primary_key_columns = metadata_for_reading->getPrimaryKey().column_names; + const auto & primary_key_columns = storage_snapshot->metadata->getPrimaryKey().column_names; NameSet primary_key_columns_set(primary_key_columns.begin(), primary_key_columns.end()); const auto & partition_key_required_columns = partition_key_expression->getRequiredColumns(); @@ -1215,12 +1214,12 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( /// we will store lonely parts with level > 0 to use parallel select on them. RangesInDataParts non_intersecting_parts_by_primary_key; - auto sorting_expr = metadata_for_reading->getSortingKey().expression; + auto sorting_expr = storage_snapshot->metadata->getSortingKey().expression; if (prewhere_info) { NameSet sorting_columns; - for (const auto & column : metadata_for_reading->getSortingKey().expression->getRequiredColumnsWithTypes()) + for (const auto & column : storage_snapshot->metadata->getSortingKey().expression->getRequiredColumnsWithTypes()) sorting_columns.insert(column.name); restorePrewhereInputs(*prewhere_info, sorting_columns); } @@ -1251,7 +1250,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( if (new_parts.empty()) continue; - if (num_streams > 1 && metadata_for_reading->hasPrimaryKey()) + if (num_streams > 1 && storage_snapshot->metadata->hasPrimaryKey()) { // Let's split parts into non intersecting parts ranges and layers to ensure data parallelism of FINAL. auto in_order_reading_step_getter = [this, &column_names, &info](auto parts) @@ -1271,7 +1270,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( data.merging_params.is_deleted_column.empty() && !reader_settings.read_in_order; SplitPartsWithRangesByPrimaryKeyResult split_ranges_result = splitPartsWithRangesByPrimaryKey( - metadata_for_reading->getPrimaryKey(), + storage_snapshot->metadata->getPrimaryKey(), sorting_expr, std::move(new_parts), num_streams, @@ -1303,7 +1302,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( if (pipes.empty()) continue; - Names sort_columns = metadata_for_reading->getSortingKeyColumns(); + Names sort_columns = storage_snapshot->metadata->getSortingKeyColumns(); SortDescription sort_description; sort_description.compile_sort_description = settings.compile_sort_description; sort_description.min_count_to_compile_sort_description = settings.min_count_to_compile_sort_description; @@ -1311,7 +1310,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); - Names partition_key_columns = metadata_for_reading->getPartitionKey().column_names; + Names partition_key_columns = storage_snapshot->metadata->getPartitionKey().column_names; for (size_t i = 0; i < sort_columns_size; ++i) sort_description.emplace_back(sort_columns[i], 1, 1); @@ -1368,7 +1367,7 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead( return selectRangesToRead( std::move(parts), std::move(alter_conversions), - metadata_for_reading, + storage_snapshot->metadata, query_info, context, requested_num_streams, @@ -1532,7 +1531,7 @@ void ReadFromMergeTree::applyFilters(ActionDAGNodes added_filter_nodes) prepared_parts, context, query_info, - metadata_for_reading); + storage_snapshot->metadata); } } @@ -1701,7 +1700,7 @@ bool ReadFromMergeTree::requestReadingInOrder(size_t prefix_size, int direction, /// update sort info for output stream SortDescription sort_description; - const Names & sorting_key_columns = metadata_for_reading->getSortingKeyColumns(); + const Names & sorting_key_columns = storage_snapshot->metadata->getSortingKeyColumns(); const Block & header = output_stream->header; const int sort_direction = getSortDirection(); for (const auto & column_name : sorting_key_columns) @@ -1743,7 +1742,7 @@ void ReadFromMergeTree::updatePrewhereInfo(const PrewhereInfoPtr & prewhere_info updateSortDescriptionForOutputStream( *output_stream, - storage_snapshot->getMetadataForQuery()->getSortingKeyColumns(), + storage_snapshot->metadata->getSortingKeyColumns(), getSortDirection(), query_info.input_order_info, prewhere_info, @@ -1869,7 +1868,7 @@ Pipe ReadFromMergeTree::spreadMarkRanges( throw Exception(ErrorCodes::LOGICAL_ERROR, "Optimization isn't supposed to be used for queries with final"); /// Add columns needed to calculate the sorting expression and the sign. - for (const auto & column : metadata_for_reading->getColumnsRequiredForSortingKey()) + for (const auto & column : storage_snapshot->metadata->getColumnsRequiredForSortingKey()) { if (!names.contains(column)) { @@ -1963,10 +1962,6 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons fmt::format("{}.{}", data.getStorageID().getFullNameNotQuoted(), part.data_part->info.partition_id)); } context->getQueryContext()->addQueryAccessInfo(partition_names); - - if (storage_snapshot->projection) - context->getQueryContext()->addQueryAccessInfo( - Context::QualifiedProjectionName{.storage_id = data.getStorageID(), .projection_name = storage_snapshot->projection->name}); } ProfileEvents::increment(ProfileEvents::SelectedParts, result.selected_parts); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index a12f53924c3..f12da5d10bc 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -171,7 +171,7 @@ public: AnalysisResultPtr selectRangesToRead(bool find_exact_ranges = false) const; - StorageMetadataPtr getStorageMetadata() const { return metadata_for_reading; } + StorageMetadataPtr getStorageMetadata() const { return storage_snapshot->metadata; } /// Returns `false` if requested reading cannot be performed. bool requestReadingInOrder(size_t prefix_size, int direction, size_t limit); @@ -216,8 +216,6 @@ private: const MergeTreeData & data; ExpressionActionsSettings actions_settings; - StorageMetadataPtr metadata_for_reading; - const MergeTreeReadTask::BlockSizeParams block_size; size_t requested_num_streams; diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 264b2b397f4..b030d4335af 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -75,7 +75,7 @@ void IMergeTreeReader::fillVirtualColumns(Columns & columns, size_t rows) const throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Filling of virtual columns is supported only for LoadedMergeTreeDataPartInfoForReader"); const auto & data_part = loaded_part_info->getDataPart(); - const auto & storage_columns = storage_snapshot->getMetadataForQuery()->getColumns(); + const auto & storage_columns = storage_snapshot->metadata->getColumns(); const auto & virtual_columns = storage_snapshot->virtual_columns; auto it = requested_columns.begin(); diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index aaa4ecd8eee..3dbb9d64f2f 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -71,8 +71,7 @@ bool injectRequiredColumnsRecursively( /// Column doesn't have default value and don't exist in part /// don't need to add to required set. - auto metadata_snapshot = storage_snapshot->getMetadataForQuery(); - const auto column_default = metadata_snapshot->getColumns().getDefault(column_name); + const auto column_default = storage_snapshot->metadata->getColumns().getDefault(column_name); if (!column_default) return false; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 2e10f5a0227..2aee3d2ce72 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7137,12 +7137,7 @@ UInt64 MergeTreeData::estimateNumberOfRowsToRead( MergeTreeDataSelectExecutor reader(*this); auto result_ptr = reader.estimateNumMarksToRead( - parts, - storage_snapshot->getMetadataForQuery()->getColumns().getAll().getNames(), - storage_snapshot->metadata, - query_info, - query_context, - query_context->getSettingsRef().max_threads); + parts, {}, storage_snapshot->metadata, query_info, query_context, query_context->getSettingsRef().max_threads); UInt64 total_rows = result_ptr->selected_rows; if (query_info.trivial_limit > 0 && query_info.trivial_limit < total_rows) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 7c268d36a7b..e88844e2d31 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -602,7 +602,7 @@ std::vector ReadFromMerge::createChildrenPlans(SelectQ ASTPtr required_columns_expr_list = std::make_shared(); ASTPtr column_expr; - auto sample_block = merge_storage_snapshot->getMetadataForQuery()->getSampleBlock(); + auto sample_block = merge_storage_snapshot->metadata->getSampleBlock(); for (const auto & column : real_column_names) { diff --git a/src/Storages/StorageSnapshot.cpp b/src/Storages/StorageSnapshot.cpp index aada25168f8..13a46990556 100644 --- a/src/Storages/StorageSnapshot.cpp +++ b/src/Storages/StorageSnapshot.cpp @@ -63,7 +63,6 @@ std::shared_ptr StorageSnapshot::clone(DataPtr data_) const { auto res = std::make_shared(storage, metadata, object_columns); - res->projection = projection; res->data = std::move(data_); return res; @@ -79,7 +78,7 @@ ColumnsDescription StorageSnapshot::getAllColumnsDescription() const NamesAndTypesList StorageSnapshot::getColumns(const GetColumnsOptions & options) const { - auto all_columns = getMetadataForQuery()->getColumns().get(options); + auto all_columns = metadata->getColumns().get(options); if (options.with_extended_objects) extendObjectColumns(all_columns, object_columns, options.with_subcolumns); @@ -113,7 +112,7 @@ NamesAndTypesList StorageSnapshot::getColumnsByNames(const GetColumnsOptions & o std::optional StorageSnapshot::tryGetColumn(const GetColumnsOptions & options, const String & column_name) const { - const auto & columns = getMetadataForQuery()->getColumns(); + const auto & columns = metadata->getColumns(); auto column = columns.tryGetColumn(options, column_name); if (column && (!column->type->hasDynamicSubcolumnsDeprecated() || !options.with_extended_objects)) return column; @@ -189,7 +188,7 @@ Block StorageSnapshot::getSampleBlockForColumns(const Names & column_names) cons { Block res; - const auto & columns = getMetadataForQuery()->getColumns(); + const auto & columns = metadata->getColumns(); for (const auto & column_name : column_names) { auto column = columns.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); @@ -221,7 +220,7 @@ Block StorageSnapshot::getSampleBlockForColumns(const Names & column_names) cons ColumnsDescription StorageSnapshot::getDescriptionForColumns(const Names & column_names) const { ColumnsDescription res; - const auto & columns = getMetadataForQuery()->getColumns(); + const auto & columns = metadata->getColumns(); for (const auto & name : column_names) { auto column = columns.tryGetColumnOrSubcolumnDescription(GetColumnsOptions::All, name); @@ -257,7 +256,7 @@ namespace void StorageSnapshot::check(const Names & column_names) const { - const auto & columns = getMetadataForQuery()->getColumns(); + const auto & columns = metadata->getColumns(); auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withSubcolumns(); if (column_names.empty()) diff --git a/src/Storages/StorageSnapshot.h b/src/Storages/StorageSnapshot.h index 89e97f2abb8..e94d1fc9a26 100644 --- a/src/Storages/StorageSnapshot.h +++ b/src/Storages/StorageSnapshot.h @@ -30,9 +30,6 @@ struct StorageSnapshot using DataPtr = std::unique_ptr; DataPtr data; - /// Projection that is used in query. - mutable const ProjectionDescription * projection = nullptr; - StorageSnapshot( const IStorage & storage_, StorageMetadataPtr metadata_); @@ -82,11 +79,6 @@ struct StorageSnapshot void check(const Names & column_names) const; DataTypePtr getConcreteType(const String & column_name) const; - - void addProjection(const ProjectionDescription * projection_) const { projection = projection_; } - - /// If we have a projection then we should use its metadata. - StorageMetadataPtr getMetadataForQuery() const { return projection ? projection->metadata : metadata; } }; using StorageSnapshotPtr = std::shared_ptr; From 36c57ca50bf54180dc1a68bfe097112f3dc13a6f Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Mon, 29 Jul 2024 19:50:28 +0000 Subject: [PATCH 029/129] only check in the create mode to prevent failure to start --- src/Storages/StorageFactory.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 71f70a807a8..a059d624cd8 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -202,11 +202,12 @@ StoragePtr StorageFactory::get( "projections", [](StorageFeatures features) { return features.supports_projections; }); - /// Now let's handle the merge tree family, projection is fully supported in (Replictaed)MergeTree, - /// but also allowed in non-throw mode with other mergetree family members. + /// Now let's handle the merge tree family. Note we only handle in the mode of CREATE due to backward compatibility. + /// Otherwise, it would fail to start in the case of existing projections with special mergetree. + /// Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other mergetree family members. chassert(query.storage->engine); - if (std::string_view engine_name(query.storage->engine->name); - engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree") + if (std::string_view engine_name(query.storage->engine->name); mode == LoadingStrictnessLevel::CREATE + && engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree") { /// default throw mode in deduplicate_merge_projection_mode bool projection_allowed = false; From 09619e6006f122fb3a8352328a07f42bfd284d17 Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Mon, 29 Jul 2024 20:57:21 +0000 Subject: [PATCH 030/129] consider the case of alter table add projection --- src/Storages/MergeTree/MergeTreeData.cpp | 10 +++++++ ...ojection_merge_special_mergetree.reference | 1 + ...206_projection_merge_special_mergetree.sql | 26 +++++++++++++++++-- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 78a551591a6..677c4a92cda 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3211,6 +3211,16 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context queryToString(mutation_commands.ast())); } + /// Block the case of alter table add projection for special merge trees. + if (std::any_of(commands.begin(), commands.end(), [](const AlterCommand & c) { return c.type == AlterCommand::ADD_PROJECTION; })) + { + if (auto storage_name = getName(); storage_name != "MergeTree" && storage_name != "ReplicatedMergeTree" + && settings_from_storage->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other" + " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode."); + } + commands.apply(new_metadata, local_context); if (AlterCommands::hasFullTextIndex(new_metadata) && !settings.allow_experimental_full_text_index) diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.reference b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.reference index e69de29bb2d..1a9cc2b7fbf 100644 --- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.reference +++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.reference @@ -0,0 +1 @@ +p diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql index 25517fbba30..e0a4f4f8cec 100644 --- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql +++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql @@ -59,8 +59,7 @@ OPTIMIZE TABLE tp FINAL; -- expecting no projection SYSTEM FLUSH LOGS; SELECT - name, - part_name + name FROM system.projection_parts WHERE (database = currentDatabase()) AND (`table` = 'tp') AND (active = 1); @@ -81,4 +80,27 @@ ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw'; OPTIMIZE TABLE tp DEDUPLICATE; -- { serverError NOT_IMPLEMENTED } +DROP TABLE tp; + +-- test alter add projection case +CREATE TABLE tp ( + type Int32, + eventcnt UInt64 +) engine = ReplacingMergeTree order by type; + +ALTER TABLE tp ADD PROJECTION p (SELECT sum(eventcnt), type GROUP BY type); -- { serverError NOT_IMPLEMENTED } + +ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'drop'; + +ALTER TABLE tp ADD PROJECTION p (SELECT sum(eventcnt), type GROUP BY type); + +INSERT INTO tp SELECT number%3, 1 FROM numbers(3); + +SYSTEM FLUSH LOGS; +-- expecting projection p +SELECT + name +FROM system.projection_parts +WHERE (database = currentDatabase()) AND (`table` = 'tp') AND (active = 1); + DROP TABLE tp; \ No newline at end of file From cd036886ec9ba030da7b6b6151bb81e5cc3f7636 Mon Sep 17 00:00:00 2001 From: heguangnan Date: Tue, 30 Jul 2024 15:53:48 +0800 Subject: [PATCH 031/129] add test --- ...unt_distinct_null_key_memory_leak.reference | 0 ...214_count_distinct_null_key_memory_leak.sql | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.reference create mode 100644 tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql diff --git a/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.reference b/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql b/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql new file mode 100644 index 00000000000..847d3742dc3 --- /dev/null +++ b/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql @@ -0,0 +1,18 @@ +-- Tags: no-fasttest + +DROP TABLE IF EXISTS testnull; +CREATE TABLE testnull +( + `a` Nullable(String), + `b` Nullable(String), + `c` Nullable(String) +) +ENGINE = MergeTree +PARTITION BY tuple() +ORDER BY c +SETTINGS index_granularity = 8192, allow_nullable_key=1; + +INSERT INTO testnull(b,c) SELECT toString(rand64()) AS b, toString(rand64()) AS c FROM numbers(1000000) +SELECT count(distinct b) FROM testnull GROUP BY a SETTINGS max_memory_usage = 54748364; -- {serverError MEMORY_LIMIT_EXCEEDED} + +DROP TABLE testnull; \ No newline at end of file From aec431f68bf16b45a6b36deb61c146e08cb4f644 Mon Sep 17 00:00:00 2001 From: heguangnan Date: Tue, 30 Jul 2024 22:54:05 +0800 Subject: [PATCH 032/129] fix test --- .../0_stateless/03214_count_distinct_null_key_memory_leak.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql b/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql index 847d3742dc3..d8428ec6b4a 100644 --- a/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql +++ b/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql @@ -12,7 +12,7 @@ PARTITION BY tuple() ORDER BY c SETTINGS index_granularity = 8192, allow_nullable_key=1; -INSERT INTO testnull(b,c) SELECT toString(rand64()) AS b, toString(rand64()) AS c FROM numbers(1000000) +INSERT INTO testnull(b,c) SELECT toString(rand64()) AS b, toString(rand64()) AS c FROM numbers(1000000); SELECT count(distinct b) FROM testnull GROUP BY a SETTINGS max_memory_usage = 54748364; -- {serverError MEMORY_LIMIT_EXCEEDED} DROP TABLE testnull; \ No newline at end of file From 4bf7aa1950f65aa82e85962ab3643f7df0e8bf2a Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Wed, 31 Jul 2024 13:30:14 +0200 Subject: [PATCH 033/129] Revert "Revert "Add settings to replace external engines to Null during create"" --- docs/en/operations/settings/settings.md | 12 + src/Core/Settings.h | 2 + src/Core/SettingsChangesHistory.cpp | 4 +- src/Interpreters/InterpreterCreateQuery.cpp | 35 +++ .../test_restore_external_engines/__init__.py | 0 .../configs/backups_disk.xml | 14 ++ .../configs/remote_servers.xml | 21 ++ .../test_restore_external_engines/test.py | 218 ++++++++++++++++++ 8 files changed, 305 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_restore_external_engines/__init__.py create mode 100644 tests/integration/test_restore_external_engines/configs/backups_disk.xml create mode 100644 tests/integration/test_restore_external_engines/configs/remote_servers.xml create mode 100644 tests/integration/test_restore_external_engines/test.py diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index c3f697c3bdc..8739414464e 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -5608,3 +5608,15 @@ Default value: `10000000`. Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached. Default value: `1GiB`. + +## restore_replace_external_engines_to_null + +For testing purposes. Replaces all external engines to Null to not initiate external connections. + +Default value: `False` + +## restore_replace_external_table_functions_to_null + +For testing purposes. Replaces all external table functions to Null to not initiate external connections. + +Default value: `False` \ No newline at end of file diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 4fc2034b855..27b71558bd3 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -893,6 +893,8 @@ class IColumn; M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \ M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \ M(UInt64, extract_key_value_pairs_max_pairs_per_row, 1000, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory.", 0) ALIAS(extract_kvp_max_pairs_per_row) \ + M(Bool, restore_replace_external_engines_to_null, false, "Replace all the external table engines to Null on restore. Useful for testing purposes", 0) \ + M(Bool, restore_replace_external_table_functions_to_null, false, "Replace all table functions to Null on restore. Useful for testing purposes", 0) \ \ \ /* ###################################### */ \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 9faf77e9087..8bea0b1eed3 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -80,7 +80,9 @@ static std::initializer_listno_empty_args = true; storage.set(storage.engine, engine_ast); } + + void setNullTableEngine(ASTStorage & storage) + { + auto engine_ast = std::make_shared(); + engine_ast->name = "Null"; + engine_ast->no_empty_args = true; + storage.set(storage.engine, engine_ast); + } + } void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const { if (create.as_table_function) + { + if (getContext()->getSettingsRef().restore_replace_external_table_functions_to_null) + { + const auto & factory = TableFunctionFactory::instance(); + + auto properties = factory.tryGetProperties(create.as_table_function->as()->name); + if (properties && properties->allow_readonly) + return; + if (!create.storage) + { + auto storage_ast = std::make_shared(); + create.set(create.storage, storage_ast); + } + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "Storage should not be created yet, it's a bug."); + create.as_table_function = nullptr; + setNullTableEngine(*create.storage); + } return; + } if (create.is_dictionary || create.is_ordinary_view || create.is_live_view || create.is_window_view) return; @@ -1015,6 +1043,13 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one. setDefaultTableEngine(*create.storage, getContext()->getSettingsRef().default_table_engine.value); } + /// For external tables with restore_replace_external_engine_to_null setting we replace external engines to + /// Null table engine. + else if (getContext()->getSettingsRef().restore_replace_external_engines_to_null) + { + if (StorageFactory::instance().getStorageFeatures(create.storage->engine->name).source_access_type != AccessType::NONE) + setNullTableEngine(*create.storage); + } return; } diff --git a/tests/integration/test_restore_external_engines/__init__.py b/tests/integration/test_restore_external_engines/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_restore_external_engines/configs/backups_disk.xml b/tests/integration/test_restore_external_engines/configs/backups_disk.xml new file mode 100644 index 00000000000..f7d666c6542 --- /dev/null +++ b/tests/integration/test_restore_external_engines/configs/backups_disk.xml @@ -0,0 +1,14 @@ + + + + + local + /backups/ + + + + + backups + /backups/ + + diff --git a/tests/integration/test_restore_external_engines/configs/remote_servers.xml b/tests/integration/test_restore_external_engines/configs/remote_servers.xml new file mode 100644 index 00000000000..76ad3618339 --- /dev/null +++ b/tests/integration/test_restore_external_engines/configs/remote_servers.xml @@ -0,0 +1,21 @@ + + + + + true + + replica1 + 9000 + + + replica2 + 9000 + + + replica3 + 9000 + + + + + diff --git a/tests/integration/test_restore_external_engines/test.py b/tests/integration/test_restore_external_engines/test.py new file mode 100644 index 00000000000..cf189f2a6ed --- /dev/null +++ b/tests/integration/test_restore_external_engines/test.py @@ -0,0 +1,218 @@ +import pytest + +import pymysql.cursors +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +configs = ["configs/remote_servers.xml", "configs/backups_disk.xml"] + +node1 = cluster.add_instance( + "replica1", + with_zookeeper=True, + with_mysql8=True, + main_configs=configs, + external_dirs=["/backups/"], +) +node2 = cluster.add_instance( + "replica2", + with_zookeeper=True, + with_mysql8=True, + main_configs=configs, + external_dirs=["/backups/"], +) +node3 = cluster.add_instance( + "replica3", + with_zookeeper=True, + with_mysql8=True, + main_configs=configs, + external_dirs=["/backups/"], +) +nodes = [node1, node2, node3] + +backup_id_counter = 0 + + +def new_backup_name(): + global backup_id_counter + backup_id_counter += 1 + return f"Disk('backups', '{backup_id_counter}/')" + + +def cleanup_nodes(nodes, dbname): + for node in nodes: + node.query(f"DROP DATABASE IF EXISTS {dbname} SYNC") + + +def fill_nodes(nodes, dbname): + cleanup_nodes(nodes, dbname) + for node in nodes: + node.query( + f"CREATE DATABASE {dbname} ENGINE = Replicated('/clickhouse/databases/{dbname}', 'default', '{node.name}')" + ) + + +def drop_mysql_table(conn, tableName): + with conn.cursor() as cursor: + cursor.execute(f"DROP TABLE IF EXISTS `clickhouse`.`{tableName}`") + + +def get_mysql_conn(cluster): + conn = pymysql.connect( + user="root", + password="clickhouse", + host=cluster.mysql8_ip, + port=cluster.mysql8_port, + ) + return conn + + +def fill_tables(cluster, dbname): + fill_nodes(nodes, dbname) + + conn = get_mysql_conn(cluster) + + with conn.cursor() as cursor: + cursor.execute("DROP DATABASE IF EXISTS clickhouse") + cursor.execute("CREATE DATABASE clickhouse") + cursor.execute("DROP TABLE IF EXISTS clickhouse.inference_table") + cursor.execute( + "CREATE TABLE clickhouse.inference_table (id INT PRIMARY KEY, data BINARY(16) NOT NULL)" + ) + cursor.execute( + "INSERT INTO clickhouse.inference_table VALUES (100, X'9fad5e9eefdfb449')" + ) + conn.commit() + + parameters = "'mysql80:3306', 'clickhouse', 'inference_table', 'root', 'clickhouse'" + + node1.query( + f"CREATE TABLE {dbname}.mysql_schema_inference_engine ENGINE=MySQL({parameters})" + ) + node1.query( + f"CREATE TABLE {dbname}.mysql_schema_inference_function AS mysql({parameters})" + ) + + node1.query(f"CREATE TABLE {dbname}.merge_tree (id UInt64, b String) ORDER BY id") + node1.query(f"INSERT INTO {dbname}.merge_tree VALUES (100, 'abc')") + + expected = "id\tInt32\t\t\t\t\t\ndata\tFixedString(16)\t\t\t\t\t\n" + assert ( + node1.query(f"DESCRIBE TABLE {dbname}.mysql_schema_inference_engine") + == expected + ) + assert ( + node1.query(f"DESCRIBE TABLE {dbname}.mysql_schema_inference_function") + == expected + ) + assert node1.query(f"SELECT id FROM mysql({parameters})") == "100\n" + assert ( + node1.query(f"SELECT id FROM {dbname}.mysql_schema_inference_engine") == "100\n" + ) + assert ( + node1.query(f"SELECT id FROM {dbname}.mysql_schema_inference_function") + == "100\n" + ) + assert node1.query(f"SELECT id FROM {dbname}.merge_tree") == "100\n" + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + + except Exception as ex: + print(ex) + + finally: + cluster.shutdown() + + +def test_restore_table(start_cluster): + fill_tables(cluster, "replicated") + backup_name = new_backup_name() + node2.query(f"SYSTEM SYNC DATABASE REPLICA replicated") + + node2.query(f"BACKUP DATABASE replicated TO {backup_name}") + + node2.query("DROP TABLE replicated.mysql_schema_inference_engine") + node2.query("DROP TABLE replicated.mysql_schema_inference_function") + + node3.query(f"SYSTEM SYNC DATABASE REPLICA replicated") + + assert node3.query("EXISTS replicated.mysql_schema_inference_engine") == "0\n" + assert node3.query("EXISTS replicated.mysql_schema_inference_function") == "0\n" + + node3.query( + f"RESTORE DATABASE replicated FROM {backup_name} SETTINGS allow_different_database_def=true" + ) + node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated") + + assert ( + node1.query( + "SELECT count(), sum(id) FROM replicated.mysql_schema_inference_engine" + ) + == "1\t100\n" + ) + assert ( + node1.query( + "SELECT count(), sum(id) FROM replicated.mysql_schema_inference_function" + ) + == "1\t100\n" + ) + assert ( + node1.query("SELECT count(), sum(id) FROM replicated.merge_tree") == "1\t100\n" + ) + cleanup_nodes(nodes, "replicated") + + +def test_restore_table_null(start_cluster): + fill_tables(cluster, "replicated2") + + backup_name = new_backup_name() + node2.query(f"SYSTEM SYNC DATABASE REPLICA replicated2") + + node2.query(f"BACKUP DATABASE replicated2 TO {backup_name}") + + node2.query("DROP TABLE replicated2.mysql_schema_inference_engine") + node2.query("DROP TABLE replicated2.mysql_schema_inference_function") + + node3.query(f"SYSTEM SYNC DATABASE REPLICA replicated2") + + assert node3.query("EXISTS replicated2.mysql_schema_inference_engine") == "0\n" + assert node3.query("EXISTS replicated2.mysql_schema_inference_function") == "0\n" + + node3.query( + f"RESTORE DATABASE replicated2 FROM {backup_name} SETTINGS allow_different_database_def=1, allow_different_table_def=1 SETTINGS restore_replace_external_engines_to_null=1, restore_replace_external_table_functions_to_null=1" + ) + node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated2") + + assert ( + node1.query( + "SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_engine" + ) + == "0\t0\n" + ) + assert ( + node1.query( + "SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_function" + ) + == "0\t0\n" + ) + assert ( + node1.query("SELECT count(), sum(id) FROM replicated2.merge_tree") == "1\t100\n" + ) + assert ( + node1.query( + "SELECT engine FROM system.tables where database = 'replicated2' and name like '%mysql%'" + ) + == "Null\nNull\n" + ) + assert ( + node1.query( + "SELECT engine FROM system.tables where database = 'replicated2' and name like '%merge_tree%'" + ) + == "MergeTree\n" + ) + cleanup_nodes(nodes, "replicated2") From 64a7a413619cda2edb336079740cca2a3d6503f7 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Wed, 31 Jul 2024 13:42:38 +0200 Subject: [PATCH 034/129] Update SettingsChangesHistory.cpp --- src/Core/SettingsChangesHistory.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 8bea0b1eed3..5e846868478 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -57,6 +57,9 @@ String ClickHouseVersion::toString() const /// Note: please check if the key already exists to prevent duplicate entries. static std::initializer_list> settings_changes_history_initializer = { + {"24.8", {{"restore_replace_external_table_functions_to_null", false, false, "New setting."}, + {"restore_replace_external_engines_to_null", false, false, "New setting."} + }}, {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."}, {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"}, {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"}, @@ -80,9 +83,7 @@ static std::initializer_list Date: Wed, 31 Jul 2024 21:35:55 +0800 Subject: [PATCH 035/129] fix test --- .../0_stateless/03214_count_distinct_null_key_memory_leak.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql b/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql index d8428ec6b4a..84804e4e016 100644 --- a/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql +++ b/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql @@ -13,6 +13,6 @@ ORDER BY c SETTINGS index_granularity = 8192, allow_nullable_key=1; INSERT INTO testnull(b,c) SELECT toString(rand64()) AS b, toString(rand64()) AS c FROM numbers(1000000); -SELECT count(distinct b) FROM testnull GROUP BY a SETTINGS max_memory_usage = 54748364; -- {serverError MEMORY_LIMIT_EXCEEDED} +SELECT count(distinct b) FROM testnull GROUP BY a SETTINGS max_memory_usage = 10000000; -- {serverError MEMORY_LIMIT_EXCEEDED} DROP TABLE testnull; \ No newline at end of file From a592ad3624dedd3080b95422151b17bb92f3c10a Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Wed, 31 Jul 2024 23:06:53 +0000 Subject: [PATCH 036/129] change error code --- src/Storages/MergeTree/MergeTreeData.cpp | 5 +++-- src/Storages/StorageFactory.cpp | 7 ++++--- src/Storages/StorageMergeTree.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- .../0_stateless/03174_projection_deduplicate.sql | 2 +- .../03206_projection_merge_special_mergetree.sql | 14 +++++++------- 6 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 677c4a92cda..7a0980a0e3b 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3216,9 +3216,10 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context { if (auto storage_name = getName(); storage_name != "MergeTree" && storage_name != "ReplicatedMergeTree" && settings_from_storage->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other" - " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode."); + " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode." + " Current storage name is {}.", storage_name); } commands.apply(new_metadata, local_context); diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index a059d624cd8..7360d351e8a 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -20,7 +20,7 @@ namespace ErrorCodes extern const int FUNCTION_CANNOT_HAVE_PARAMETERS; extern const int BAD_ARGUMENTS; extern const int DATA_TYPE_CANNOT_BE_USED_IN_TABLES; - extern const int NOT_IMPLEMENTED; + extern const int SUPPORT_IS_DISABLED; } @@ -223,9 +223,10 @@ StoragePtr StorageFactory::get( } } if (!projection_allowed) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other" - " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode."); + " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode." + " Current storage name is {}.", engine_name); } } } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index a5d434796ba..84393a3f1b0 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1582,7 +1582,7 @@ bool StorageMergeTree::optimize( if (deduplicate && getInMemoryMetadataPtr()->hasProjections() && getSettings()->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "OPTIMIZE DEDUPLICATE query is not supported for table {} as it has projections. " "User should drop all the projections manually before running the query, " "or consider drop or rebuild option of deduplicate_merge_projection_mode", diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 3751883df24..ad578242010 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5796,7 +5796,7 @@ bool StorageReplicatedMergeTree::optimize( if (deduplicate && getInMemoryMetadataPtr()->hasProjections() && getSettings()->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "OPTIMIZE DEDUPLICATE query is not supported for table {} as it has projections. " "User should drop all the projections manually before running the query, " "or consider drop or rebuild option of deduplicate_merge_projection_mode", diff --git a/tests/queries/0_stateless/03174_projection_deduplicate.sql b/tests/queries/0_stateless/03174_projection_deduplicate.sql index 46222b69dc7..f43f0a1f236 100644 --- a/tests/queries/0_stateless/03174_projection_deduplicate.sql +++ b/tests/queries/0_stateless/03174_projection_deduplicate.sql @@ -17,7 +17,7 @@ PRIMARY KEY id; INSERT INTO test_projection_deduplicate VALUES (1, 'one'); INSERT INTO test_projection_deduplicate VALUES (1, 'one'); -OPTIMIZE TABLE test_projection_deduplicate DEDUPLICATE; -- { serverError NOT_IMPLEMENTED } +OPTIMIZE TABLE test_projection_deduplicate DEDUPLICATE; -- { serverError SUPPORT_IS_DISABLED } SELECT * FROM test_projection_deduplicate; diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql index e0a4f4f8cec..d3448138396 100644 --- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql +++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql @@ -9,7 +9,7 @@ CREATE TABLE tp ( INSERT INTO tp SELECT number%3, 1 FROM numbers(3); -OPTIMIZE TABLE tp DEDUPLICATE; -- { serverError NOT_IMPLEMENTED } +OPTIMIZE TABLE tp DEDUPLICATE; -- { serverError SUPPORT_IS_DISABLED } DROP TABLE tp; @@ -26,7 +26,7 @@ OPTIMIZE TABLE tp DEDUPLICATE; ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw'; -OPTIMIZE TABLE tp DEDUPLICATE; -- { serverError NOT_IMPLEMENTED } +OPTIMIZE TABLE tp DEDUPLICATE; -- { serverError SUPPORT_IS_DISABLED } DROP TABLE tp; @@ -36,14 +36,14 @@ CREATE TABLE tp ( type Int32, eventcnt UInt64, PROJECTION p (select sum(eventcnt), type group by type) -) engine = ReplacingMergeTree order by type; -- { serverError NOT_IMPLEMENTED } +) engine = ReplacingMergeTree order by type; -- { serverError SUPPORT_IS_DISABLED } CREATE TABLE tp ( type Int32, eventcnt UInt64, PROJECTION p (select sum(eventcnt), type group by type) ) engine = ReplacingMergeTree order by type -SETTINGS deduplicate_merge_projection_mode = 'throw'; -- { serverError NOT_IMPLEMENTED } +SETTINGS deduplicate_merge_projection_mode = 'throw'; -- { serverError SUPPORT_IS_DISABLED } CREATE TABLE tp ( type Int32, @@ -65,7 +65,7 @@ WHERE (database = currentDatabase()) AND (`table` = 'tp') AND (active = 1); ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw'; -OPTIMIZE TABLE tp DEDUPLICATE; -- { serverError NOT_IMPLEMENTED } +OPTIMIZE TABLE tp DEDUPLICATE; -- { serverError SUPPORT_IS_DISABLED } DROP TABLE tp; @@ -78,7 +78,7 @@ SETTINGS deduplicate_merge_projection_mode = 'rebuild'; ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw'; -OPTIMIZE TABLE tp DEDUPLICATE; -- { serverError NOT_IMPLEMENTED } +OPTIMIZE TABLE tp DEDUPLICATE; -- { serverError SUPPORT_IS_DISABLED } DROP TABLE tp; @@ -88,7 +88,7 @@ CREATE TABLE tp ( eventcnt UInt64 ) engine = ReplacingMergeTree order by type; -ALTER TABLE tp ADD PROJECTION p (SELECT sum(eventcnt), type GROUP BY type); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE tp ADD PROJECTION p (SELECT sum(eventcnt), type GROUP BY type); -- { serverError SUPPORT_IS_DISABLED } ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'drop'; From 928d5df3f02559fab4fd9bbd258f6e9c6ac4c18a Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Thu, 1 Aug 2024 12:32:43 +0200 Subject: [PATCH 037/129] added tests for https://github.com/ClickHouse/ClickHouse/pull/65475 Support writting page index into parquet file --- .../test/integration/runner/requirements.txt | 1 + .../test_parquet_page_index/__init__.py | 0 .../test_parquet_page_index/test.py | 77 +++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 tests/integration/test_parquet_page_index/__init__.py create mode 100644 tests/integration/test_parquet_page_index/test.py diff --git a/docker/test/integration/runner/requirements.txt b/docker/test/integration/runner/requirements.txt index 8a77d8abf77..428986b5562 100644 --- a/docker/test/integration/runner/requirements.txt +++ b/docker/test/integration/runner/requirements.txt @@ -74,6 +74,7 @@ protobuf==4.25.2 psycopg2-binary==2.9.6 py4j==0.10.9.5 py==1.11.0 +pyarrow==17.0.0 pycparser==2.22 pycryptodome==3.20.0 pymongo==3.11.0 diff --git a/tests/integration/test_parquet_page_index/__init__.py b/tests/integration/test_parquet_page_index/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_parquet_page_index/test.py b/tests/integration/test_parquet_page_index/test.py new file mode 100644 index 00000000000..366216ea2c6 --- /dev/null +++ b/tests/integration/test_parquet_page_index/test.py @@ -0,0 +1,77 @@ +import pytest +from helpers.cluster import ClickHouseCluster +import pyarrow.parquet as pq +import os +import time + +cluster = ClickHouseCluster(__file__) +path_to_userfiles = "/var/lib/clickhouse/user_files/" +path_to_external_dirs = "/ClickHouse/tests/integration/test_parquet_page_index/_instances" +node = cluster.add_instance("node", external_dirs=[path_to_userfiles]) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def check_page_index(file_path): + metadata = pq.read_metadata(file_path) + assert metadata, "pyarrow.parquet library can't read parquet file written by Clickhouse" + return metadata.row_group(0).column(0).has_offset_index + + +def delete_if_exists(file_path): + if os.path.exists(file_path): + os.remove(file_path) + + +@pytest.mark.parametrize("query, expected_result", { + ("SElECT number, number+1 FROM system.numbers LIMIT 100 " + "INTO OUTFILE '{file_name}' FORMAT Parquet " + "SETTINGS output_format_parquet_use_custom_encoder = false, " + "output_format_parquet_write_page_index = true;", True), + ("SElECT number, number+1 FROM system.numbers LIMIT 100 " + "INTO OUTFILE '{file_name}' FORMAT Parquet " + "SETTINGS output_format_parquet_use_custom_encoder = false, " + "output_format_parquet_write_page_index = false;", False), + # # default settings: + # # output_format_parquet_use_custom_encoder = true + ("SElECT number, number+1 FROM system.numbers LIMIT 100 " + "INTO OUTFILE '{file_name}' FORMAT Parquet;", False), +}) +def test_parquet_page_index_select_into_outfile(query, expected_result, start_cluster): + file_name = 'export.parquet' + query = query.format(file_name=file_name) + delete_if_exists(file_name) + assert node.query(query) == '' + assert check_page_index(file_name) == expected_result, "Page offset index have wrong value" + delete_if_exists(file_name) + + +@pytest.mark.parametrize("query, expected_result", { + ("INSERT INTO TABLE FUNCTION file('{file_name}') " + "SElECT number, number+1 FROM system.numbers LIMIT 100 " + "SETTINGS output_format_parquet_use_custom_encoder=false, " + "output_format_parquet_write_page_index=true FORMAT Parquet", True), + ("INSERT INTO TABLE FUNCTION file('{file_name}') " + "SElECT number, number+1 FROM system.numbers LIMIT 100 " + "SETTINGS output_format_parquet_use_custom_encoder=false, " + "output_format_parquet_write_page_index=false FORMAT Parquet", False), + # # default settings: + # # output_format_parquet_use_custom_encoder = true + ("INSERT INTO TABLE FUNCTION file('{file_name}') " + "SElECT number, number+1 FROM system.numbers LIMIT 100 FORMAT Parquet", False), +}) +def test_parquet_page_index_insert_into_table_function_file(query, expected_result, start_cluster): + file_name = 'export.parquet' + query = query.format(file_name=file_name) + file_path = f"{path_to_external_dirs}{path_to_userfiles}{file_name}" + delete_if_exists(file_path) + assert node.query(query) == '' + assert check_page_index(file_path) == expected_result, "Page offset index have wrong value" + delete_if_exists(file_path) From 572ad2f6fe193d69e9c2c1c64125b3ffdff1de0d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 1 Aug 2024 11:11:52 +0000 Subject: [PATCH 038/129] Automatic style fix --- .../test_parquet_page_index/test.py | 108 ++++++++++++------ 1 file changed, 71 insertions(+), 37 deletions(-) diff --git a/tests/integration/test_parquet_page_index/test.py b/tests/integration/test_parquet_page_index/test.py index 366216ea2c6..a820d1b5bc5 100644 --- a/tests/integration/test_parquet_page_index/test.py +++ b/tests/integration/test_parquet_page_index/test.py @@ -6,7 +6,9 @@ import time cluster = ClickHouseCluster(__file__) path_to_userfiles = "/var/lib/clickhouse/user_files/" -path_to_external_dirs = "/ClickHouse/tests/integration/test_parquet_page_index/_instances" +path_to_external_dirs = ( + "/ClickHouse/tests/integration/test_parquet_page_index/_instances" +) node = cluster.add_instance("node", external_dirs=[path_to_userfiles]) @@ -21,7 +23,9 @@ def start_cluster(): def check_page_index(file_path): metadata = pq.read_metadata(file_path) - assert metadata, "pyarrow.parquet library can't read parquet file written by Clickhouse" + assert ( + metadata + ), "pyarrow.parquet library can't read parquet file written by Clickhouse" return metadata.row_group(0).column(0).has_offset_index @@ -30,48 +34,78 @@ def delete_if_exists(file_path): os.remove(file_path) -@pytest.mark.parametrize("query, expected_result", { - ("SElECT number, number+1 FROM system.numbers LIMIT 100 " - "INTO OUTFILE '{file_name}' FORMAT Parquet " - "SETTINGS output_format_parquet_use_custom_encoder = false, " - "output_format_parquet_write_page_index = true;", True), - ("SElECT number, number+1 FROM system.numbers LIMIT 100 " - "INTO OUTFILE '{file_name}' FORMAT Parquet " - "SETTINGS output_format_parquet_use_custom_encoder = false, " - "output_format_parquet_write_page_index = false;", False), - # # default settings: - # # output_format_parquet_use_custom_encoder = true - ("SElECT number, number+1 FROM system.numbers LIMIT 100 " - "INTO OUTFILE '{file_name}' FORMAT Parquet;", False), -}) +@pytest.mark.parametrize( + "query, expected_result", + { + ( + "SElECT number, number+1 FROM system.numbers LIMIT 100 " + "INTO OUTFILE '{file_name}' FORMAT Parquet " + "SETTINGS output_format_parquet_use_custom_encoder = false, " + "output_format_parquet_write_page_index = true;", + True, + ), + ( + "SElECT number, number+1 FROM system.numbers LIMIT 100 " + "INTO OUTFILE '{file_name}' FORMAT Parquet " + "SETTINGS output_format_parquet_use_custom_encoder = false, " + "output_format_parquet_write_page_index = false;", + False, + ), + # # default settings: + # # output_format_parquet_use_custom_encoder = true + ( + "SElECT number, number+1 FROM system.numbers LIMIT 100 " + "INTO OUTFILE '{file_name}' FORMAT Parquet;", + False, + ), + }, +) def test_parquet_page_index_select_into_outfile(query, expected_result, start_cluster): - file_name = 'export.parquet' + file_name = "export.parquet" query = query.format(file_name=file_name) delete_if_exists(file_name) - assert node.query(query) == '' - assert check_page_index(file_name) == expected_result, "Page offset index have wrong value" + assert node.query(query) == "" + assert ( + check_page_index(file_name) == expected_result + ), "Page offset index have wrong value" delete_if_exists(file_name) -@pytest.mark.parametrize("query, expected_result", { - ("INSERT INTO TABLE FUNCTION file('{file_name}') " - "SElECT number, number+1 FROM system.numbers LIMIT 100 " - "SETTINGS output_format_parquet_use_custom_encoder=false, " - "output_format_parquet_write_page_index=true FORMAT Parquet", True), - ("INSERT INTO TABLE FUNCTION file('{file_name}') " - "SElECT number, number+1 FROM system.numbers LIMIT 100 " - "SETTINGS output_format_parquet_use_custom_encoder=false, " - "output_format_parquet_write_page_index=false FORMAT Parquet", False), - # # default settings: - # # output_format_parquet_use_custom_encoder = true - ("INSERT INTO TABLE FUNCTION file('{file_name}') " - "SElECT number, number+1 FROM system.numbers LIMIT 100 FORMAT Parquet", False), -}) -def test_parquet_page_index_insert_into_table_function_file(query, expected_result, start_cluster): - file_name = 'export.parquet' +@pytest.mark.parametrize( + "query, expected_result", + { + ( + "INSERT INTO TABLE FUNCTION file('{file_name}') " + "SElECT number, number+1 FROM system.numbers LIMIT 100 " + "SETTINGS output_format_parquet_use_custom_encoder=false, " + "output_format_parquet_write_page_index=true FORMAT Parquet", + True, + ), + ( + "INSERT INTO TABLE FUNCTION file('{file_name}') " + "SElECT number, number+1 FROM system.numbers LIMIT 100 " + "SETTINGS output_format_parquet_use_custom_encoder=false, " + "output_format_parquet_write_page_index=false FORMAT Parquet", + False, + ), + # # default settings: + # # output_format_parquet_use_custom_encoder = true + ( + "INSERT INTO TABLE FUNCTION file('{file_name}') " + "SElECT number, number+1 FROM system.numbers LIMIT 100 FORMAT Parquet", + False, + ), + }, +) +def test_parquet_page_index_insert_into_table_function_file( + query, expected_result, start_cluster +): + file_name = "export.parquet" query = query.format(file_name=file_name) file_path = f"{path_to_external_dirs}{path_to_userfiles}{file_name}" delete_if_exists(file_path) - assert node.query(query) == '' - assert check_page_index(file_path) == expected_result, "Page offset index have wrong value" + assert node.query(query) == "" + assert ( + check_page_index(file_path) == expected_result + ), "Page offset index have wrong value" delete_if_exists(file_path) From 4adc9523e403ab103ed3dec537b02566287b76ee Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Thu, 1 Aug 2024 14:38:07 +0200 Subject: [PATCH 039/129] added a stateless test for ENGINE=File(Parquet) --- .../0_stateless/03215_parquet_index.reference | 300 ++++++++++++++++++ .../0_stateless/03215_parquet_index.sql | 17 + 2 files changed, 317 insertions(+) create mode 100644 tests/queries/0_stateless/03215_parquet_index.reference create mode 100644 tests/queries/0_stateless/03215_parquet_index.sql diff --git a/tests/queries/0_stateless/03215_parquet_index.reference b/tests/queries/0_stateless/03215_parquet_index.reference new file mode 100644 index 00000000000..334f2f3824e --- /dev/null +++ b/tests/queries/0_stateless/03215_parquet_index.reference @@ -0,0 +1,300 @@ +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +16 16 +17 17 +18 18 +19 19 +20 20 +21 21 +22 22 +23 23 +24 24 +25 25 +26 26 +27 27 +28 28 +29 29 +30 30 +31 31 +32 32 +33 33 +34 34 +35 35 +36 36 +37 37 +38 38 +39 39 +40 40 +41 41 +42 42 +43 43 +44 44 +45 45 +46 46 +47 47 +48 48 +49 49 +50 50 +51 51 +52 52 +53 53 +54 54 +55 55 +56 56 +57 57 +58 58 +59 59 +60 60 +61 61 +62 62 +63 63 +64 64 +65 65 +66 66 +67 67 +68 68 +69 69 +70 70 +71 71 +72 72 +73 73 +74 74 +75 75 +76 76 +77 77 +78 78 +79 79 +80 80 +81 81 +82 82 +83 83 +84 84 +85 85 +86 86 +87 87 +88 88 +89 89 +90 90 +91 91 +92 92 +93 93 +94 94 +95 95 +96 96 +97 97 +98 98 +99 99 +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +16 16 +17 17 +18 18 +19 19 +20 20 +21 21 +22 22 +23 23 +24 24 +25 25 +26 26 +27 27 +28 28 +29 29 +30 30 +31 31 +32 32 +33 33 +34 34 +35 35 +36 36 +37 37 +38 38 +39 39 +40 40 +41 41 +42 42 +43 43 +44 44 +45 45 +46 46 +47 47 +48 48 +49 49 +50 50 +51 51 +52 52 +53 53 +54 54 +55 55 +56 56 +57 57 +58 58 +59 59 +60 60 +61 61 +62 62 +63 63 +64 64 +65 65 +66 66 +67 67 +68 68 +69 69 +70 70 +71 71 +72 72 +73 73 +74 74 +75 75 +76 76 +77 77 +78 78 +79 79 +80 80 +81 81 +82 82 +83 83 +84 84 +85 85 +86 86 +87 87 +88 88 +89 89 +90 90 +91 91 +92 92 +93 93 +94 94 +95 95 +96 96 +97 97 +98 98 +99 99 +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +16 16 +17 17 +18 18 +19 19 +20 20 +21 21 +22 22 +23 23 +24 24 +25 25 +26 26 +27 27 +28 28 +29 29 +30 30 +31 31 +32 32 +33 33 +34 34 +35 35 +36 36 +37 37 +38 38 +39 39 +40 40 +41 41 +42 42 +43 43 +44 44 +45 45 +46 46 +47 47 +48 48 +49 49 +50 50 +51 51 +52 52 +53 53 +54 54 +55 55 +56 56 +57 57 +58 58 +59 59 +60 60 +61 61 +62 62 +63 63 +64 64 +65 65 +66 66 +67 67 +68 68 +69 69 +70 70 +71 71 +72 72 +73 73 +74 74 +75 75 +76 76 +77 77 +78 78 +79 79 +80 80 +81 81 +82 82 +83 83 +84 84 +85 85 +86 86 +87 87 +88 88 +89 89 +90 90 +91 91 +92 92 +93 93 +94 94 +95 95 +96 96 +97 97 +98 98 +99 99 diff --git a/tests/queries/0_stateless/03215_parquet_index.sql b/tests/queries/0_stateless/03215_parquet_index.sql new file mode 100644 index 00000000000..5b176ff70ba --- /dev/null +++ b/tests/queries/0_stateless/03215_parquet_index.sql @@ -0,0 +1,17 @@ +-- default settings. +DROP TABLE IF EXISTS test_parquet; +CREATE TABLE test_parquet (col1 int, col2 String) ENGINE=File(Parquet); +INSERT INTO test_parquet SELECT number, toString(number) FROM numbers(100); +SELECT col1, col2 FROM test_parquet; + +-- Parquet will have indexes in columns. We are not checking that indexes exist here, there is an integration test test_parquet_page_index for that. We just check that a setting doesn't break the SELECT +DROP TABLE IF EXISTS test_parquet; +CREATE TABLE test_parquet (col1 int, col2 String) ENGINE=File(Parquet) SETTINGS output_format_parquet_use_custom_encoder=false, output_format_parquet_write_page_index=true; +INSERT INTO test_parquet SELECT number, toString(number) FROM numbers(100); +SELECT col1, col2 FROM test_parquet; + +-- Parquet will not have indexes in columns. +DROP TABLE IF EXISTS test_parquet; +CREATE TABLE test_parquet (col1 int, col2 String) ENGINE=File(Parquet) SETTINGS output_format_parquet_use_custom_encoder=false, output_format_parquet_write_page_index=false; +INSERT INTO test_parquet SELECT number, toString(number) FROM numbers(100); +SELECT col1, col2 FROM test_parquet; From a8e7c8ae9a5e345fa1725d664fd490891e4fa9a5 Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Thu, 1 Aug 2024 14:45:46 +0200 Subject: [PATCH 040/129] fix for parallel execution --- tests/integration/test_parquet_page_index/test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_parquet_page_index/test.py b/tests/integration/test_parquet_page_index/test.py index a820d1b5bc5..db291e20b74 100644 --- a/tests/integration/test_parquet_page_index/test.py +++ b/tests/integration/test_parquet_page_index/test.py @@ -61,7 +61,7 @@ def delete_if_exists(file_path): }, ) def test_parquet_page_index_select_into_outfile(query, expected_result, start_cluster): - file_name = "export.parquet" + file_name = f"export{time.time()}.parquet" query = query.format(file_name=file_name) delete_if_exists(file_name) assert node.query(query) == "" @@ -76,14 +76,14 @@ def test_parquet_page_index_select_into_outfile(query, expected_result, start_cl { ( "INSERT INTO TABLE FUNCTION file('{file_name}') " - "SElECT number, number+1 FROM system.numbers LIMIT 100 " + "SELECT number, number+1 FROM system.numbers LIMIT 100 " "SETTINGS output_format_parquet_use_custom_encoder=false, " "output_format_parquet_write_page_index=true FORMAT Parquet", True, ), ( "INSERT INTO TABLE FUNCTION file('{file_name}') " - "SElECT number, number+1 FROM system.numbers LIMIT 100 " + "SELECT number, number+1 FROM system.numbers LIMIT 100 " "SETTINGS output_format_parquet_use_custom_encoder=false, " "output_format_parquet_write_page_index=false FORMAT Parquet", False, @@ -92,7 +92,7 @@ def test_parquet_page_index_select_into_outfile(query, expected_result, start_cl # # output_format_parquet_use_custom_encoder = true ( "INSERT INTO TABLE FUNCTION file('{file_name}') " - "SElECT number, number+1 FROM system.numbers LIMIT 100 FORMAT Parquet", + "SELECT number, number+1 FROM system.numbers LIMIT 100 FORMAT Parquet", False, ), }, @@ -100,7 +100,7 @@ def test_parquet_page_index_select_into_outfile(query, expected_result, start_cl def test_parquet_page_index_insert_into_table_function_file( query, expected_result, start_cluster ): - file_name = "export.parquet" + file_name = f"export{time.time()}.parquet" query = query.format(file_name=file_name) file_path = f"{path_to_external_dirs}{path_to_userfiles}{file_name}" delete_if_exists(file_path) From 4e9761acf93a58a93186f59d3ad083fd438329dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 1 Aug 2024 18:59:17 +0200 Subject: [PATCH 041/129] Don't run ASAN unit tests under gdb --- docker/test/unit/Dockerfile | 2 +- docker/test/unit/run.sh | 21 ++++++++++++++++++++- tests/ci/unit_tests_check.py | 5 ++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile index af44dc930b2..5f907d94d39 100644 --- a/docker/test/unit/Dockerfile +++ b/docker/test/unit/Dockerfile @@ -4,4 +4,4 @@ ARG FROM_TAG=latest FROM clickhouse/test-base:$FROM_TAG COPY run.sh / -CMD ["/bin/bash", "/run.sh"] +ENTRYPOINT ["/run.sh"] diff --git a/docker/test/unit/run.sh b/docker/test/unit/run.sh index 7323c384d9c..ba11f568218 100644 --- a/docker/test/unit/run.sh +++ b/docker/test/unit/run.sh @@ -2,4 +2,23 @@ set -x -timeout 40m gdb -q -ex 'set print inferior-events off' -ex 'set confirm off' -ex 'set print thread-events off' -ex run -ex bt -ex quit --args ./unit_tests_dbms --gtest_output='json:test_output/test_result.json' | tee test_output/test_result.txt +if [ "$#" -ne 1 ]; then + echo "Expected exactly one argument" + exit 1 +fi + +if [ "$1" = "GDB" ]; +then + timeout 40m \ + gdb -q -ex "set print inferior-events off" -ex "set confirm off" -ex "set print thread-events off" -ex run -ex bt -ex quit --args \ + ./unit_tests_dbms --gtest_output='json:test_output/test_result.json' \ + | tee test_output/test_result.txt +elif [ "$1" = "NO_GDB" ]; +then + timeout 40m \ + ./unit_tests_dbms --gtest_output='json:test_output/test_result.json' \ + | tee test_output/test_result.txt +else + echo "Unknown argument: $1" + exit 1 +fi diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py index b66a4312657..716625d7077 100644 --- a/tests/ci/unit_tests_check.py +++ b/tests/ci/unit_tests_check.py @@ -174,10 +174,13 @@ def main(): test_output = temp_path / "test_output" test_output.mkdir(parents=True, exist_ok=True) + # Don't run ASAN under gdb since that breaks leak detection + gdb_enabled = "NO_GDB" if "asan" in check_name else "GDB" + run_command = ( f"docker run --cap-add=SYS_PTRACE --volume={tests_binary}:/unit_tests_dbms " "--security-opt seccomp=unconfined " # required to issue io_uring sys-calls - f"--volume={test_output}:/test_output {docker_image}" + f"--volume={test_output}:/test_output {docker_image} ${gdb_enabled}" ) run_log_path = test_output / "run.log" From c5a8653daf7dc9cb1031c2ac4b2be3623117848d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 1 Aug 2024 19:27:12 +0200 Subject: [PATCH 042/129] Playing with Docker and the CI --- docker/test/unit/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile index 5f907d94d39..9f4b86aa0ca 100644 --- a/docker/test/unit/Dockerfile +++ b/docker/test/unit/Dockerfile @@ -4,4 +4,5 @@ ARG FROM_TAG=latest FROM clickhouse/test-base:$FROM_TAG COPY run.sh / +RUN chmod +x run.sh ENTRYPOINT ["/run.sh"] From 35b6112b7bb06f18cd5b07860ec1b9c6ce38014f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 1 Aug 2024 19:50:13 +0200 Subject: [PATCH 043/129] Try to stop on leaks --- docker/test/base/Dockerfile | 2 ++ docker/test/stateless/run.sh | 2 +- tests/queries/shell_config.sh | 3 --- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index a81826ed6b5..38c4df459ae 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -28,12 +28,14 @@ RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 abort_on_error=1 history_ RUN echo "UBSAN_OPTIONS='print_stacktrace=1 max_allocation_size_mb=32768'" >> /etc/environment RUN echo "MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1 max_allocation_size_mb=32768'" >> /etc/environment RUN echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt max_allocation_size_mb=32768'" >> /etc/environment +RUN echo "ASAN_OPTIONS='halt_on_error=1 abort_on_error=1'" >> /etc/environment # Sanitizer options for current shell (not current, but the one that will be spawned on "docker run") # (but w/o verbosity for TSAN, otherwise test.reference will not match) ENV TSAN_OPTIONS='halt_on_error=1 abort_on_error=1 history_size=7 memory_limit_mb=46080 second_deadlock_stack=1 max_allocation_size_mb=32768' ENV UBSAN_OPTIONS='print_stacktrace=1 max_allocation_size_mb=32768' ENV MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1 max_allocation_size_mb=32768' ENV LSAN_OPTIONS='max_allocation_size_mb=32768' +ENV ASAN_OPTIONS='halt_on_error=1 abort_on_error=1' # for external_symbolizer_path RUN ln -s /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index f9f96c76d59..c359d8a1847 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -164,7 +164,7 @@ do done setup_logs_replication -attach_gdb_to_clickhouse || true # FIXME: to not break old builds, clean on 2023-09-01 +attach_gdb_to_clickhouse function fn_exists() { declare -F "$1" > /dev/null; diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index f7017958635..9e5bf75d335 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -1,9 +1,6 @@ #!/usr/bin/env bash # shellcheck disable=SC2120 -# Don't check for ODR violation, since we may test shared build with ASAN -export ASAN_OPTIONS=detect_odr_violation=0 - # If ClickHouse was built with coverage - dump the coverage information at exit # (in other cases this environment variable has no effect) export CLICKHOUSE_WRITE_COVERAGE="coverage" From 900e08d6e7c433758d13f5a669c2112bb3856007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 2 Aug 2024 11:32:09 +0200 Subject: [PATCH 044/129] Try fix --- tests/ci/unit_tests_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py index 716625d7077..6430fa78801 100644 --- a/tests/ci/unit_tests_check.py +++ b/tests/ci/unit_tests_check.py @@ -180,7 +180,7 @@ def main(): run_command = ( f"docker run --cap-add=SYS_PTRACE --volume={tests_binary}:/unit_tests_dbms " "--security-opt seccomp=unconfined " # required to issue io_uring sys-calls - f"--volume={test_output}:/test_output {docker_image} ${gdb_enabled}" + f"--volume={test_output}:/test_output {docker_image} {gdb_enabled}" ) run_log_path = test_output / "run.log" From e2b686efea175e4ddc6472849934aa953f13138d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 2 Aug 2024 12:26:38 +0200 Subject: [PATCH 045/129] Fix test --- src/Common/tests/gtest_lsan.cpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Common/tests/gtest_lsan.cpp b/src/Common/tests/gtest_lsan.cpp index f6e1984ec58..7fc4ad2749e 100644 --- a/src/Common/tests/gtest_lsan.cpp +++ b/src/Common/tests/gtest_lsan.cpp @@ -14,20 +14,21 @@ /// because of broken getauxval() [1]. /// /// [1]: https://github.com/ClickHouse/ClickHouse/pull/33957 -TEST(Common, LSan) +TEST(SanitizerDeathTest, LSan) { - int sanitizers_exit_code = 1; - - ASSERT_EXIT({ - std::thread leak_in_thread([]() + EXPECT_DEATH( { - void * leak = malloc(4096); - ASSERT_NE(leak, nullptr); - }); - leak_in_thread.join(); + std::thread leak_in_thread( + []() + { + void * leak = malloc(4096); + ASSERT_NE(leak, nullptr); + }); + leak_in_thread.join(); - __lsan_do_leak_check(); - }, ::testing::ExitedWithCode(sanitizers_exit_code), ".*LeakSanitizer: detected memory leaks.*"); + __lsan_do_leak_check(); + }, + ".*LeakSanitizer: detected memory leaks.*"); } #endif From 97f1f6e22174916cc3b401fcebdf01dcd0fb0107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 2 Aug 2024 12:46:37 +0200 Subject: [PATCH 046/129] Don't hide errors on clickhouse local runs --- docker/test/stateless/stress_tests.lib | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/test/stateless/stress_tests.lib b/docker/test/stateless/stress_tests.lib index 682da1df837..2a833b17f14 100644 --- a/docker/test/stateless/stress_tests.lib +++ b/docker/test/stateless/stress_tests.lib @@ -308,7 +308,8 @@ function collect_query_and_trace_logs() { for table in query_log trace_log metric_log do - clickhouse-local --config-file=/etc/clickhouse-server/config.xml --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst ||: + # Don't ignore errors here, it leads to ignore sanitizer reports when running clickhouse-local + clickhouse-local --config-file=/etc/clickhouse-server/config.xml --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst done } From e5cf376c4ff3742fb0d3127ef6b50e08180eb153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 2 Aug 2024 12:56:49 +0200 Subject: [PATCH 047/129] Don't attach gdb in ASAN runs of fuzzer, stateless or stress checks --- docker/test/fuzzer/run-fuzzer.sh | 106 +++++++++++++++------------ docker/test/stateless/attach_gdb.lib | 78 +++++++++++--------- 2 files changed, 101 insertions(+), 83 deletions(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index b8f967ed9c2..ae1b9e94bed 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -193,53 +193,60 @@ function fuzz kill -0 $server_pid - # Set follow-fork-mode to parent, because we attach to clickhouse-server, not to watchdog - # and clickhouse-server can do fork-exec, for example, to run some bridge. - # Do not set nostop noprint for all signals, because some it may cause gdb to hang, - # explicitly ignore non-fatal signals that are used by server. - # Number of SIGRTMIN can be determined only in runtime. - RTMIN=$(kill -l SIGRTMIN) - echo " -set follow-fork-mode parent -handle SIGHUP nostop noprint pass -handle SIGINT nostop noprint pass -handle SIGQUIT nostop noprint pass -handle SIGPIPE nostop noprint pass -handle SIGTERM nostop noprint pass -handle SIGUSR1 nostop noprint pass -handle SIGUSR2 nostop noprint pass -handle SIG$RTMIN nostop noprint pass -info signals -continue -backtrace full -thread apply all backtrace full -info registers -disassemble /s -up -disassemble /s -up -disassemble /s -p \"done\" -detach -quit -" > script.gdb + IS_ASAN=$(clickhouse-client --query "SELECT count() FROM system.build_options WHERE name = 'CXX_FLAGS' AND position('sanitize=address' IN value)") + if [[ "$IS_ASAN" = "1" ]]; + then + echo "ASAN build detected. Not using gdb since it disables LeakSanitizer detections" + else + # Set follow-fork-mode to parent, because we attach to clickhouse-server, not to watchdog + # and clickhouse-server can do fork-exec, for example, to run some bridge. + # Do not set nostop noprint for all signals, because some it may cause gdb to hang, + # explicitly ignore non-fatal signals that are used by server. + # Number of SIGRTMIN can be determined only in runtime. + RTMIN=$(kill -l SIGRTMIN) + echo " + set follow-fork-mode parent + handle SIGHUP nostop noprint pass + handle SIGINT nostop noprint pass + handle SIGQUIT nostop noprint pass + handle SIGPIPE nostop noprint pass + handle SIGTERM nostop noprint pass + handle SIGUSR1 nostop noprint pass + handle SIGUSR2 nostop noprint pass + handle SIG$RTMIN nostop noprint pass + info signals + continue + backtrace full + thread apply all backtrace full + info registers + disassemble /s + up + disassemble /s + up + disassemble /s + p \"done\" + detach + quit + " > script.gdb - gdb -batch -command script.gdb -p $server_pid & - sleep 5 - # gdb will send SIGSTOP, spend some time loading debug info, and then send SIGCONT, wait for it (up to send_timeout, 300s) - time clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'" ||: + gdb -batch -command script.gdb -p $server_pid & + sleep 5 + # gdb will send SIGSTOP, spend some time loading debug info, and then send SIGCONT, wait for it (up to send_timeout, 300s) + time clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'" ||: + + # Check connectivity after we attach gdb, because it might cause the server + # to freeze, and the fuzzer will fail. In debug build, it can take a lot of time. + for _ in {1..180} + do + if clickhouse-client --query "select 1" + then + break + fi + sleep 1 + done + kill -0 $server_pid # This checks that it is our server that is started and not some other one + fi - # Check connectivity after we attach gdb, because it might cause the server - # to freeze, and the fuzzer will fail. In debug build, it can take a lot of time. - for _ in {1..180} - do - if clickhouse-client --query "select 1" - then - break - fi - sleep 1 - done - kill -0 $server_pid # This checks that it is our server that is started and not some other one echo 'Server started and responded.' setup_logs_replication @@ -264,8 +271,13 @@ quit # The fuzzer_pid belongs to the timeout process. actual_fuzzer_pid=$(ps -o pid= --ppid "$fuzzer_pid") - echo "Attaching gdb to the fuzzer itself" - gdb -batch -command script.gdb -p $actual_fuzzer_pid & + if [[ "$IS_ASAN" = "1" ]]; + then + echo "ASAN build detected. Not using gdb since it disables LeakSanitizer detections" + else + echo "Attaching gdb to the fuzzer itself" + gdb -batch -command script.gdb -p $actual_fuzzer_pid & + fi # Wait for the fuzzer to complete. # Note that the 'wait || ...' thing is required so that the script doesn't diff --git a/docker/test/stateless/attach_gdb.lib b/docker/test/stateless/attach_gdb.lib index d288288bb17..d0cac24481f 100644 --- a/docker/test/stateless/attach_gdb.lib +++ b/docker/test/stateless/attach_gdb.lib @@ -5,43 +5,49 @@ source /utils.lib function attach_gdb_to_clickhouse() { - # Set follow-fork-mode to parent, because we attach to clickhouse-server, not to watchdog - # and clickhouse-server can do fork-exec, for example, to run some bridge. - # Do not set nostop noprint for all signals, because some it may cause gdb to hang, - # explicitly ignore non-fatal signals that are used by server. - # Number of SIGRTMIN can be determined only in runtime. - RTMIN=$(kill -l SIGRTMIN) - echo " -set follow-fork-mode parent -handle SIGHUP nostop noprint pass -handle SIGINT nostop noprint pass -handle SIGQUIT nostop noprint pass -handle SIGPIPE nostop noprint pass -handle SIGTERM nostop noprint pass -handle SIGUSR1 nostop noprint pass -handle SIGUSR2 nostop noprint pass -handle SIG$RTMIN nostop noprint pass -info signals -continue -backtrace full -thread apply all backtrace full -info registers -disassemble /s -up -disassemble /s -up -disassemble /s -p \"done\" -detach -quit -" > script.gdb + IS_ASAN=$(clickhouse-client --query "SELECT count() FROM system.build_options WHERE name = 'CXX_FLAGS' AND position('sanitize=address' IN value)") + if [[ "$IS_ASAN" = "1" ]]; + then + echo "ASAN build detected. Not using gdb since it disables LeakSanitizer detections" + else + # Set follow-fork-mode to parent, because we attach to clickhouse-server, not to watchdog + # and clickhouse-server can do fork-exec, for example, to run some bridge. + # Do not set nostop noprint for all signals, because some it may cause gdb to hang, + # explicitly ignore non-fatal signals that are used by server. + # Number of SIGRTMIN can be determined only in runtime. + RTMIN=$(kill -l SIGRTMIN) + echo " + set follow-fork-mode parent + handle SIGHUP nostop noprint pass + handle SIGINT nostop noprint pass + handle SIGQUIT nostop noprint pass + handle SIGPIPE nostop noprint pass + handle SIGTERM nostop noprint pass + handle SIGUSR1 nostop noprint pass + handle SIGUSR2 nostop noprint pass + handle SIG$RTMIN nostop noprint pass + info signals + continue + backtrace full + thread apply all backtrace full + info registers + disassemble /s + up + disassemble /s + up + disassemble /s + p \"done\" + detach + quit + " > script.gdb - # FIXME Hung check may work incorrectly because of attached gdb - # We cannot attach another gdb to get stacktraces if some queries hung - gdb -batch -command script.gdb -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log & - sleep 5 - # gdb will send SIGSTOP, spend some time loading debug info and then send SIGCONT, wait for it (up to send_timeout, 300s) - run_with_retry 60 clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'" + # FIXME Hung check may work incorrectly because of attached gdb + # We cannot attach another gdb to get stacktraces if some queries hung + gdb -batch -command script.gdb -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log & + sleep 5 + # gdb will send SIGSTOP, spend some time loading debug info and then send SIGCONT, wait for it (up to send_timeout, 300s) + run_with_retry 60 clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'" + fi } # vi: ft=bash From c9b29ad11351b774d834ada642951a81d33b14e7 Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Fri, 2 Aug 2024 13:53:48 +0200 Subject: [PATCH 048/129] squash! fix for parallel execution --- tests/integration/test_parquet_page_index/test.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_parquet_page_index/test.py b/tests/integration/test_parquet_page_index/test.py index db291e20b74..59dbab09be5 100644 --- a/tests/integration/test_parquet_page_index/test.py +++ b/tests/integration/test_parquet_page_index/test.py @@ -6,9 +6,6 @@ import time cluster = ClickHouseCluster(__file__) path_to_userfiles = "/var/lib/clickhouse/user_files/" -path_to_external_dirs = ( - "/ClickHouse/tests/integration/test_parquet_page_index/_instances" -) node = cluster.add_instance("node", external_dirs=[path_to_userfiles]) @@ -45,7 +42,7 @@ def delete_if_exists(file_path): True, ), ( - "SElECT number, number+1 FROM system.numbers LIMIT 100 " + "SELECT number, number+1 FROM system.numbers LIMIT 100 " "INTO OUTFILE '{file_name}' FORMAT Parquet " "SETTINGS output_format_parquet_use_custom_encoder = false, " "output_format_parquet_write_page_index = false;", @@ -54,7 +51,7 @@ def delete_if_exists(file_path): # # default settings: # # output_format_parquet_use_custom_encoder = true ( - "SElECT number, number+1 FROM system.numbers LIMIT 100 " + "SELECT number, number+1 FROM system.numbers LIMIT 100 " "INTO OUTFILE '{file_name}' FORMAT Parquet;", False, ), @@ -102,7 +99,7 @@ def test_parquet_page_index_insert_into_table_function_file( ): file_name = f"export{time.time()}.parquet" query = query.format(file_name=file_name) - file_path = f"{path_to_external_dirs}{path_to_userfiles}{file_name}" + file_path = f"{cluster.instances_dir}{path_to_userfiles}{file_name}" delete_if_exists(file_path) assert node.query(query) == "" assert ( From a94df1fb9af032c8ccccb704186513561e28e11c Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 31 Jul 2024 23:50:48 +0200 Subject: [PATCH 049/129] enable parallel_view_processing in perf tests --- tests/performance/materialized_view_parallel_insert.xml | 4 ++++ tests/performance/views_max_insert_threads.xml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tests/performance/materialized_view_parallel_insert.xml b/tests/performance/materialized_view_parallel_insert.xml index df0a23182c3..c3e7cdc50c0 100644 --- a/tests/performance/materialized_view_parallel_insert.xml +++ b/tests/performance/materialized_view_parallel_insert.xml @@ -1,4 +1,8 @@ + + 1 + + CREATE MATERIALIZED VIEW hits_mv ENGINE MergeTree PARTITION BY toYYYYMM(EventDate) diff --git a/tests/performance/views_max_insert_threads.xml b/tests/performance/views_max_insert_threads.xml index 2988984f5d8..c16fb330b35 100644 --- a/tests/performance/views_max_insert_threads.xml +++ b/tests/performance/views_max_insert_threads.xml @@ -1,5 +1,9 @@ + + 1 + + create table views_max_insert_threads_null (a UInt64) Engine = Null create materialized view views_max_insert_threads_mv Engine = Null AS select now() as ts, max(a) from views_max_insert_threads_null group by ts From c3a1381d70e6d5da59e4b29e7a147e24e7fcd90b Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 1 Aug 2024 00:10:59 +0200 Subject: [PATCH 050/129] some corner case optimize --- src/Interpreters/Squashing.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Interpreters/Squashing.cpp b/src/Interpreters/Squashing.cpp index 488177c3b4f..483112df6cb 100644 --- a/src/Interpreters/Squashing.cpp +++ b/src/Interpreters/Squashing.cpp @@ -106,6 +106,17 @@ Chunk Squashing::convertToChunk(CurrentData && data) const Chunk Squashing::squash(std::vector && input_chunks, Chunk::ChunkInfoCollection && infos) { + if (input_chunks.size() == 1) + { + /// this is just optimization, no logic changes + Chunk result = std::move(input_chunks.front()); + infos.appendIfUniq(std::move(result.getChunkInfos())); + result.setChunkInfos(infos); + + chassert(result); + return result; + } + std::vector mutable_columns = {}; size_t rows = 0; for (const Chunk & chunk : input_chunks) From 86bab5a78a9593862815869c5964f557159352ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 2 Aug 2024 16:09:43 +0200 Subject: [PATCH 051/129] Unit test: Don't return ok if leaks are detected --- docker/test/unit/run.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/test/unit/run.sh b/docker/test/unit/run.sh index ba11f568218..210f31170a5 100644 --- a/docker/test/unit/run.sh +++ b/docker/test/unit/run.sh @@ -1,6 +1,9 @@ #!/bin/bash set -x +# Need to keep error from tests after `tee`. Otherwise we don't alert on asan errors +set -o pipefail +set -e if [ "$#" -ne 1 ]; then echo "Expected exactly one argument" From 28ec383739d9a4974549e8c6491797f0eaafaffb Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Sat, 3 Aug 2024 02:07:24 +0000 Subject: [PATCH 052/129] add sharedmergetree --- src/Storages/MergeTree/MergeTreeData.cpp | 5 +++-- src/Storages/StorageFactory.cpp | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index a158e375ae9..e849c4b794f 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3216,10 +3216,11 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context /// Block the case of alter table add projection for special merge trees. if (std::any_of(commands.begin(), commands.end(), [](const AlterCommand & c) { return c.type == AlterCommand::ADD_PROJECTION; })) { - if (auto storage_name = getName(); storage_name != "MergeTree" && storage_name != "ReplicatedMergeTree" + const std::unordered_set allowed_storages{"MergeTree", "ReplicatedMergeTree", "SharedMergeTree"}; + if (auto storage_name = getName(); !allowed_storages.contains(storage_name) && settings_from_storage->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, - "Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other" + "Projection is fully supported in (Replictaed, Shared)MergeTree, but also allowed in non-throw mode with other" " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode." " Current storage name is {}.", storage_name); } diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 7360d351e8a..557f53a9ada 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -204,10 +204,10 @@ StoragePtr StorageFactory::get( /// Now let's handle the merge tree family. Note we only handle in the mode of CREATE due to backward compatibility. /// Otherwise, it would fail to start in the case of existing projections with special mergetree. - /// Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other mergetree family members. chassert(query.storage->engine); - if (std::string_view engine_name(query.storage->engine->name); mode == LoadingStrictnessLevel::CREATE - && engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree") + const std::unordered_set allowed_engines{"MergeTree", "ReplicatedMergeTree", "SharedMergeTree"}; + if (auto engine_name(query.storage->engine->name); mode == LoadingStrictnessLevel::CREATE + && !allowed_engines.contains(engine_name)) { /// default throw mode in deduplicate_merge_projection_mode bool projection_allowed = false; @@ -224,7 +224,7 @@ StoragePtr StorageFactory::get( } if (!projection_allowed) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, - "Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other" + "Projection is fully supported in (Replictaed, Shared)MergeTree, but also allowed in non-throw mode with other" " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode." " Current storage name is {}.", engine_name); } From abb747498d04fd6d6cac71d6097741b41bf572e3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 3 Aug 2024 16:54:44 +0200 Subject: [PATCH 053/129] Fix test `02789_reading_from_s3_with_connection_pool` --- ...89_reading_from_s3_with_connection_pool.sh | 84 +++++++++++-------- 1 file changed, 50 insertions(+), 34 deletions(-) diff --git a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh index 751b2798243..39399842db1 100755 --- a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh +++ b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh @@ -14,43 +14,59 @@ SETTINGS disk = 's3_disk', min_bytes_for_wide_part = 0; INSERT INTO test_s3 SELECT number, number FROM numbers_mt(1e7); " -query="SELECT a, b FROM test_s3" -query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1) -${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" -${CLICKHOUSE_CLIENT} -nm --query " -SELECT ProfileEvents['DiskConnectionsPreserved'] > 0 -FROM system.query_log -WHERE type = 'QueryFinish' - AND current_database = currentDatabase() - AND query_id='$query_id'; -" + +# This (reusing connections from the pool) is not guaranteed to always happen, +# (due to random time difference between the queries and random activity in parallel) +# but should happen most of the time. + +while true +do + query="SELECT a, b FROM test_s3" + query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1) + ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" + + RES=$(${CLICKHOUSE_CLIENT} -nm --query " + SELECT ProfileEvents['DiskConnectionsPreserved'] > 0 + FROM system.query_log + WHERE type = 'QueryFinish' + AND current_database = currentDatabase() + AND query_id='$query_id'; + ") + + [[ $RES -eq 1 ]] && echo "$RES" && break; +done # Test connection pool in ReadWriteBufferFromHTTP -query_id=$(${CLICKHOUSE_CLIENT} -nq " -create table mut (n int, m int, k int) engine=ReplicatedMergeTree('/test/02441/{database}/mut', '1') order by n; -set insert_keeper_fault_injection_probability=0; -insert into mut values (1, 2, 3), (10, 20, 30); +while true +do + query_id=$(${CLICKHOUSE_CLIENT} -nq " + create table mut (n int, m int, k int) engine=ReplicatedMergeTree('/test/02441/{database}/mut', '1') order by n; + set insert_keeper_fault_injection_probability=0; + insert into mut values (1, 2, 3), (10, 20, 30); -system stop merges mut; -alter table mut delete where n = 10; + system stop merges mut; + alter table mut delete where n = 10; -select queryID() from( - -- a funny way to wait for a MUTATE_PART to be assigned - select sleepEachRow(2) from url('http://localhost:8123/?param_tries={1..10}&query=' || encodeURLComponent( - 'select 1 where ''MUTATE_PART'' not in (select type from system.replication_queue where database=''' || currentDatabase() || ''' and table=''mut'')' - ), 'LineAsString', 's String') - -- queryID() will be returned for each row, since the query above doesn't return anything we need to return a fake row - union all - select 1 -) limit 1 settings max_threads=1; -" 2>&1) -${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" -${CLICKHOUSE_CLIENT} -nm --query " -SELECT ProfileEvents['StorageConnectionsPreserved'] > 0 -FROM system.query_log -WHERE type = 'QueryFinish' - AND current_database = currentDatabase() - AND query_id='$query_id'; -" + select queryID() from( + -- a funny way to wait for a MUTATE_PART to be assigned + select sleepEachRow(2) from url('http://localhost:8123/?param_tries={1..10}&query=' || encodeURLComponent( + 'select 1 where ''MUTATE_PART'' not in (select type from system.replication_queue where database=''' || currentDatabase() || ''' and table=''mut'')' + ), 'LineAsString', 's String') + -- queryID() will be returned for each row, since the query above doesn't return anything we need to return a fake row + union all + select 1 + ) limit 1 settings max_threads=1; + " 2>&1) + ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" + RES=$(${CLICKHOUSE_CLIENT} -nm --query " + SELECT ProfileEvents['StorageConnectionsPreserved'] > 0 + FROM system.query_log + WHERE type = 'QueryFinish' + AND current_database = currentDatabase() + AND query_id='$query_id'; + ") + + [[ $RES -eq 1 ]] && echo "$RES" && break; +done From e0362b00f5a24ea19e16b27b71963efe64a174c0 Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Mon, 5 Aug 2024 11:52:07 +0200 Subject: [PATCH 054/129] squash! fix for parallel execution --- tests/integration/test_parquet_page_index/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_parquet_page_index/test.py b/tests/integration/test_parquet_page_index/test.py index 59dbab09be5..778b6618a61 100644 --- a/tests/integration/test_parquet_page_index/test.py +++ b/tests/integration/test_parquet_page_index/test.py @@ -35,7 +35,7 @@ def delete_if_exists(file_path): "query, expected_result", { ( - "SElECT number, number+1 FROM system.numbers LIMIT 100 " + "SELECT number, number+1 FROM system.numbers LIMIT 100 " "INTO OUTFILE '{file_name}' FORMAT Parquet " "SETTINGS output_format_parquet_use_custom_encoder = false, " "output_format_parquet_write_page_index = true;", From b53e757656e298bb308862a8294cde5718e37580 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 24 Jul 2024 10:01:50 +0000 Subject: [PATCH 055/129] Check argument types in DataTypeAggregateFunction ctor --- src/DataTypes/DataTypeAggregateFunction.cpp | 27 +++++++++++++++++++++ src/DataTypes/DataTypeAggregateFunction.h | 8 +----- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp index 09175617bf1..ee42e4fea11 100644 --- a/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/src/DataTypes/DataTypeAggregateFunction.cpp @@ -33,6 +33,33 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } + +DataTypeAggregateFunction::DataTypeAggregateFunction(AggregateFunctionPtr function_, const DataTypes & argument_types_, + const Array & parameters_, std::optional version_) + : function(std::move(function_)) + , argument_types(argument_types_) + , parameters(parameters_) + , version(version_) +{ + Strings argument_type_names; + for (const auto & argument_type : argument_types) + argument_type_names.push_back(argument_type->getName()); + + Strings function_argument_type_names; + const auto & function_argument_types = function->getArgumentTypes(); + for (const auto & argument_type : function_argument_types) + function_argument_type_names.push_back(argument_type->getName()); + + size_t argument_types_size = std::max(argument_types.size(), function_argument_types.size()); + for (size_t i = 0; i < argument_types_size; ++i) + { + if (argument_types.size() != function_argument_types.size() || !argument_types[i]->equals(*function_argument_types[i])) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Data type AggregateFunction {} got argument types different from function argument types: [{}] != [{}]", + function->getName(), fmt::join(argument_type_names, ", "), fmt::join(function_argument_type_names, ", ")); + } +} + String DataTypeAggregateFunction::getFunctionName() const { return function->getName(); diff --git a/src/DataTypes/DataTypeAggregateFunction.h b/src/DataTypes/DataTypeAggregateFunction.h index 52ed151107e..e3a4f9726d9 100644 --- a/src/DataTypes/DataTypeAggregateFunction.h +++ b/src/DataTypes/DataTypeAggregateFunction.h @@ -30,13 +30,7 @@ public: static constexpr bool is_parametric = true; DataTypeAggregateFunction(AggregateFunctionPtr function_, const DataTypes & argument_types_, - const Array & parameters_, std::optional version_ = std::nullopt) - : function(std::move(function_)) - , argument_types(argument_types_) - , parameters(parameters_) - , version(version_) - { - } + const Array & parameters_, std::optional version_ = std::nullopt); size_t getVersion() const; From 55fd2e04e331a58b83516fbefa6bad921fa842a3 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 24 Jul 2024 17:00:59 +0000 Subject: [PATCH 056/129] wip --- src/Storages/StorageBuffer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 04e6d6676d1..4ae9e029e1b 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -271,6 +271,8 @@ void StorageBuffer::read( } else { + if (processed_stage > QueryProcessingStage::FetchColumns) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot read from Buffer table with different structure in non-initial stage of query execution"); /// There is a struct mismatch and we need to convert read blocks from the destination table. const Block header = metadata_snapshot->getSampleBlock(); Names columns_intersection = column_names; From 12d917b74268ecb6a86b032d5c00c418c8a48f4e Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 24 Jul 2024 17:01:26 +0000 Subject: [PATCH 057/129] wip --- src/DataTypes/DataTypeAggregateFunction.cpp | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp index ee42e4fea11..a4cd3b9e511 100644 --- a/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/src/DataTypes/DataTypeAggregateFunction.cpp @@ -41,23 +41,6 @@ DataTypeAggregateFunction::DataTypeAggregateFunction(AggregateFunctionPtr functi , parameters(parameters_) , version(version_) { - Strings argument_type_names; - for (const auto & argument_type : argument_types) - argument_type_names.push_back(argument_type->getName()); - - Strings function_argument_type_names; - const auto & function_argument_types = function->getArgumentTypes(); - for (const auto & argument_type : function_argument_types) - function_argument_type_names.push_back(argument_type->getName()); - - size_t argument_types_size = std::max(argument_types.size(), function_argument_types.size()); - for (size_t i = 0; i < argument_types_size; ++i) - { - if (argument_types.size() != function_argument_types.size() || !argument_types[i]->equals(*function_argument_types[i])) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Data type AggregateFunction {} got argument types different from function argument types: [{}] != [{}]", - function->getName(), fmt::join(argument_type_names, ", "), fmt::join(function_argument_type_names, ", ")); - } } String DataTypeAggregateFunction::getFunctionName() const From 1329b5eb0b17d5499639bbb973aab7b17c95b644 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 26 Jul 2024 16:22:54 +0000 Subject: [PATCH 058/129] Fix Buffer over Distributed --- src/Storages/StorageBuffer.cpp | 46 ++++++++++++-- src/Storages/StorageBuffer.h | 1 + ...r_over_distributed_type_mismatch.reference | 18 ++++++ ..._buffer_over_distributed_type_mismatch.sql | 60 +++++++++++++++++++ 4 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.reference create mode 100644 tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 4ae9e029e1b..f753d369d2d 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -1,3 +1,7 @@ +#include + +#include +#include #include #include #include @@ -23,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -232,6 +235,12 @@ QueryProcessingStage::Enum StorageBuffer::getQueryProcessingStage( return QueryProcessingStage::FetchColumns; } +bool StorageBuffer::isRemote() const +{ + auto destination = getDestinationTable(); + return destination && destination->isRemote(); +} + void StorageBuffer::read( QueryPlan & query_plan, const Names & column_names, @@ -242,6 +251,29 @@ void StorageBuffer::read( size_t max_block_size, size_t num_streams) { + bool allow_experimental_analyzer = local_context->getSettingsRef().allow_experimental_analyzer; + + if (allow_experimental_analyzer && processed_stage > QueryProcessingStage::FetchColumns) + { + /** For query processing stages after FetchColumns, we do not allow using the same table more than once in the query. + * For example: SELECT * FROM buffer t1 JOIN buffer t2 USING (column) + * In that case, we will execute this query separately for the destination table and for the buffer, resulting in incorrect results. + */ + const auto & current_storage_id = getStorageID(); + auto table_nodes = extractAllTableReferences(query_info.query_tree); + size_t count_of_current_storage = 0; + for (const auto & node : table_nodes) + { + const auto & table_node = node->as(); + if (table_node.getStorageID().getFullNameNotQuoted() == current_storage_id.getFullNameNotQuoted()) + { + count_of_current_storage++; + if (count_of_current_storage > 1) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "StorageBuffer over Distributed does not support using the same table more than once in the query"); + } + } + } + const auto & metadata_snapshot = storage_snapshot->metadata; if (auto destination = getDestinationTable()) @@ -271,8 +303,6 @@ void StorageBuffer::read( } else { - if (processed_stage > QueryProcessingStage::FetchColumns) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot read from Buffer table with different structure in non-initial stage of query execution"); /// There is a struct mismatch and we need to convert read blocks from the destination table. const Block header = metadata_snapshot->getSampleBlock(); Names columns_intersection = column_names; @@ -330,13 +360,17 @@ void StorageBuffer::read( } } + src_table_query_info.merge_storage_snapshot = storage_snapshot; destination->read( query_plan, columns_intersection, destination_snapshot, src_table_query_info, local_context, processed_stage, max_block_size, num_streams); - if (query_plan.isInitialized()) + if (query_plan.isInitialized() && processed_stage <= QueryProcessingStage::FetchColumns) { - + /** The code below converts columns from metadata_snapshot to columns from destination_metadata_snapshot. + * This conversion is not applicable for processed_stage > FetchColumns. + * Instead, we rely on the converting actions at the end of this function. + */ auto actions = addMissingDefaults( query_plan.getCurrentDataStream().header, header_after_adding_defaults.getNamesAndTypesList(), @@ -399,7 +433,7 @@ void StorageBuffer::read( /// TODO: Find a way to support projections for StorageBuffer if (processed_stage > QueryProcessingStage::FetchColumns) { - if (local_context->getSettingsRef().allow_experimental_analyzer) + if (allow_experimental_analyzer) { auto storage = std::make_shared( getStorageID(), diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index cd6dd7b933f..02376f286b1 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -84,6 +84,7 @@ public: QueryProcessingStage::Enum processed_stage, size_t max_block_size, size_t num_streams) override; + bool isRemote() const override; bool supportsParallelInsert() const override { return true; } diff --git a/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.reference b/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.reference new file mode 100644 index 00000000000..1dc3acfeccb --- /dev/null +++ b/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.reference @@ -0,0 +1,18 @@ +100 +100 +101 +101 +101 +102 +101 +101 +102 +100 +100 +101 +101 +101 +102 +101 +101 +102 diff --git a/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql b/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql new file mode 100644 index 00000000000..5a7c89074cf --- /dev/null +++ b/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql @@ -0,0 +1,60 @@ + +DROP TABLE IF EXISTS realtimedrep; +CREATE TABLE realtimedrep (`amount` Int32) ENGINE = MergeTree() ORDER BY tuple(); +INSERT INTO realtimedrep FORMAT Values (100); + +DROP TABLE IF EXISTS realtimedistributed; +CREATE TABLE realtimedistributed (`amount` Int32) ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), realtimedrep, rand()); + +DROP TABLE IF EXISTS realtimebuff__fuzz_19; +CREATE TABLE realtimebuff__fuzz_19 (`amount` UInt32) ENGINE = Buffer(currentDatabase(), 'realtimedistributed', 16, 3600, 36000, 10000, 1000000, 10000000, 100000000); +INSERT INTO realtimebuff__fuzz_19 FORMAT Values (101); + +DROP TABLE IF EXISTS realtimebuff__fuzz_20; +CREATE TABLE realtimebuff__fuzz_20 (`amount` Nullable(Int32)) ENGINE = Buffer(currentDatabase(), 'realtimedistributed', 16, 3600, 36000, 10000, 1000000, 10000000, 100000000); +INSERT INTO realtimebuff__fuzz_20 FORMAT Values (101); + +SELECT amount FROM realtimebuff__fuzz_19 t1 ORDER BY ALL; +SELECT amount + 1 FROM realtimebuff__fuzz_19 t1 ORDER BY ALL; +SELECT amount + 1 FROM realtimebuff__fuzz_20 t1 ORDER BY ALL; +SELECT sum(amount) = 100 FROM realtimebuff__fuzz_19 ORDER BY ALL; -- { serverError CANNOT_CONVERT_TYPE } +SELECT sum(amount) = 100 FROM realtimebuff__fuzz_20 ORDER BY ALL; -- { serverError CANNOT_CONVERT_TYPE } + +SELECT amount FROM realtimebuff__fuzz_19 t1 +JOIN (SELECT number :: UInt32 AS amount FROM numbers(3) ) t2 ON t1.amount = t2.amount +ORDER BY ALL +SETTINGS allow_experimental_analyzer = 0; -- { serverError UNKNOWN_IDENTIFIER } + +SELECT amount FROM realtimebuff__fuzz_19 t1 +JOIN (SELECT number :: UInt32 AS amount FROM numbers(3) ) t2 ON t1.amount = t2.amount +ORDER BY ALL +SETTINGS allow_experimental_analyzer = 1; + +SELECT amount FROM realtimebuff__fuzz_19 t1 +JOIN (SELECT number :: UInt32 AS amount FROM numbers(300) ) t2 ON t1.amount = t2.amount +ORDER BY ALL +SETTINGS allow_experimental_analyzer = 0; -- { serverError UNKNOWN_IDENTIFIER } + +SELECT amount FROM realtimebuff__fuzz_19 t1 +JOIN (SELECT number :: UInt32 AS amount FROM numbers(300) ) t2 ON t1.amount = t2.amount +ORDER BY ALL +SETTINGS allow_experimental_analyzer = 1; + +SELECT t2.amount + 1 FROM (SELECT number :: UInt32 AS amount FROM numbers(300) ) t1 +JOIN realtimebuff__fuzz_19 t2 USING (amount) +ORDER BY ALL +; + +SELECT t2.amount + 1 FROM (SELECT number :: UInt32 AS amount FROM numbers(300) ) t1 +JOIN realtimebuff__fuzz_19 t2 ON t1.amount = t2.amount +ORDER BY ALL +; + +SELECT amount FROM realtimebuff__fuzz_19 t1 +JOIN realtimebuff__fuzz_19 t2 ON t1.amount = t2.amount +; -- { serverError NOT_IMPLEMENTED,UNKNOWN_IDENTIFIER } + +SELECT amount FROM realtimebuff__fuzz_19 t1 +JOIN realtimebuff__fuzz_19 t2 ON t1.amount = t2.amount +JOIN realtimebuff__fuzz_19 t3 ON t1.amount = t3.amount +; -- { serverError NOT_IMPLEMENTED,AMBIGUOUS_COLUMN_NAME } From 2d92cd71a83ed07e6f22c2ba37d39926837a9df0 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 5 Aug 2024 09:36:09 +0000 Subject: [PATCH 059/129] add some comments --- src/Interpreters/inplaceBlockConversions.cpp | 9 +++++++-- src/Storages/MergeTree/IMergeTreeReader.cpp | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp index 429d467ffbf..68254768a7d 100644 --- a/src/Interpreters/inplaceBlockConversions.cpp +++ b/src/Interpreters/inplaceBlockConversions.cpp @@ -309,6 +309,7 @@ static bool hasDefault(const StorageMetadataPtr & metadata_snapshot, const NameA static String removeTupleElementsFromSubcolumn(String subcolumn_name, const Names & tuple_elements) { + /// Add a dot to the end of name for convenience. subcolumn_name += "."; for (const auto & elem : tuple_elements) { @@ -396,19 +397,23 @@ void fillMissingColumns( Names tuple_elements; auto serialization = IDataType::getSerialization(*requested_column); + /// For Nested columns collect names of tuple elements and skip them while getting the base type of array. IDataType::forEachSubcolumn([&](const auto & path, const auto &, const auto &) { if (path.back().type == ISerialization::Substream::TupleElement) tuple_elements.push_back(path.back().name_of_substream); }, ISerialization::SubstreamData(serialization)); + /// The number of dimensions that belongs to the array itself but not shared in Nested column. + /// For example for column "n Nested(a UInt64, b Array(UInt64))" this value is 0 for `n.a` and 1 for `n.b`. size_t num_empty_dimensions = num_dimensions - current_offsets.size(); + auto base_type = getBaseTypeOfArray(requested_column->getTypeInStorage(), tuple_elements); auto scalar_type = createArrayOfType(base_type, num_empty_dimensions); - size_t data_size = assert_cast(*current_offsets.back()).getData().back(); - auto subcolumn_name = removeTupleElementsFromSubcolumn(requested_column->getSubcolumnName(), tuple_elements); + /// Remove names of tuple elements because they are already processed by 'getBaseTypeOfArray'. + auto subcolumn_name = removeTupleElementsFromSubcolumn(requested_column->getSubcolumnName(), tuple_elements); res_columns[i] = createColumnWithDefaultValue(*scalar_type, subcolumn_name, data_size); for (auto it = current_offsets.rbegin(); it != current_offsets.rend(); ++it) diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 05a0b5a7dbc..e0b2710c61f 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -150,6 +150,7 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns NamesAndTypesList full_requested_columns; /// Convert columns list to block. And convert subcolumns to full columns. + /// Defaults should be executed on full columns to get correct values for subcolumns. /// TODO: rewrite with columns interface. It will be possible after changes in ExpressionActions. auto it = original_requested_columns.begin(); From a296beb39084492a6825879ff5258f4377ead75b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 5 Aug 2024 13:31:53 +0200 Subject: [PATCH 060/129] Unit test: Mark as FAILURE if retcode != 0 --- tests/ci/unit_tests_check.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py index 6430fa78801..9cc8ec379bf 100644 --- a/tests/ci/unit_tests_check.py +++ b/tests/ci/unit_tests_check.py @@ -197,6 +197,11 @@ def main(): subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {TEMP_PATH}", shell=True) state, description, test_results = process_results(test_output) + if retcode != 0 and state == SUCCESS: + # The process might have failed without reporting it in the test_output (e.g. LeakSanitizer) + state = FAILURE + description = "Invalid return code. Check run.log" + additional_files = [run_log_path] + [ p for p in test_output.iterdir() if not p.is_dir() ] From ff0b8889ab8bf347343e18b4e9a407b3cab8264b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 5 Aug 2024 14:36:24 +0200 Subject: [PATCH 061/129] Update 02789_reading_from_s3_with_connection_pool.sh --- .../0_stateless/02789_reading_from_s3_with_connection_pool.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh index 39399842db1..5a37d51233d 100755 --- a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh +++ b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-random-settings, no-replicated-database +# Tags: no-fasttest, no-random-settings, no-replicated-database, no-distributed-cache CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From e9506202d699c24e9b7a3e5b643bc9871680e5dc Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 5 Aug 2024 16:40:54 +0200 Subject: [PATCH 062/129] Add debug logging --- src/Storages/WindowView/StorageWindowView.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index 65bf6768b1b..bf934ed00d9 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -1055,6 +1055,8 @@ void StorageWindowView::threadFuncFireProc() /// TODO: consider using time_t instead (for every timestamp in this class) UInt32 timestamp_now = now(); + LOG_TRACE(log, "Now: {}, next fire signal: {}, max watermark: {}", timestamp_now, next_fire_signal, max_watermark); + while (next_fire_signal <= timestamp_now) { try @@ -1072,6 +1074,9 @@ void StorageWindowView::threadFuncFireProc() if (slide_kind > IntervalKind::Kind::Day) slide_interval *= 86400; next_fire_signal += slide_interval; + + LOG_TRACE(log, "Now: {}, next fire signal: {}, max watermark: {}, max fired watermark: {}, slide interval: {}", + timestamp_now, next_fire_signal, max_watermark, max_fired_watermark, slide_interval); } if (max_watermark >= timestamp_now) @@ -1433,16 +1438,19 @@ void StorageWindowView::writeIntoWindowView( while (window_view.modifying_query) std::this_thread::sleep_for(std::chrono::milliseconds(100)); - if (!window_view.is_proctime && window_view.max_watermark == 0 && block.rows() > 0) + const size_t block_rows = block.rows(); + if (!window_view.is_proctime && window_view.max_watermark == 0 && block_rows > 0) { std::lock_guard lock(window_view.fire_signal_mutex); const auto & window_column = block.getByName(window_view.timestamp_column_name); const ColumnUInt32::Container & window_end_data = static_cast(*window_column.column).getData(); UInt32 first_record_timestamp = window_end_data[0]; window_view.max_watermark = window_view.getWindowUpperBound(first_record_timestamp); + + LOG_TRACE(window_view.log, "New max watermark: {}", window_view.max_watermark); } - Pipe pipe(std::make_shared(block)); + Pipe pipe(std::make_shared(std::move(block))); UInt32 lateness_bound = 0; UInt32 t_max_watermark = 0; @@ -1649,6 +1657,8 @@ void StorageWindowView::writeIntoWindowView( auto executor = builder.execute(); executor->execute(builder.getNumThreads(), local_context->getSettingsRef().use_concurrency_control); + + LOG_TRACE(window_view.log, "Wrote {} rows into inner table ({})", block_rows, inner_table->getStorageID().getFullTableName()); } void StorageWindowView::startup() From 146b8afce61ca2db2cdfd0621b0378746e90de59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 5 Aug 2024 17:20:41 +0200 Subject: [PATCH 063/129] Disable bad tests --- .../Nodes/tests/gtest_throttler_constraint.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp b/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp index 6cfccb252fa..363e286c91c 100644 --- a/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp +++ b/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp @@ -10,7 +10,9 @@ using namespace DB; using ResourceTest = ResourceTestClass; -TEST(SchedulerThrottlerConstraint, LeakyBucketConstraint) +/// Test disabled because of leaks in the test themselves: https://github.com/ClickHouse/ClickHouse/issues/67678 + +TEST(DISABLED_SchedulerThrottlerConstraint, LeakyBucketConstraint) { ResourceTest t; EventQueue::TimePoint start = std::chrono::system_clock::now(); @@ -40,7 +42,7 @@ TEST(SchedulerThrottlerConstraint, LeakyBucketConstraint) t.consumed("A", 10); } -TEST(SchedulerThrottlerConstraint, Unlimited) +TEST(DISABLED_SchedulerThrottlerConstraint, Unlimited) { ResourceTest t; EventQueue::TimePoint start = std::chrono::system_clock::now(); @@ -57,7 +59,7 @@ TEST(SchedulerThrottlerConstraint, Unlimited) } } -TEST(SchedulerThrottlerConstraint, Pacing) +TEST(DISABLED_SchedulerThrottlerConstraint, Pacing) { ResourceTest t; EventQueue::TimePoint start = std::chrono::system_clock::now(); @@ -77,7 +79,7 @@ TEST(SchedulerThrottlerConstraint, Pacing) } } -TEST(SchedulerThrottlerConstraint, BucketFilling) +TEST(DISABLED_SchedulerThrottlerConstraint, BucketFilling) { ResourceTest t; EventQueue::TimePoint start = std::chrono::system_clock::now(); @@ -111,7 +113,7 @@ TEST(SchedulerThrottlerConstraint, BucketFilling) t.consumed("A", 3); } -TEST(SchedulerThrottlerConstraint, PeekAndAvgLimits) +TEST(DISABLED_SchedulerThrottlerConstraint, PeekAndAvgLimits) { ResourceTest t; EventQueue::TimePoint start = std::chrono::system_clock::now(); @@ -139,7 +141,7 @@ TEST(SchedulerThrottlerConstraint, PeekAndAvgLimits) } } -TEST(SchedulerThrottlerConstraint, ThrottlerAndFairness) +TEST(DISABLED_SchedulerThrottlerConstraint, ThrottlerAndFairness) { ResourceTest t; EventQueue::TimePoint start = std::chrono::system_clock::now(); From 4e8d11c48e9ac0107940d730ccf8d35b7ac3573e Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 5 Aug 2024 16:46:42 +0000 Subject: [PATCH 064/129] Add fuzzers to 03208_buffer_over_distributed_type_mismatch --- ...208_buffer_over_distributed_type_mismatch.sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql b/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql index 5a7c89074cf..333c445403d 100644 --- a/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql +++ b/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql @@ -58,3 +58,19 @@ SELECT amount FROM realtimebuff__fuzz_19 t1 JOIN realtimebuff__fuzz_19 t2 ON t1.amount = t2.amount JOIN realtimebuff__fuzz_19 t3 ON t1.amount = t3.amount ; -- { serverError NOT_IMPLEMENTED,AMBIGUOUS_COLUMN_NAME } + + +-- fuzzers: + +SELECT + toLowCardinality(1) + materialize(toLowCardinality(2)) +FROM realtimebuff__fuzz_19 +GROUP BY toLowCardinality(1) +FORMAT Null +; + +SELECT intDivOrZero(intDivOrZero(toLowCardinality(-128), toLowCardinality(-1)) = 0, materialize(toLowCardinality(4))) +FROM realtimebuff__fuzz_19 GROUP BY materialize(toLowCardinality(-127)), intDivOrZero(0, 0) = toLowCardinality(toLowCardinality(0)) +WITH TOTALS ORDER BY ALL DESC NULLS FIRST +FORMAT Null +; From 457686c379cb884a579b86cbfd6a9abb3ec16d1a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 10 Jul 2024 19:39:58 +0200 Subject: [PATCH 065/129] Simplify StorageDistributed ctors Signed-off-by: Azat Khuzhin --- src/Storages/StorageDistributed.cpp | 32 ---------------------- src/Storages/StorageDistributed.h | 14 ---------- src/TableFunctions/TableFunctionRemote.cpp | 19 ++----------- 3 files changed, 3 insertions(+), 62 deletions(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 3e38ddf830a..eb9483127fc 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -373,38 +373,6 @@ StorageDistributed::StorageDistributed( } -StorageDistributed::StorageDistributed( - const StorageID & id_, - const ColumnsDescription & columns_, - const ConstraintsDescription & constraints_, - ASTPtr remote_table_function_ptr_, - const String & cluster_name_, - ContextPtr context_, - const ASTPtr & sharding_key_, - const String & storage_policy_name_, - const String & relative_data_path_, - const DistributedSettings & distributed_settings_, - LoadingStrictnessLevel mode, - ClusterPtr owned_cluster_) - : StorageDistributed( - id_, - columns_, - constraints_, - String{}, - String{}, - String{}, - cluster_name_, - context_, - sharding_key_, - storage_policy_name_, - relative_data_path_, - distributed_settings_, - mode, - std::move(owned_cluster_), - remote_table_function_ptr_) -{ -} - QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage( ContextPtr local_context, QueryProcessingStage::Enum to_stage, diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index 85a8de86953..a9e10dd14ba 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -63,20 +63,6 @@ public: ClusterPtr owned_cluster_ = {}, ASTPtr remote_table_function_ptr_ = {}); - StorageDistributed( - const StorageID & id_, - const ColumnsDescription & columns_, - const ConstraintsDescription & constraints_, - ASTPtr remote_table_function_ptr_, - const String & cluster_name_, - ContextPtr context_, - const ASTPtr & sharding_key_, - const String & storage_policy_name_, - const String & relative_data_path_, - const DistributedSettings & distributed_settings_, - LoadingStrictnessLevel mode, - ClusterPtr owned_cluster_ = {}); - ~StorageDistributed() override; std::string getName() const override { return "Distributed"; } diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index e60c31b2d77..5d136993018 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -306,21 +306,7 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & /*ast_function*/, Con cached_columns = getActualTableStructure(context, is_insert_query); assert(cluster); - StoragePtr res = remote_table_function_ptr - ? std::make_shared( - StorageID(getDatabaseName(), table_name), - cached_columns, - ConstraintsDescription{}, - remote_table_function_ptr, - String{}, - context, - sharding_key, - String{}, - String{}, - DistributedSettings{}, - LoadingStrictnessLevel::CREATE, - cluster) - : std::make_shared( + StoragePtr res = std::make_shared( StorageID(getDatabaseName(), table_name), cached_columns, ConstraintsDescription{}, @@ -334,7 +320,8 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & /*ast_function*/, Con String{}, DistributedSettings{}, LoadingStrictnessLevel::CREATE, - cluster); + cluster, + remote_table_function_ptr); res->startup(); return res; From 9d7710684b91a9a5b0d7c2a34b709232979ebe65 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 10 Jul 2024 19:57:09 +0200 Subject: [PATCH 066/129] tests/test_distributed_inter_server_secret: get_query_user_info return list Signed-off-by: Azat Khuzhin --- .../test_distributed_inter_server_secret/test.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index 7ecb2cda257..9389b901a30 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -116,10 +116,10 @@ def start_cluster(): cluster.shutdown() -# @return -- [user, initial_user] +# @return -- [[user, initial_user]] def get_query_user_info(node, query_pattern): node.query("SYSTEM FLUSH LOGS") - return ( + lines = ( node.query( """ SELECT user, initial_user @@ -133,8 +133,10 @@ def get_query_user_info(node, query_pattern): ) ) .strip() - .split("\t") + .split("\n") ) + lines = map(lambda x: x.split("\t"), lines) + return list(lines) # @return -- [user, initial_user] @@ -331,19 +333,19 @@ def test_secure_disagree_insert(): def test_user_insecure_cluster(user, password): id_ = "query-dist_insecure-" + user n1.query(f"SELECT *, '{id_}' FROM dist_insecure", user=user, password=password) - assert get_query_user_info(n1, id_) == [ + assert get_query_user_info(n1, id_)[0] == [ user, user, ] # due to prefer_localhost_replica - assert get_query_user_info(n2, id_) == ["default", user] + assert get_query_user_info(n2, id_)[0] == ["default", user] @users def test_user_secure_cluster(user, password): id_ = "query-dist_secure-" + user n1.query(f"SELECT *, '{id_}' FROM dist_secure", user=user, password=password) - assert get_query_user_info(n1, id_) == [user, user] - assert get_query_user_info(n2, id_) == [user, user] + assert get_query_user_info(n1, id_)[0] == [user, user] + assert get_query_user_info(n2, id_)[0] == [user, user] @users From 0dccc34a2004b075e290ebfd5ecc9fea3077a205 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 10 Jul 2024 19:23:34 +0200 Subject: [PATCH 067/129] Fix cluster() for inter-server secret (preserve initial user as before) The behaviour of cluster() with inter-server secret had been changed in #63013, after it started to use "default" user, and this introduces a regression. The intention of that patch was to adjust only remote(), since it only it accept custom user, which should be ignored. Fixes: https://github.com/ClickHouse/ClickHouse/issues/66287 Fixes: https://github.com/ClickHouse/ClickHouse/issues/66352 Signed-off-by: Azat Khuzhin --- src/Storages/StorageDistributed.cpp | 6 ++++-- src/Storages/StorageDistributed.h | 5 ++++- src/TableFunctions/TableFunctionRemote.cpp | 3 ++- .../test_distributed_inter_server_secret/test.py | 15 ++++++++++++++- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index eb9483127fc..e146e95f89f 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -308,7 +308,8 @@ StorageDistributed::StorageDistributed( const DistributedSettings & distributed_settings_, LoadingStrictnessLevel mode, ClusterPtr owned_cluster_, - ASTPtr remote_table_function_ptr_) + ASTPtr remote_table_function_ptr_, + bool is_remote_function_) : IStorage(id_) , WithContext(context_->getGlobalContext()) , remote_database(remote_database_) @@ -322,6 +323,7 @@ StorageDistributed::StorageDistributed( , relative_data_path(relative_data_path_) , distributed_settings(distributed_settings_) , rng(randomSeed()) + , is_remote_function(is_remote_function_) { if (!distributed_settings.flush_on_detach && distributed_settings.background_insert_batch) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Settings flush_on_detach=0 and background_insert_batch=1 are incompatible"); @@ -869,7 +871,7 @@ void StorageDistributed::read( sharding_key_column_name, distributed_settings, shard_filter_generator, - /* is_remote_function= */ static_cast(owned_cluster)); + is_remote_function); /// This is a bug, it is possible only when there is no shards to query, and this is handled earlier. if (!query_plan.isInitialized()) diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index a9e10dd14ba..8a5585e9fd0 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -61,7 +61,8 @@ public: const DistributedSettings & distributed_settings_, LoadingStrictnessLevel mode, ClusterPtr owned_cluster_ = {}, - ASTPtr remote_table_function_ptr_ = {}); + ASTPtr remote_table_function_ptr_ = {}, + bool is_remote_function_ = false); ~StorageDistributed() override; @@ -273,6 +274,8 @@ private: // For random shard index generation mutable std::mutex rng_mutex; pcg64 rng; + + bool is_remote_function; }; } diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 5d136993018..8a877ff0802 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -321,7 +321,8 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & /*ast_function*/, Con DistributedSettings{}, LoadingStrictnessLevel::CREATE, cluster, - remote_table_function_ptr); + remote_table_function_ptr, + !is_cluster_function); res->startup(); return res; diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index 9389b901a30..36d7e044f1c 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -418,7 +418,7 @@ def test_per_user_protocol_settings_secure_cluster(user, password): ) -def test_secure_cluster_distributed_over_distributed_different_users(): +def test_secure_cluster_distributed_over_distributed_different_users_remote(): # This works because we will have initial_user='default' n1.query( "SELECT * FROM remote('n1', currentDatabase(), dist_secure)", user="new_user" @@ -433,3 +433,16 @@ def test_secure_cluster_distributed_over_distributed_different_users(): # and stuff). with pytest.raises(QueryRuntimeException): n1.query("SELECT * FROM dist_over_dist_secure", user="new_user") + + +def test_secure_cluster_distributed_over_distributed_different_users_cluster(): + id_ = "cluster-user" + n1.query( + f"SELECT *, '{id_}' FROM cluster(secure, currentDatabase(), dist_secure)", + user="nopass", + settings={ + "prefer_localhost_replica": 0, + }, + ) + assert get_query_user_info(n1, id_) == [["nopass", "nopass"]] * 4 + assert get_query_user_info(n2, id_) == [["nopass", "nopass"]] * 3 From 618789d1960182ff91d614545a16d1332768e008 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 22 Jun 2024 20:42:27 +0200 Subject: [PATCH 068/129] Do not pass logs for keeper explicitly in the image to allow overriding Right now, if you will start keeper like this: $ cat /tmp/local.yaml --- logger: log: "@remove": remove errorlog: "@remove": remove console: 1 $ docker run --name keeper -v /tmp/local.yaml:/etc/clickhouse-keeper/conf.d/local.yaml --rm -it clickhouse/clickhouse-keeper This will still not work, because the --log-file/--errorlog-file passed explicitly. So this patch removes this in case config is valid, but, if keeper is launched with embedded config, this had been left as-is, to keep previous behavior. Signed-off-by: Azat Khuzhin --- docker/keeper/entrypoint.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/keeper/entrypoint.sh b/docker/keeper/entrypoint.sh index 1390ad9ce74..68bd0ef9d87 100644 --- a/docker/keeper/entrypoint.sh +++ b/docker/keeper/entrypoint.sh @@ -40,8 +40,6 @@ fi DATA_DIR="${CLICKHOUSE_DATA_DIR:-/var/lib/clickhouse}" LOG_DIR="${LOG_DIR:-/var/log/clickhouse-keeper}" -LOG_PATH="${LOG_DIR}/clickhouse-keeper.log" -ERROR_LOG_PATH="${LOG_DIR}/clickhouse-keeper.err.log" COORDINATION_DIR="${DATA_DIR}/coordination" COORDINATION_LOG_DIR="${DATA_DIR}/coordination/log" COORDINATION_SNAPSHOT_DIR="${DATA_DIR}/coordination/snapshots" @@ -84,7 +82,7 @@ if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then # There is a config file. It is already tested with gosu (if it is readably by keeper user) if [ -f "$KEEPER_CONFIG" ]; then - exec $gosu /usr/bin/clickhouse-keeper --config-file="$KEEPER_CONFIG" --log-file="$LOG_PATH" --errorlog-file="$ERROR_LOG_PATH" "$@" + exec $gosu /usr/bin/clickhouse-keeper --config-file="$KEEPER_CONFIG" "$@" fi # There is no config file. Will use embedded one From 205303c2868645e2aebf76f9b49ab77960579c56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 5 Aug 2024 20:37:43 +0200 Subject: [PATCH 069/129] Disable more bad tests --- .../Nodes/tests/gtest_resource_class_fair.cpp | 12 +++++++----- .../Nodes/tests/gtest_resource_class_priority.cpp | 10 ++++++---- .../Nodes/tests/gtest_throttler_constraint.cpp | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/Common/Scheduler/Nodes/tests/gtest_resource_class_fair.cpp b/src/Common/Scheduler/Nodes/tests/gtest_resource_class_fair.cpp index 4f0e8c80734..16cce309c2a 100644 --- a/src/Common/Scheduler/Nodes/tests/gtest_resource_class_fair.cpp +++ b/src/Common/Scheduler/Nodes/tests/gtest_resource_class_fair.cpp @@ -8,7 +8,9 @@ using namespace DB; using ResourceTest = ResourceTestClass; -TEST(SchedulerFairPolicy, Factory) +/// Tests disabled because of leaks in the test themselves: https://github.com/ClickHouse/ClickHouse/issues/67678 + +TEST(DISABLED_SchedulerFairPolicy, Factory) { ResourceTest t; @@ -17,7 +19,7 @@ TEST(SchedulerFairPolicy, Factory) EXPECT_TRUE(dynamic_cast(fair.get()) != nullptr); } -TEST(SchedulerFairPolicy, FairnessWeights) +TEST(DISABLED_SchedulerFairPolicy, FairnessWeights) { ResourceTest t; @@ -41,7 +43,7 @@ TEST(SchedulerFairPolicy, FairnessWeights) t.consumed("B", 20); } -TEST(SchedulerFairPolicy, Activation) +TEST(DISABLED_SchedulerFairPolicy, Activation) { ResourceTest t; @@ -77,7 +79,7 @@ TEST(SchedulerFairPolicy, Activation) t.consumed("B", 10); } -TEST(SchedulerFairPolicy, FairnessMaxMin) +TEST(DISABLED_SchedulerFairPolicy, FairnessMaxMin) { ResourceTest t; @@ -101,7 +103,7 @@ TEST(SchedulerFairPolicy, FairnessMaxMin) t.consumed("A", 20); } -TEST(SchedulerFairPolicy, HierarchicalFairness) +TEST(DISABLED_SchedulerFairPolicy, HierarchicalFairness) { ResourceTest t; diff --git a/src/Common/Scheduler/Nodes/tests/gtest_resource_class_priority.cpp b/src/Common/Scheduler/Nodes/tests/gtest_resource_class_priority.cpp index a447b7f6780..d3d38aae048 100644 --- a/src/Common/Scheduler/Nodes/tests/gtest_resource_class_priority.cpp +++ b/src/Common/Scheduler/Nodes/tests/gtest_resource_class_priority.cpp @@ -8,7 +8,9 @@ using namespace DB; using ResourceTest = ResourceTestClass; -TEST(SchedulerPriorityPolicy, Factory) +/// Tests disabled because of leaks in the test themselves: https://github.com/ClickHouse/ClickHouse/issues/67678 + +TEST(DISABLED_SchedulerPriorityPolicy, Factory) { ResourceTest t; @@ -17,7 +19,7 @@ TEST(SchedulerPriorityPolicy, Factory) EXPECT_TRUE(dynamic_cast(prio.get()) != nullptr); } -TEST(SchedulerPriorityPolicy, Priorities) +TEST(DISABLED_SchedulerPriorityPolicy, Priorities) { ResourceTest t; @@ -51,7 +53,7 @@ TEST(SchedulerPriorityPolicy, Priorities) t.consumed("C", 0); } -TEST(SchedulerPriorityPolicy, Activation) +TEST(DISABLED_SchedulerPriorityPolicy, Activation) { ResourceTest t; @@ -92,7 +94,7 @@ TEST(SchedulerPriorityPolicy, Activation) t.consumed("C", 0); } -TEST(SchedulerPriorityPolicy, SinglePriority) +TEST(DISABLED_SchedulerPriorityPolicy, SinglePriority) { ResourceTest t; diff --git a/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp b/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp index 363e286c91c..2bc24cdb292 100644 --- a/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp +++ b/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp @@ -10,7 +10,7 @@ using namespace DB; using ResourceTest = ResourceTestClass; -/// Test disabled because of leaks in the test themselves: https://github.com/ClickHouse/ClickHouse/issues/67678 +/// Tests disabled because of leaks in the test themselves: https://github.com/ClickHouse/ClickHouse/issues/67678 TEST(DISABLED_SchedulerThrottlerConstraint, LeakyBucketConstraint) { From 5cdf12a9277993f7adf97c5083c7e92f914a5ae7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 4 Aug 2024 22:31:17 +0000 Subject: [PATCH 070/129] Automatic style fix From 29f06e8037316d61034700eff8e41c9b8730f317 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 5 Aug 2024 15:17:32 +0200 Subject: [PATCH 071/129] Update 00002_log_and_exception_messages_formatting.sql --- .../0_stateless/00002_log_and_exception_messages_formatting.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql index b806aff3a00..efd961a0fda 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -1,5 +1,7 @@ -- Tags: no-parallel, no-fasttest, no-ubsan, no-batch, no-flaky-check -- no-parallel because we want to run this test when most of the other tests already passed +-- This is not a regular test. It is intended to run once after other tests to validate certain statistics about the whole test runs. +-- TODO: I advice to put in inside clickhouse-test instead. -- If this test fails, see the "Top patterns of log messages" diagnostics in the end of run.log From 61aff5efae873bde8631919386ba7f5c9db50af1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 5 Aug 2024 15:17:48 +0200 Subject: [PATCH 072/129] Update 00002_log_and_exception_messages_formatting.sql --- .../0_stateless/00002_log_and_exception_messages_formatting.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql index efd961a0fda..53321afc94c 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -1,7 +1,7 @@ -- Tags: no-parallel, no-fasttest, no-ubsan, no-batch, no-flaky-check -- no-parallel because we want to run this test when most of the other tests already passed -- This is not a regular test. It is intended to run once after other tests to validate certain statistics about the whole test runs. --- TODO: I advice to put in inside clickhouse-test instead. +-- TODO: I advise to put in inside clickhouse-test instead. -- If this test fails, see the "Top patterns of log messages" diagnostics in the end of run.log From 8703941a0f5933a8a2adbc4003942b5ba22496f1 Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Mon, 5 Aug 2024 22:36:30 +0000 Subject: [PATCH 073/129] Apply https://github.com/ClickHouse/zlib-ng/pull/18 --- contrib/zlib-ng | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/zlib-ng b/contrib/zlib-ng index c19ba056b7c..a2fbeffdc30 160000 --- a/contrib/zlib-ng +++ b/contrib/zlib-ng @@ -1 +1 @@ -Subproject commit c19ba056b7cc8029bb80f509956090c7ded58032 +Subproject commit a2fbeffdc30a8b0ce6d54ee31208e2688eac4c9f From 087d014f2ae050b7a45f72a7cf87f5de5a8179cf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 6 Aug 2024 03:17:17 +0200 Subject: [PATCH 074/129] Revert "Revert "Add a test for #47892"" --- .../02968_full_sorting_join_fuzz.reference | 3 +++ .../0_stateless/02968_full_sorting_join_fuzz.sql | 15 +++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 tests/queries/0_stateless/02968_full_sorting_join_fuzz.reference create mode 100644 tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql diff --git a/tests/queries/0_stateless/02968_full_sorting_join_fuzz.reference b/tests/queries/0_stateless/02968_full_sorting_join_fuzz.reference new file mode 100644 index 00000000000..0e9f5cc1db1 --- /dev/null +++ b/tests/queries/0_stateless/02968_full_sorting_join_fuzz.reference @@ -0,0 +1,3 @@ +[NULL] [] 100 0 + +[NULL] [] 100 0 diff --git a/tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql b/tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql new file mode 100644 index 00000000000..802eda57df3 --- /dev/null +++ b/tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql @@ -0,0 +1,15 @@ +SET max_bytes_in_join = 0, join_algorithm = 'full_sorting_merge', max_block_size = 10240; + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; + +CREATE TABLE t1 (`key` UInt32, `s` String) ENGINE = MergeTree ORDER BY key; +CREATE TABLE t2 (`key` UInt32, `s` String) ENGINE = MergeTree ORDER BY key; + +INSERT INTO t1 SELECT (sipHash64(number, 'x') % 10000000) + 1 AS key, concat('val', toString(number)) AS s FROM numbers_mt(10000000); +INSERT INTO t2 SELECT (sipHash64(number, 'y') % 1000000) + 1 AS key, concat('val', toString(number)) AS s FROM numbers_mt(1000000); + +SELECT materialize([NULL]), [], 100, count(materialize(NULL)) FROM t1 ALL INNER JOIN t2 ON t1.key = t2.key PREWHERE 10 WHERE t2.key WITH TOTALS; + +DROP TABLE t1; +DROP TABLE t2; From 15a4d13cdad3fe9181703d5c0a993b8dcb8cb263 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 6 Aug 2024 04:33:02 +0200 Subject: [PATCH 075/129] Update 02968_full_sorting_join_fuzz.sql --- tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql b/tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql index 802eda57df3..85ca740cef2 100644 --- a/tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql +++ b/tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql @@ -9,7 +9,7 @@ CREATE TABLE t2 (`key` UInt32, `s` String) ENGINE = MergeTree ORDER BY key; INSERT INTO t1 SELECT (sipHash64(number, 'x') % 10000000) + 1 AS key, concat('val', toString(number)) AS s FROM numbers_mt(10000000); INSERT INTO t2 SELECT (sipHash64(number, 'y') % 1000000) + 1 AS key, concat('val', toString(number)) AS s FROM numbers_mt(1000000); -SELECT materialize([NULL]), [], 100, count(materialize(NULL)) FROM t1 ALL INNER JOIN t2 ON t1.key = t2.key PREWHERE 10 WHERE t2.key WITH TOTALS; +SELECT materialize([NULL]), [], 100, count(materialize(NULL)) FROM t1 ALL INNER JOIN t2 ON t1.key = t2.key PREWHERE 10 WHERE t2.key != 0 WITH TOTALS; DROP TABLE t1; DROP TABLE t2; From 8e6de033355485a770f86b848e7574728a0eecf4 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 6 Aug 2024 09:39:00 +0200 Subject: [PATCH 076/129] Remove unsupported files --- .../queries/0_stateless/03215_parsing_archive_name_s3.sql | 8 ++++++++ tests/queries/0_stateless/data_minio/::03215_archive.csv | 1 - .../0_stateless/data_minio/test.zip::03215_archive.csv | 1 - .../0_stateless/data_minio/test::03215_archive.csv | 1 - 4 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 tests/queries/0_stateless/data_minio/::03215_archive.csv delete mode 100644 tests/queries/0_stateless/data_minio/test.zip::03215_archive.csv delete mode 100644 tests/queries/0_stateless/data_minio/test::03215_archive.csv diff --git a/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql b/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql index e34be475c5a..e0d63d313fa 100644 --- a/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql +++ b/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql @@ -1,7 +1,15 @@ -- Tags: no-fasttest -- Tag no-fasttest: Depends on AWS +SET s3_truncate_on_insert=1; + +INSERT INTO FUNCTION s3(s3_conn, filename='::03215_archive.csv') SELECT 1; SELECT _file, _path FROM s3(s3_conn, filename='::03215_archive.csv') ORDER BY (_file, _path); + SELECT _file, _path FROM s3(s3_conn, filename='test :: 03215_archive.csv') ORDER BY (_file, _path); -- { serverError S3_ERROR } + +INSERT INTO FUNCTION s3(s3_conn, filename='test::03215_archive.csv') SELECT 1; SELECT _file, _path FROM s3(s3_conn, filename='test::03215_archive.csv') ORDER BY (_file, _path); + +INSERT INTO FUNCTION s3(s3_conn, filename='test.zip::03215_archive.csv') SETTINGS allow_archive_path_syntax=0 SELECT 1; SELECT _file, _path FROM s3(s3_conn, filename='test.zip::03215_archive.csv') ORDER BY (_file, _path) SETTINGS allow_archive_path_syntax=0; diff --git a/tests/queries/0_stateless/data_minio/::03215_archive.csv b/tests/queries/0_stateless/data_minio/::03215_archive.csv deleted file mode 100644 index d00491fd7e5..00000000000 --- a/tests/queries/0_stateless/data_minio/::03215_archive.csv +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests/queries/0_stateless/data_minio/test.zip::03215_archive.csv b/tests/queries/0_stateless/data_minio/test.zip::03215_archive.csv deleted file mode 100644 index d00491fd7e5..00000000000 --- a/tests/queries/0_stateless/data_minio/test.zip::03215_archive.csv +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests/queries/0_stateless/data_minio/test::03215_archive.csv b/tests/queries/0_stateless/data_minio/test::03215_archive.csv deleted file mode 100644 index d00491fd7e5..00000000000 --- a/tests/queries/0_stateless/data_minio/test::03215_archive.csv +++ /dev/null @@ -1 +0,0 @@ -1 From 27569815d519c8ad70842798b989874cc04b3271 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 5 Aug 2024 17:01:00 +0200 Subject: [PATCH 077/129] Fix query cache randomization in stress tests Right now it fails with [1]: 2024.08.05 12:53:53.659422 [ 3559 ] {123237a2-5f08-47e4-996e-21e20bc4a51a} executeQuery: Code: 704. DB::Exception: The query result was not cached because the query contains a non-deterministic function. Use setting `query_cache_nondeterministic_function_handling = 'save'` or `= 'ignore'` to cache the query result regardless or to omit caching. (QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS) (version 24.8.1.1356) (from [::1]:42534) (in query: SELECT version(), arrayStringConcat(groupArray(value), ' ') FROM system.build_options WHERE name IN ('GIT_HASH', 'GIT_BRANCH') ), Stack trace (when copying this message, always include the lines below): Uexpected exception, will not retry: HTTPError : Code: 500. Code: 704. DB::Exception: The query result was not cached because the query contains a non-deterministic function. Use setting `query_cache_nondeterministic_function_handling = 'save'` or `= 'ignore'` to cache the query result regardless or to omit caching. (QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS) (version 24.8.1.1356) [1]: https://s3.amazonaws.com/clickhouse-test-reports/67737/e68c9c8d16f37f6c25739076c9b071ed97952269/stress_test__asan_/stress_test_run_10.txt Signed-off-by: Azat Khuzhin --- tests/ci/stress.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/ci/stress.py b/tests/ci/stress.py index 8b8b17dd2a9..a3cdd8b3d6f 100755 --- a/tests/ci/stress.py +++ b/tests/ci/stress.py @@ -47,6 +47,8 @@ def get_options(i: int, upgrade_check: bool) -> str: if i > 0 and random.random() < 1 / 3: client_options.append("use_query_cache=1") + client_options.append("query_cache_nondeterministic_function_handling='ignore'") + client_options.append("query_cache_system_table_handling='ignore'") if i % 5 == 1: client_options.append("memory_tracker_fault_probability=0.001") From b2987e4f4dd267293b8b4f96f86f2202a7671b84 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 6 Aug 2024 11:42:18 +0200 Subject: [PATCH 078/129] Update StorageWindowView.cpp --- src/Storages/WindowView/StorageWindowView.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index bf934ed00d9..5830c844582 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -1450,7 +1450,7 @@ void StorageWindowView::writeIntoWindowView( LOG_TRACE(window_view.log, "New max watermark: {}", window_view.max_watermark); } - Pipe pipe(std::make_shared(std::move(block))); + Pipe pipe(std::make_shared(block)); UInt32 lateness_bound = 0; UInt32 t_max_watermark = 0; From 50ed7da27dc1bcdbb03f5cf7e3d80ae95ecf643f Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Tue, 6 Aug 2024 12:02:53 +0200 Subject: [PATCH 079/129] squash! fix for parallel execution --- tests/queries/0_stateless/03215_parquet_index.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/03215_parquet_index.sql b/tests/queries/0_stateless/03215_parquet_index.sql index 5b176ff70ba..0afccdf6f3b 100644 --- a/tests/queries/0_stateless/03215_parquet_index.sql +++ b/tests/queries/0_stateless/03215_parquet_index.sql @@ -4,12 +4,14 @@ CREATE TABLE test_parquet (col1 int, col2 String) ENGINE=File(Parquet); INSERT INTO test_parquet SELECT number, toString(number) FROM numbers(100); SELECT col1, col2 FROM test_parquet; + -- Parquet will have indexes in columns. We are not checking that indexes exist here, there is an integration test test_parquet_page_index for that. We just check that a setting doesn't break the SELECT DROP TABLE IF EXISTS test_parquet; CREATE TABLE test_parquet (col1 int, col2 String) ENGINE=File(Parquet) SETTINGS output_format_parquet_use_custom_encoder=false, output_format_parquet_write_page_index=true; INSERT INTO test_parquet SELECT number, toString(number) FROM numbers(100); SELECT col1, col2 FROM test_parquet; + -- Parquet will not have indexes in columns. DROP TABLE IF EXISTS test_parquet; CREATE TABLE test_parquet (col1 int, col2 String) ENGINE=File(Parquet) SETTINGS output_format_parquet_use_custom_encoder=false, output_format_parquet_write_page_index=false; From 4a8be15ca8bffc445a6c66cacb5226946531e105 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 6 Aug 2024 10:25:33 +0000 Subject: [PATCH 080/129] Bump rocksdb to v8.11.4 --- contrib/rocksdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/rocksdb b/contrib/rocksdb index 49ce8a1064d..5c2be544f55 160000 --- a/contrib/rocksdb +++ b/contrib/rocksdb @@ -1 +1 @@ -Subproject commit 49ce8a1064dd1ad89117899839bf136365e49e79 +Subproject commit 5c2be544f5509465957706c955b6d623e889ac4e From 7c15ad3966a982a3597034f8a66d91f2750dd2d0 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 6 Aug 2024 10:15:53 +0000 Subject: [PATCH 081/129] do not try to create azure container if not needed --- src/Common/ProfileEvents.cpp | 4 +- .../AzureBlobStorageCommon.cpp | 45 ++++++++++++++++++- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 2b090136e2e..c6cf6c04af3 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -457,6 +457,7 @@ The server successfully detected this situation and will download merged part fr M(AzureDeleteObjects, "Number of Azure blob storage API DeleteObject(s) calls.") \ M(AzureListObjects, "Number of Azure blob storage API ListObjects calls.") \ M(AzureGetProperties, "Number of Azure blob storage API GetProperties calls.") \ + M(AzureCreateContainer, "Number of Azure blob storage API CreateContainer calls.") \ \ M(DiskAzureGetObject, "Number of Disk Azure API GetObject calls.") \ M(DiskAzureUpload, "Number of Disk Azure blob storage API Upload calls") \ @@ -464,8 +465,9 @@ The server successfully detected this situation and will download merged part fr M(DiskAzureCommitBlockList, "Number of Disk Azure blob storage API CommitBlockList calls") \ M(DiskAzureCopyObject, "Number of Disk Azure blob storage API CopyObject calls") \ M(DiskAzureListObjects, "Number of Disk Azure blob storage API ListObjects calls.") \ - M(DiskAzureDeleteObjects, "Number of Azure blob storage API DeleteObject(s) calls.") \ + M(DiskAzureDeleteObjects, "Number of Disk Azure blob storage API DeleteObject(s) calls.") \ M(DiskAzureGetProperties, "Number of Disk Azure blob storage API GetProperties calls.") \ + M(DiskAzureCreateContainer, "Number of Disk Azure blob storage API CreateContainer calls.") \ \ M(ReadBufferFromAzureMicroseconds, "Time spent on reading from Azure.") \ M(ReadBufferFromAzureInitMicroseconds, "Time spent initializing connection to Azure.") \ diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp index 0aa3b9c40b5..9043edd66a0 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp @@ -11,6 +11,14 @@ #include #include +namespace ProfileEvents +{ + extern const Event AzureGetProperties; + extern const Event DiskAzureGetProperties; + extern const Event AzureCreateContainer; + extern const Event DiskAzureCreateContainer; +} + namespace DB { @@ -214,20 +222,53 @@ void processURL(const String & url, const String & container_name, Endpoint & en } } +static bool containerExists(const ContainerClient & client) +{ + ProfileEvents::increment(ProfileEvents::AzureGetProperties); + if (client.GetClickhouseOptions().IsClientForDisk) + ProfileEvents::increment(ProfileEvents::DiskAzureGetProperties); + + try + { + client.GetProperties(); + return true; + } + catch (const Azure::Storage::StorageException & e) + { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) + return false; + throw; + } +} + std::unique_ptr getContainerClient(const ConnectionParams & params, bool readonly) { if (params.endpoint.container_already_exists.value_or(false) || readonly) + { return params.createForContainer(); + } + + if (!params.endpoint.container_already_exists.has_value()) + { + auto container_client = params.createForContainer(); + if (containerExists(*container_client)) + return container_client; + } try { auto service_client = params.createForService(); + + ProfileEvents::increment(ProfileEvents::AzureCreateContainer); + if (params.client_options.ClickhouseOptions.IsClientForDisk) + ProfileEvents::increment(ProfileEvents::DiskAzureCreateContainer); + return std::make_unique(service_client->CreateBlobContainer(params.endpoint.container_name).Value); } catch (const Azure::Storage::StorageException & e) { - /// If container_already_exists is not set (in config), ignore already exists error. - /// (Conflict - The specified container already exists) + /// If container_already_exists is not set (in config), ignore already exists error. Conflict - The specified container already exists. + /// To avoid race with creation of container handle this error despite that we have already checked the existence of container. if (!params.endpoint.container_already_exists.has_value() && e.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict) return params.createForContainer(); throw; From 7e87650a001279317ce79c36415160639a83a4da Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 6 Aug 2024 10:31:13 +0000 Subject: [PATCH 082/129] Bump rocksdb to v9.0.1 --- contrib/rocksdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/rocksdb b/contrib/rocksdb index 5c2be544f55..36540d43550 160000 --- a/contrib/rocksdb +++ b/contrib/rocksdb @@ -1 +1 @@ -Subproject commit 5c2be544f5509465957706c955b6d623e889ac4e +Subproject commit 36540d43550da19e96515bd731ee416a8787672c From f0a6ae220530fa9b34d23aa722ed8816842410ce Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 6 Aug 2024 10:32:40 +0000 Subject: [PATCH 083/129] Bump rocksdb to v9.1.1 --- contrib/rocksdb | 2 +- contrib/rocksdb-cmake/CMakeLists.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/rocksdb b/contrib/rocksdb index 36540d43550..6f7cabeac80 160000 --- a/contrib/rocksdb +++ b/contrib/rocksdb @@ -1 +1 @@ -Subproject commit 36540d43550da19e96515bd731ee416a8787672c +Subproject commit 6f7cabeac80a3a6150be2c8a8369fcecb107bf43 diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt index 57c056532c6..0157f31d5d9 100644 --- a/contrib/rocksdb-cmake/CMakeLists.txt +++ b/contrib/rocksdb-cmake/CMakeLists.txt @@ -150,6 +150,7 @@ set(SOURCES ${ROCKSDB_SOURCE_DIR}/db/memtable_list.cc ${ROCKSDB_SOURCE_DIR}/db/merge_helper.cc ${ROCKSDB_SOURCE_DIR}/db/merge_operator.cc + ${ROCKSDB_SOURCE_DIR}/db/multi_cf_iterator.cc ${ROCKSDB_SOURCE_DIR}/db/output_validator.cc ${ROCKSDB_SOURCE_DIR}/db/periodic_task_scheduler.cc ${ROCKSDB_SOURCE_DIR}/db/range_del_aggregator.cc From b3f5ddcd658efc8ad84b32481239750e8999ba0a Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 6 Aug 2024 10:37:35 +0000 Subject: [PATCH 084/129] Bump rocksdb to 9.2.1 --- contrib/rocksdb | 2 +- contrib/rocksdb-cmake/CMakeLists.txt | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/contrib/rocksdb b/contrib/rocksdb index 6f7cabeac80..08f93221f50 160000 --- a/contrib/rocksdb +++ b/contrib/rocksdb @@ -1 +1 @@ -Subproject commit 6f7cabeac80a3a6150be2c8a8369fcecb107bf43 +Subproject commit 08f93221f50700f19f11555fb46abfe708a716d1 diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt index 0157f31d5d9..cff8f832f23 100644 --- a/contrib/rocksdb-cmake/CMakeLists.txt +++ b/contrib/rocksdb-cmake/CMakeLists.txt @@ -88,6 +88,7 @@ set(SOURCES ${ROCKSDB_SOURCE_DIR}/cache/sharded_cache.cc ${ROCKSDB_SOURCE_DIR}/cache/tiered_secondary_cache.cc ${ROCKSDB_SOURCE_DIR}/db/arena_wrapped_db_iter.cc + ${ROCKSDB_SOURCE_DIR}/db/attribute_group_iterator_impl.cc ${ROCKSDB_SOURCE_DIR}/db/blob/blob_contents.cc ${ROCKSDB_SOURCE_DIR}/db/blob/blob_fetcher.cc ${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_addition.cc @@ -104,6 +105,7 @@ set(SOURCES ${ROCKSDB_SOURCE_DIR}/db/blob/prefetch_buffer_collection.cc ${ROCKSDB_SOURCE_DIR}/db/builder.cc ${ROCKSDB_SOURCE_DIR}/db/c.cc + ${ROCKSDB_SOURCE_DIR}/db/coalescing_iterator.cc ${ROCKSDB_SOURCE_DIR}/db/column_family.cc ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction.cc ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_iterator.cc @@ -150,7 +152,6 @@ set(SOURCES ${ROCKSDB_SOURCE_DIR}/db/memtable_list.cc ${ROCKSDB_SOURCE_DIR}/db/merge_helper.cc ${ROCKSDB_SOURCE_DIR}/db/merge_operator.cc - ${ROCKSDB_SOURCE_DIR}/db/multi_cf_iterator.cc ${ROCKSDB_SOURCE_DIR}/db/output_validator.cc ${ROCKSDB_SOURCE_DIR}/db/periodic_task_scheduler.cc ${ROCKSDB_SOURCE_DIR}/db/range_del_aggregator.cc @@ -389,6 +390,7 @@ set(SOURCES ${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_prepared_txn_db.cc ${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_unprepared_txn.cc ${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_unprepared_txn_db.cc + ${ROCKSDB_SOURCE_DIR}/utilities/types_util.cc ${ROCKSDB_SOURCE_DIR}/utilities/ttl/db_ttl_impl.cc ${ROCKSDB_SOURCE_DIR}/utilities/wal_filter.cc ${ROCKSDB_SOURCE_DIR}/utilities/write_batch_with_index/write_batch_with_index.cc @@ -419,10 +421,12 @@ if(HAS_ARMV8_CRC) endif(HAS_ARMV8_CRC) list(APPEND SOURCES - "${ROCKSDB_SOURCE_DIR}/port/port_posix.cc" - "${ROCKSDB_SOURCE_DIR}/env/env_posix.cc" - "${ROCKSDB_SOURCE_DIR}/env/fs_posix.cc" - "${ROCKSDB_SOURCE_DIR}/env/io_posix.cc") + ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_follower.cc + ${ROCKSDB_SOURCE_DIR}/port/port_posix.cc + ${ROCKSDB_SOURCE_DIR}/env/env_posix.cc + ${ROCKSDB_SOURCE_DIR}/env/fs_on_demand.cc + ${ROCKSDB_SOURCE_DIR}/env/fs_posix.cc + ${ROCKSDB_SOURCE_DIR}/env/io_posix.cc) add_library(_rocksdb ${SOURCES}) add_library(ch_contrib::rocksdb ALIAS _rocksdb) From e7110be48c7b2b752cd51fd72a23fe956248077c Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 6 Aug 2024 10:43:21 +0000 Subject: [PATCH 085/129] Bump rocksdb to v9.3.1 --- contrib/rocksdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/rocksdb b/contrib/rocksdb index 08f93221f50..c5201abc4a9 160000 --- a/contrib/rocksdb +++ b/contrib/rocksdb @@ -1 +1 @@ -Subproject commit 08f93221f50700f19f11555fb46abfe708a716d1 +Subproject commit c5201abc4a983450f9423435a4405829be23d0a8 From 2a2ba4e685022d3687565586c06dc56dfc276f10 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 6 Aug 2024 10:47:03 +0000 Subject: [PATCH 086/129] Bump rocksdb to 9.4.0 --- contrib/rocksdb | 2 +- contrib/rocksdb-cmake/CMakeLists.txt | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/rocksdb b/contrib/rocksdb index c5201abc4a9..5f003e4a22d 160000 --- a/contrib/rocksdb +++ b/contrib/rocksdb @@ -1 +1 @@ -Subproject commit c5201abc4a983450f9423435a4405829be23d0a8 +Subproject commit 5f003e4a22d2e48e37c98d9620241237cd30dd24 diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt index cff8f832f23..b39a739d04c 100644 --- a/contrib/rocksdb-cmake/CMakeLists.txt +++ b/contrib/rocksdb-cmake/CMakeLists.txt @@ -126,6 +126,7 @@ set(SOURCES ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_write.cc ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_compaction_flush.cc ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_files.cc + ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_follower.cc ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_open.cc ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_debug.cc ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_experimental.cc @@ -183,6 +184,7 @@ set(SOURCES ${ROCKSDB_SOURCE_DIR}/env/env_encryption.cc ${ROCKSDB_SOURCE_DIR}/env/file_system.cc ${ROCKSDB_SOURCE_DIR}/env/file_system_tracer.cc + ${ROCKSDB_SOURCE_DIR}/env/fs_on_demand.cc ${ROCKSDB_SOURCE_DIR}/env/fs_remap.cc ${ROCKSDB_SOURCE_DIR}/env/mock_env.cc ${ROCKSDB_SOURCE_DIR}/env/unique_id_gen.cc @@ -370,6 +372,7 @@ set(SOURCES ${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/volatile_tier_impl.cc ${ROCKSDB_SOURCE_DIR}/utilities/simulator_cache/cache_simulator.cc ${ROCKSDB_SOURCE_DIR}/utilities/simulator_cache/sim_cache.cc + ${ROCKSDB_SOURCE_DIR}/utilities/table_properties_collectors/compact_for_tiering_collector.cc ${ROCKSDB_SOURCE_DIR}/utilities/table_properties_collectors/compact_on_deletion_collector.cc ${ROCKSDB_SOURCE_DIR}/utilities/trace/file_trace_reader_writer.cc ${ROCKSDB_SOURCE_DIR}/utilities/trace/replayer_impl.cc @@ -421,10 +424,8 @@ if(HAS_ARMV8_CRC) endif(HAS_ARMV8_CRC) list(APPEND SOURCES - ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_follower.cc ${ROCKSDB_SOURCE_DIR}/port/port_posix.cc ${ROCKSDB_SOURCE_DIR}/env/env_posix.cc - ${ROCKSDB_SOURCE_DIR}/env/fs_on_demand.cc ${ROCKSDB_SOURCE_DIR}/env/fs_posix.cc ${ROCKSDB_SOURCE_DIR}/env/io_posix.cc) From df0dac2f5b509438cce28214b78765c46439aa8c Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 6 Aug 2024 11:17:41 +0000 Subject: [PATCH 087/129] Enable jemalloc and liburing in rocksdb --- contrib/CMakeLists.txt | 2 +- contrib/rocksdb-cmake/CMakeLists.txt | 30 +++++++++++++++++----------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 977efda15ff..eb3afe0ccdf 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -71,7 +71,6 @@ add_contrib (zlib-ng-cmake zlib-ng) add_contrib (bzip2-cmake bzip2) add_contrib (minizip-ng-cmake minizip-ng) add_contrib (snappy-cmake snappy) -add_contrib (rocksdb-cmake rocksdb) add_contrib (thrift-cmake thrift) # parquet/arrow/orc add_contrib (arrow-cmake arrow) # requires: snappy, thrift, double-conversion @@ -148,6 +147,7 @@ add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift, avro, arro add_contrib (cppkafka-cmake cppkafka) add_contrib (libpqxx-cmake libpqxx) add_contrib (libpq-cmake libpq) +add_contrib (rocksdb-cmake rocksdb) # requires: jemalloc, snappy, zlib, lz4, zstd, liburing add_contrib (nuraft-cmake NuRaft) add_contrib (fast_float-cmake fast_float) add_contrib (idna-cmake idna) diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt index b39a739d04c..7e5e9a28d0f 100644 --- a/contrib/rocksdb-cmake/CMakeLists.txt +++ b/contrib/rocksdb-cmake/CMakeLists.txt @@ -5,36 +5,38 @@ if (NOT ENABLE_ROCKSDB OR NO_SSE3_OR_HIGHER) # assumes SSE4.2 and PCLMUL return() endif() -# not in original build system, otherwise xxHash.cc fails to compile with ClickHouse C++23 default -set (CMAKE_CXX_STANDARD 20) - -# Always disable jemalloc for rocksdb by default because it introduces non-standard jemalloc APIs -option(WITH_JEMALLOC "build with JeMalloc" OFF) - -option(WITH_LIBURING "build with liburing" OFF) # TODO could try to enable this conditionally, depending on ClickHouse's ENABLE_LIBURING - # ClickHouse cannot be compiled without snappy, lz4, zlib, zstd option(WITH_SNAPPY "build with SNAPPY" ON) option(WITH_LZ4 "build with lz4" ON) option(WITH_ZLIB "build with zlib" ON) option(WITH_ZSTD "build with zstd" ON) -if(WITH_SNAPPY) +if (ENABLE_JEMALLOC) + add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE) + list (APPEND THIRDPARTY_LIBS ch_contrib::jemalloc) +endif () + +if (ENABLE_LIBURING) + add_definitions(-DROCKSDB_IOURING_PRESENT) + list (APPEND THIRDPARTY_LIBS ch_contrib::liburing) +endif () + +if (WITH_SNAPPY) add_definitions(-DSNAPPY) list(APPEND THIRDPARTY_LIBS ch_contrib::snappy) endif() -if(WITH_ZLIB) +if (WITH_ZLIB) add_definitions(-DZLIB) list(APPEND THIRDPARTY_LIBS ch_contrib::zlib) endif() -if(WITH_LZ4) +if (WITH_LZ4) add_definitions(-DLZ4) list(APPEND THIRDPARTY_LIBS ch_contrib::lz4) endif() -if(WITH_ZSTD) +if (WITH_ZSTD) add_definitions(-DZSTD) list(APPEND THIRDPARTY_LIBS ch_contrib::zstd) endif() @@ -433,5 +435,9 @@ add_library(_rocksdb ${SOURCES}) add_library(ch_contrib::rocksdb ALIAS _rocksdb) target_link_libraries(_rocksdb PRIVATE ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) +# Not in the native build system but useful anyways: +# Make all functions in xxHash.h inline. Beneficial for performance: https://github.com/Cyan4973/xxHash/tree/v0.8.2#build-modifiers +target_compile_definitions (_rocksdb PRIVATE XXH_INLINE_ALL) + # SYSTEM is required to overcome some issues target_include_directories(_rocksdb SYSTEM BEFORE INTERFACE "${ROCKSDB_SOURCE_DIR}/include") From 523767c0c0725773c074e1ad00229b27865a34f2 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Tue, 30 Jul 2024 17:26:28 +0200 Subject: [PATCH 088/129] run 01171 test in parallel --- .../01171_mv_select_insert_isolation_long.sh | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh index 2fb58e4cc57..27f063c34d8 100755 --- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh +++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-parallel, no-ordinary-database -# Test is too heavy, avoid parallel run in Flaky Check +# Tags: long, no-ordinary-database # shellcheck disable=SC2119 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -22,11 +21,6 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE tmp (x UInt32, nm Int32) ENGINE=MergeTr $CLICKHOUSE_CLIENT --query "INSERT INTO src VALUES (0, 0)" -function get_now() -{ - date +%s -} - is_pid_exist() { local pid=$1 @@ -42,14 +36,14 @@ function run_until_deadline_and_at_least_times() local function_to_run=$1; shift local started_time - started_time=$(get_now) + started_time=$SECONDS local i=0 while true do $function_to_run $i "$@" - [[ $(get_now) -lt $deadline ]] || break + [[ $SECONDS -lt $deadline ]] || break i=$(($i + 1)) done @@ -165,7 +159,7 @@ if [[ $((MAIN_TIME_PART + SECOND_TIME_PART + WAIT_FINISH + LAST_TIME_GAP)) -ge exit 1 fi -START_TIME=$(get_now) +START_TIME=$SECONDS STOP_TIME=$((START_TIME + MAIN_TIME_PART)) SECOND_STOP_TIME=$((STOP_TIME + SECOND_TIME_PART)) MIN_ITERATIONS=20 From 244da490138668882db10770679b940d2c4adadf Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Tue, 30 Jul 2024 18:41:44 +0200 Subject: [PATCH 089/129] add max iteration --- .../01171_mv_select_insert_isolation_long.sh | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh index 27f063c34d8..fe5bfe529b7 100755 --- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh +++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh @@ -27,28 +27,30 @@ is_pid_exist() ps -p $pid > /dev/null } -function run_until_deadline_and_at_least_times() +function run_until_deadline_with_min_max_iterations() { set -e local deadline=$1; shift local min_iterations=$1; shift + local max_iterations=$1; shift local function_to_run=$1; shift local started_time started_time=$SECONDS - local i=0 + local iteration=0 while true do - $function_to_run $i "$@" + $function_to_run $iteration "$@" [[ $SECONDS -lt $deadline ]] || break + [[ $iteration -lt $max_iterations ]] || break - i=$(($i + 1)) + iteration=$(($iteration + 1)) done - [[ $i -gt $min_iterations ]] || echo "$i/$min_iterations : not enough iterations of $function_to_run has been made from $started_time until $deadline" >&2 + [[ $iteration -gt $min_iterations ]] || echo "$iteration/$min_iterations : not enough iterations of $function_to_run has been made from $started_time until $deadline" >&2 } function insert_commit_action() @@ -163,17 +165,18 @@ START_TIME=$SECONDS STOP_TIME=$((START_TIME + MAIN_TIME_PART)) SECOND_STOP_TIME=$((STOP_TIME + SECOND_TIME_PART)) MIN_ITERATIONS=20 +MAX_ITERATIONS=200 -run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_commit_action 1 & PID_1=$! -run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_commit_action 2 & PID_2=$! -run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_rollback_action 3 & PID_3=$! +run_until_deadline_with_min_max_iterations $STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_commit_action 1 & PID_1=$! +run_until_deadline_with_min_max_iterations $STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_commit_action 2 & PID_2=$! +run_until_deadline_with_min_max_iterations $STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_rollback_action 3 & PID_3=$! -run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS optimize_action & PID_4=$! -run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS select_action & PID_5=$! -run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS select_insert_action & PID_6=$! +run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS optimize_action & PID_4=$! +run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_action & PID_5=$! +run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_insert_action & PID_6=$! sleep 0.$RANDOM -run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS select_action & PID_7=$! -run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS select_insert_action & PID_8=$! +run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_action & PID_7=$! +run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_insert_action & PID_8=$! wait $PID_1 || echo "insert_commit_action has failed with status $?" 2>&1 wait $PID_2 || echo "second insert_commit_action has failed with status $?" 2>&1 From c896165e161b2267ae4d6bc255e028d7c0899bd9 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 31 Jul 2024 17:49:39 +0200 Subject: [PATCH 090/129] add min time --- .../01171_mv_select_insert_isolation_long.sh | 53 +++++++++++-------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh index fe5bfe529b7..ff71d37cb32 100755 --- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh +++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh @@ -27,15 +27,20 @@ is_pid_exist() ps -p $pid > /dev/null } -function run_until_deadline_with_min_max_iterations() +function run_with_time_and_iterations_limits() { set -e - local deadline=$1; shift + local min_time=$1; shift + local max_time=$1; shift local min_iterations=$1; shift local max_iterations=$1; shift local function_to_run=$1; shift + # if [ "${1:-X}" = "1" ]; then + # set -x + # fi + local started_time started_time=$SECONDS local iteration=0 @@ -44,13 +49,14 @@ function run_until_deadline_with_min_max_iterations() do $function_to_run $iteration "$@" - [[ $SECONDS -lt $deadline ]] || break - [[ $iteration -lt $max_iterations ]] || break + [[ $SECONDS -lt $max_time ]] || break + [[ $SECONDS -lt $min_time ]] || [[ $iteration -lt $max_iterations ]] || break iteration=$(($iteration + 1)) done - [[ $iteration -gt $min_iterations ]] || echo "$iteration/$min_iterations : not enough iterations of $function_to_run has been made from $started_time until $deadline" >&2 + [[ $iteration -gt $min_iterations ]] || echo "$iteration/$min_iterations : not enough iterations of $function_to_run has been made from $started_time until $max_time" >&2 + set +x } function insert_commit_action() @@ -151,43 +157,44 @@ function select_insert_action() ROLLBACK;" } -MAIN_TIME_PART=400 -SECOND_TIME_PART=30 +MIN_SECONDS=5 +MAX_SECONDS=300 WAIT_FINISH=60 -LAST_TIME_GAP=10 -if [[ $((MAIN_TIME_PART + SECOND_TIME_PART + WAIT_FINISH + LAST_TIME_GAP)) -ge 600 ]]; then +if [[ $((MAX_SECONDS + WAIT_FINISH)) -ge 550 ]]; then echo "time sttings are wrong" 2>&1 exit 1 fi START_TIME=$SECONDS -STOP_TIME=$((START_TIME + MAIN_TIME_PART)) -SECOND_STOP_TIME=$((STOP_TIME + SECOND_TIME_PART)) +MIN_TIME=$((START_TIME + MIN_SECONDS)) +MAX_TIME=$((START_TIME + MAX_SECONDS)) MIN_ITERATIONS=20 MAX_ITERATIONS=200 -run_until_deadline_with_min_max_iterations $STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_commit_action 1 & PID_1=$! -run_until_deadline_with_min_max_iterations $STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_commit_action 2 & PID_2=$! -run_until_deadline_with_min_max_iterations $STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_rollback_action 3 & PID_3=$! +run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_commit_action 1 & PID_1=$! +run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_commit_action 2 & PID_2=$! +run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_rollback_action 3 & PID_3=$! -run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS optimize_action & PID_4=$! -run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_action & PID_5=$! -run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_insert_action & PID_6=$! +run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS optimize_action & PID_4=$! +run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_action & PID_5=$! +run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_insert_action & PID_6=$! sleep 0.$RANDOM -run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_action & PID_7=$! -run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_insert_action & PID_8=$! - -wait $PID_1 || echo "insert_commit_action has failed with status $?" 2>&1 -wait $PID_2 || echo "second insert_commit_action has failed with status $?" 2>&1 -wait $PID_3 || echo "insert_rollback_action has failed with status $?" 2>&1 +run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_action & PID_7=$! +run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_insert_action & PID_8=$! +is_pid_exist $PID_1 || echo "insert_commit_action is not running" 2>&1 +is_pid_exist $PID_2 || echo "second insert_commit_action is not running" 2>&1 +is_pid_exist $PID_3 || echo "insert_rollback_action is not running" 2>&1 is_pid_exist $PID_4 || echo "optimize_action is not running" 2>&1 is_pid_exist $PID_5 || echo "select_action is not running" 2>&1 is_pid_exist $PID_6 || echo "select_insert_action is not running" 2>&1 is_pid_exist $PID_7 || echo "second select_action is not running" 2>&1 is_pid_exist $PID_8 || echo "second select_insert_action is not running" 2>&1 +wait $PID_1 || echo "insert_commit_action has failed with status $?" 2>&1 +wait $PID_2 || echo "second insert_commit_action has failed with status $?" 2>&1 +wait $PID_3 || echo "insert_rollback_action has failed with status $?" 2>&1 wait $PID_4 || echo "optimize_action has failed with status $?" 2>&1 wait $PID_5 || echo "select_action has failed with status $?" 2>&1 wait $PID_6 || echo "select_insert_action has failed with status $?" 2>&1 From 874a7dbe378e83331d6e4fab6aa82a0409fbfc5d Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 1 Aug 2024 17:39:36 +0200 Subject: [PATCH 091/129] adjust settings --- .../0_stateless/01171_mv_select_insert_isolation_long.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh index ff71d37cb32..ba6fc85ca1a 100755 --- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh +++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh @@ -158,7 +158,7 @@ function select_insert_action() } MIN_SECONDS=5 -MAX_SECONDS=300 +MAX_SECONDS=400 WAIT_FINISH=60 if [[ $((MAX_SECONDS + WAIT_FINISH)) -ge 550 ]]; then From 860050eb3d15f26c371a1aeab92fd8c0d166ce09 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 6 Aug 2024 13:30:23 +0200 Subject: [PATCH 092/129] Update src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp Co-authored-by: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> --- .../ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp index 9043edd66a0..1a0b6157a86 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp @@ -268,7 +268,7 @@ std::unique_ptr getContainerClient(const ConnectionParams & par catch (const Azure::Storage::StorageException & e) { /// If container_already_exists is not set (in config), ignore already exists error. Conflict - The specified container already exists. - /// To avoid race with creation of container handle this error despite that we have already checked the existence of container. + /// To avoid race with creation of container, handle this error despite that we have already checked the existence of container. if (!params.endpoint.container_already_exists.has_value() && e.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict) return params.createForContainer(); throw; From 851d4d3fdfdb682d0f9c9a8b1945608c6bc53086 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Tue, 6 Aug 2024 13:56:37 +0200 Subject: [PATCH 093/129] adjust MIN_ITERATIONS --- .../0_stateless/01171_mv_select_insert_isolation_long.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh index ba6fc85ca1a..f6a88c205c1 100755 --- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh +++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh @@ -169,7 +169,7 @@ fi START_TIME=$SECONDS MIN_TIME=$((START_TIME + MIN_SECONDS)) MAX_TIME=$((START_TIME + MAX_SECONDS)) -MIN_ITERATIONS=20 +MIN_ITERATIONS=15 MAX_ITERATIONS=200 run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_commit_action 1 & PID_1=$! From 5313c9539ef8f988a607da87849eaa1f12e787e1 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 6 Aug 2024 14:34:06 +0200 Subject: [PATCH 094/129] update documentation for Float32/64 and variants --- .../functions/type-conversion-functions.md | 420 +++++++++++++++++- 1 file changed, 416 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 1e618b8cdab..77dd1628fe4 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -3045,13 +3045,425 @@ toUInt256OrDefault('abc', CAST('0', 'UInt256')): 0 - [`toUInt256OrZero`](#touint256orzero). - [`toUInt256OrNull`](#touint256ornull). -## toFloat(32\|64) +## toFloat32 -## toFloat(32\|64)OrZero +Converts an input value to a value of type [`Float32`](../data-types/float.md). Throws an exception in case of an error. -## toFloat(32\|64)OrNull +**Syntax** -## toFloat(32\|64)OrDefault +```sql +toFloat32(expr) +``` + +**Arguments** + +- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions). + +Supported arguments: +- Values of type (U)Int8/16/32/64/128/256. +- String representations of (U)Int8/16/32/128/256. +- Values of type Float32/64, including `NaN` and `Inf`. +- String representations of Float32/64, including `NaN` and `Inf`. + +Unsupported arguments: +- String representations of binary and hexadecimal values, e.g. `SELECT toFloat32('0xc0fe');`. + +**Returned value** + +- 32-bit floating point value. [Float32](../data-types/float.md). + +**Example** + +Query: + +```sql +SELECT + toFloat32(42.7), + toFloat32('42.7'), + toFloat32('NaN') +FORMAT vertical; +``` + +Result: + +```response +Row 1: +────── +toFloat32(42.7): 42.7 +toFloat32('42.7'): 42.7 +toFloat32('NaN'): nan +``` + +**See also** + +- [`toFloat32OrZero`](#tofloat32orzero). +- [`toFloat32OrNull`](#tofloat32ornull). +- [`toFloat32OrDefault`](#tofloat32ordefault). + +## toFloat32OrZero + +Like [`toFloat32`](#tofloat32), this function converts an input value to a value of type [Float32](../data-types/float.md) but returns `0` in case of an error. + +**Syntax** + +```sql +toFloat32OrZero(x) +``` + +**Arguments** + +- `x` — A String representation of a number. [String](../data-types/string.md). + +Supported arguments: +- String representations of (U)Int8/16/32/128/256, Float32/64. + +Unsupported arguments (return `0`): +- String representations of binary and hexadecimal values, e.g. `SELECT toFloat32OrZero('0xc0fe');`. + +**Returned value** + +- 32-bit Float value if successful, otherwise `0`. [Float32](../data-types/float.md). + +**Example** + +Query: + +``` sql +SELECT + toFloat32OrZero('42.7'), + toFloat32OrZero('abc') +FORMAT vertical; +``` + +Result: + +```response +Row 1: +────── +toFloat32OrZero('42.7'): 42.7 +toFloat32OrZero('abc'): 0 +``` + +**See also** + +- [`toFloat32`](#tofloat32). +- [`toFloat32OrNull`](#tofloat32ornull). +- [`toFloat32OrDefault`](#tofloat32ordefault). + +## toFloat32OrNull + +Like [`toFloat32`](#tofloat32), this function converts an input value to a value of type [Float32](../data-types/float.md) but returns `NULL` in case of an error. + +**Syntax** + +```sql +toFloat32OrNull(x) +``` + +**Arguments** + +- `x` — A String representation of a number. [String](../data-types/string.md). + +Supported arguments: +- String representations of (U)Int8/16/32/128/256, Float32/64. + +Unsupported arguments (return `\N`): +- String representations of binary and hexadecimal values, e.g. `SELECT toFloat32OrNull('0xc0fe');`. + +**Returned value** + +- 32-bit Float value if successful, otherwise `\N`. [Float32](../data-types/float.md). + +**Example** + +Query: + +``` sql +SELECT + toFloat32OrNull('42.7'), + toFloat32OrNull('abc') +FORMAT vertical; +``` + +Result: + +```response +Row 1: +────── +toFloat32OrNull('42.7'): 42.7 +toFloat32OrNull('abc'): ᴺᵁᴸᴸ +``` + +**See also** + +- [`toFloat32`](#tofloat32). +- [`toFloat32OrZero`](#tofloat32orzero). +- [`toFloat32OrDefault`](#tofloat32ordefault). + +## toFloat32OrDefault + +Like [`toFloat32`](#tofloat32), this function converts an input value to a value of type [Float32](../data-types/float.md) but returns the default value in case of an error. +If no `default` value is passed then `0` is returned in case of an error. + +**Syntax** + +```sql +toFloat32OrDefault(expr[, default]) +``` + +**Arguments** + +- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md). +- `default` (optional) — The default value to return if parsing to type `Float32` is unsuccessful. [Float32](../data-types/float.md). + +Supported arguments: +- Values of type (U)Int8/16/32/64/128/256. +- String representations of (U)Int8/16/32/128/256. +- Values of type Float32/64. +- String representations of Float32/64. + +Arguments for which the default value is returned: +- String representations of binary and hexadecimal values, e.g. `SELECT toFloat32OrDefault('0xc0fe', CAST('0', 'Float32'));`. + +**Returned value** + +- 32-bit Float value if successful, otherwise returns the default value if passed or `0` if not. [Float32](../data-types/float.md). + +**Example** + +Query: + +``` sql +SELECT + toFloat32OrDefault('8', CAST('0', 'Float32')), + toFloat32OrDefault('abc', CAST('0', 'Float32')) +FORMAT vertical; +``` + +Result: + +```response +Row 1: +────── +toFloat32OrDefault('8', CAST('0', 'Float32')): 8 +toFloat32OrDefault('abc', CAST('0', 'Float32')): 0 +``` + +**See also** + +- [`toFloat32`](#tofloat32). +- [`toFloat32OrZero`](#tofloat32orzero). +- [`toFloat32OrNull`](#tofloat32ornull). + +## toFloat64 + +Converts an input value to a value of type [`Float64`](../data-types/float.md). Throws an exception in case of an error. + +**Syntax** + +```sql +toFloat64(expr) +``` + +**Arguments** + +- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions). + +Supported arguments: +- Values of type (U)Int8/16/32/64/128/256. +- String representations of (U)Int8/16/32/128/256. +- Values of type Float32/64, including `NaN` and `Inf`. +- String representations of type Float32/64, including `NaN` and `Inf`. + +Unsupported arguments: +- String representations of binary and hexadecimal values, e.g. `SELECT toFloat64('0xc0fe');`. + +**Returned value** + +- 64-bit floating point value. [Float64](../data-types/float.md). + +**Example** + +Query: + +```sql +SELECT + toFloat64(42.7), + toFloat64('42.7'), + toFloat64('NaN') +FORMAT vertical; +``` + +Result: + +```response +Row 1: +────── +toFloat64(42.7): 42.7 +toFloat64('42.7'): 42.7 +toFloat64('NaN'): nan +``` + +**See also** + +- [`toFloat64OrZero`](#tofloat64orzero). +- [`toFloat64OrNull`](#tofloat64ornull). +- [`toFloat64OrDefault`](#tofloat64ordefault). + +## toFloat64OrZero + +Like [`toFloat64`](#tofloat64), this function converts an input value to a value of type [Float64](../data-types/float.md) but returns `0` in case of an error. + +**Syntax** + +```sql +toFloat64OrZero(x) +``` + +**Arguments** + +- `x` — A String representation of a number. [String](../data-types/string.md). + +Supported arguments: +- String representations of (U)Int8/16/32/128/256, Float32/64. + +Unsupported arguments (return `0`): +- String representations of binary and hexadecimal values, e.g. `SELECT toFloat64OrZero('0xc0fe');`. + +**Returned value** + +- 64-bit Float value if successful, otherwise `0`. [Float64](../data-types/float.md). + +**Example** + +Query: + +``` sql +SELECT + toFloat64OrZero('42.7'), + toFloat64OrZero('abc') +FORMAT vertical; +``` + +Result: + +```response +Row 1: +────── +toFloat64OrZero('42.7'): 42.7 +toFloat64OrZero('abc'): 0 +``` + +**See also** + +- [`toFloat64`](#tofloat64). +- [`toFloat64OrNull`](#tofloat64ornull). +- [`toFloat64OrDefault`](#tofloat64ordefault). + +## toFloat64OrNull + +Like [`toFloat64`](#tofloat64), this function converts an input value to a value of type [Float64](../data-types/float.md) but returns `NULL` in case of an error. + +**Syntax** + +```sql +toFloat64OrNull(x) +``` + +**Arguments** + +- `x` — A String representation of a number. [String](../data-types/string.md). + +Supported arguments: +- String representations of (U)Int8/16/32/128/256, Float32/64. + +Unsupported arguments (return `\N`): +- String representations of binary and hexadecimal values, e.g. `SELECT toFloat64OrNull('0xc0fe');`. + +**Returned value** + +- 64-bit Float value if successful, otherwise `\N`. [Float64](../data-types/float.md). + +**Example** + +Query: + +``` sql +SELECT + toFloat64OrNull('42.7'), + toFloat64OrNull('abc') +FORMAT vertical; +``` + +Result: + +```response +Row 1: +────── +toFloat64OrNull('42.7'): 42.7 +toFloat64OrNull('abc'): ᴺᵁᴸᴸ +``` + +**See also** + +- [`toFloat64`](#tofloat64). +- [`toFloat64OrZero`](#tofloat64orzero). +- [`toFloat64OrDefault`](#tofloat64ordefault). + +## toFloat64OrDefault + +Like [`toFloat64`](#tofloat64), this function converts an input value to a value of type [Float64](../data-types/float.md) but returns the default value in case of an error. +If no `default` value is passed then `0` is returned in case of an error. + +**Syntax** + +```sql +toFloat64OrDefault(expr[, default]) +``` + +**Arguments** + +- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md). +- `default` (optional) — The default value to return if parsing to type `Float64` is unsuccessful. [Float64](../data-types/float.md). + +Supported arguments: +- Values of type (U)Int8/16/32/64/128/256. +- String representations of (U)Int8/16/32/128/256. +- Values of type Float32/64. +- String representations of Float32/64. + +Arguments for which the default value is returned: +- String representations of binary and hexadecimal values, e.g. `SELECT toFloat64OrDefault('0xc0fe', CAST('0', 'Float64'));`. + +**Returned value** + +- 64-bit Float value if successful, otherwise returns the default value if passed or `0` if not. [Float64](../data-types/float.md). + +**Example** + +Query: + +``` sql +SELECT + toFloat64OrDefault('8', CAST('0', 'Float64')), + toFloat64OrDefault('abc', CAST('0', 'Float64')) +FORMAT vertical; +``` + +Result: + +```response +Row 1: +────── +toFloat64OrDefault('8', CAST('0', 'Float64')): 8 +toFloat64OrDefault('abc', CAST('0', 'Float64')): 0 +``` + +**See also** + +- [`toFloat64`](#tofloat64). +- [`toFloat64OrZero`](#tofloat64orzero). +- [`toFloat64OrNull`](#tofloat64ornull). ## toDate From dbfba5ebc4d011d13ab78f91700fae12b46ce58b Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 6 Aug 2024 12:56:52 +0000 Subject: [PATCH 095/129] Validate data types in ALTER ADD/MODIFY COLUMN --- src/Storages/AlterCommands.cpp | 5 +++++ ..._type_in_alter_add_modify_column.reference | 0 ...lidate_type_in_alter_add_modify_column.sql | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 tests/queries/0_stateless/03215_validate_type_in_alter_add_modify_column.reference create mode 100644 tests/queries/0_stateless/03215_validate_type_in_alter_add_modify_column.sql diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 7891042bb96..dfb388ffdb2 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -1316,6 +1317,8 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const throw Exception(ErrorCodes::BAD_ARGUMENTS, "Data type have to be specified for column {} to add", backQuote(column_name)); + validateDataType(command.data_type, DataTypeValidationSettings(context->getSettingsRef())); + /// FIXME: Adding a new column of type Object(JSON) is broken. /// Looks like there is something around default expression for this column (method `getDefault` is not implemented for the data type Object). /// But after ALTER TABLE ADD COLUMN we need to fill existing rows with something (exactly the default value). @@ -1395,6 +1398,8 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const /// So we don't allow to do it for now. if (command.data_type) { + validateDataType(command.data_type, DataTypeValidationSettings(context->getSettingsRef())); + const GetColumnsOptions options(GetColumnsOptions::All); const auto old_data_type = all_columns.getColumn(options, column_name).type; diff --git a/tests/queries/0_stateless/03215_validate_type_in_alter_add_modify_column.reference b/tests/queries/0_stateless/03215_validate_type_in_alter_add_modify_column.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03215_validate_type_in_alter_add_modify_column.sql b/tests/queries/0_stateless/03215_validate_type_in_alter_add_modify_column.sql new file mode 100644 index 00000000000..267bc7111f4 --- /dev/null +++ b/tests/queries/0_stateless/03215_validate_type_in_alter_add_modify_column.sql @@ -0,0 +1,19 @@ +set allow_experimental_variant_type = 0; +set allow_experimental_dynamic_type = 0; +set allow_suspicious_low_cardinality_types = 0; +set allow_suspicious_fixed_string_types = 0; + +drop table if exists test; +create table test (id UInt64) engine=MergeTree order by id; +alter table test add column bad Variant(UInt32, String); -- {serverError ILLEGAL_COLUMN} +alter table test add column bad Dynamic; -- {serverError ILLEGAL_COLUMN} +alter table test add column bad LowCardinality(UInt8); -- {serverError SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY} +alter table test add column bad FixedString(10000); -- {serverError ILLEGAL_COLUMN} + +alter table test modify column id Variant(UInt32, String); -- {serverError ILLEGAL_COLUMN} +alter table test modify column id Dynamic; -- {serverError ILLEGAL_COLUMN} +alter table test modify column id LowCardinality(UInt8); -- {serverError SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY} +alter table test modify column id FixedString(10000); -- {serverError ILLEGAL_COLUMN} + +drop table test; + From e0e32b542560aeefa6863195d0207859ec5f338e Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 6 Aug 2024 13:26:12 +0000 Subject: [PATCH 096/129] Fix tests --- .../queries/0_stateless/02910_object-json-crash-add-column.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02910_object-json-crash-add-column.sql b/tests/queries/0_stateless/02910_object-json-crash-add-column.sql index b2d64be1676..bda5e958453 100644 --- a/tests/queries/0_stateless/02910_object-json-crash-add-column.sql +++ b/tests/queries/0_stateless/02910_object-json-crash-add-column.sql @@ -1,3 +1,5 @@ +SET allow_experimental_object_type=1; + DROP TABLE IF EXISTS test02910; CREATE TABLE test02910 From f37fcb776800cdb6bff4d89e5814c39bbbc46033 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Tue, 6 Aug 2024 14:32:20 +0100 Subject: [PATCH 097/129] impl --- src/Compression/fuzzers/CMakeLists.txt | 12 ++++++------ src/Storages/fuzzers/CMakeLists.txt | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Compression/fuzzers/CMakeLists.txt b/src/Compression/fuzzers/CMakeLists.txt index a693faecc14..311f1eb3d35 100644 --- a/src/Compression/fuzzers/CMakeLists.txt +++ b/src/Compression/fuzzers/CMakeLists.txt @@ -5,19 +5,19 @@ # If you want really small size of the resulted binary, just link with fuzz_compression and clickhouse_common_io clickhouse_add_executable (compressed_buffer_fuzzer compressed_buffer_fuzzer.cpp) -target_link_libraries (compressed_buffer_fuzzer PRIVATE dbms) +target_link_libraries (compressed_buffer_fuzzer PRIVATE dbms clickhouse_functions) clickhouse_add_executable (lz4_decompress_fuzzer lz4_decompress_fuzzer.cpp) -target_link_libraries (lz4_decompress_fuzzer PUBLIC dbms ch_contrib::lz4) +target_link_libraries (lz4_decompress_fuzzer PUBLIC dbms ch_contrib::lz4 clickhouse_functions) clickhouse_add_executable (delta_decompress_fuzzer delta_decompress_fuzzer.cpp) -target_link_libraries (delta_decompress_fuzzer PRIVATE dbms) +target_link_libraries (delta_decompress_fuzzer PRIVATE dbms clickhouse_functions) clickhouse_add_executable (double_delta_decompress_fuzzer double_delta_decompress_fuzzer.cpp) -target_link_libraries (double_delta_decompress_fuzzer PRIVATE dbms) +target_link_libraries (double_delta_decompress_fuzzer PRIVATE dbms clickhouse_functions) clickhouse_add_executable (encrypted_decompress_fuzzer encrypted_decompress_fuzzer.cpp) -target_link_libraries (encrypted_decompress_fuzzer PRIVATE dbms) +target_link_libraries (encrypted_decompress_fuzzer PRIVATE dbms clickhouse_functions) clickhouse_add_executable (gcd_decompress_fuzzer gcd_decompress_fuzzer.cpp) -target_link_libraries (gcd_decompress_fuzzer PRIVATE dbms) +target_link_libraries (gcd_decompress_fuzzer PRIVATE dbms clickhouse_functions) diff --git a/src/Storages/fuzzers/CMakeLists.txt b/src/Storages/fuzzers/CMakeLists.txt index e36fccec8df..ec56b853666 100644 --- a/src/Storages/fuzzers/CMakeLists.txt +++ b/src/Storages/fuzzers/CMakeLists.txt @@ -1,7 +1,7 @@ clickhouse_add_executable (mergetree_checksum_fuzzer mergetree_checksum_fuzzer.cpp) # Look at comment around fuzz_compression target declaration -target_link_libraries (mergetree_checksum_fuzzer PRIVATE dbms) +target_link_libraries (mergetree_checksum_fuzzer PRIVATE dbms clickhouse_functions) clickhouse_add_executable (columns_description_fuzzer columns_description_fuzzer.cpp) target_link_libraries (columns_description_fuzzer PRIVATE dbms clickhouse_functions) From dbcc5cf1333c6a3d8f5f8f1b7dc67d038055be2e Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Tue, 6 Aug 2024 15:39:08 +0200 Subject: [PATCH 098/129] CI: Changelog: Critical Bug Fix to Bug Fix --- tests/ci/changelog.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ci/changelog.py b/tests/ci/changelog.py index 929f0f3523a..fe47fe53a9e 100755 --- a/tests/ci/changelog.py +++ b/tests/ci/changelog.py @@ -38,7 +38,7 @@ categories_preferred_order = ( "Experimental Feature", "Performance Improvement", "Improvement", - "Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)", + #"Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)", "Bug Fix (user-visible misbehavior in an official stable release)", "Build/Testing/Packaging Improvement", "Other", @@ -294,7 +294,8 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri r"(?i)bug\Wfix", category, ) - and "Critical Bug Fix" not in category + # Map "Critical Bug Fix" to "Bug fix" category for changelog + #and "Critical Bug Fix" not in category ): category = "Bug Fix (user-visible misbehavior in an official stable release)" From b99c6c1153224b8d17dc663939b7c621dfe77ceb Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 6 Aug 2024 14:02:24 +0000 Subject: [PATCH 099/129] fix reading of size column from missed Nested in compact parts --- src/Storages/MergeTree/MergeTreeReaderCompact.cpp | 12 +++++++++++- src/Storages/MergeTree/MergeTreeReaderCompact.h | 3 ++- .../MergeTree/MergeTreeReaderCompactSingleBuffer.cpp | 7 ++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index ff0311dc1ca..d49ad61feca 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -147,7 +147,8 @@ void MergeTreeReaderCompact::readData( const NameAndTypePair & name_and_type, ColumnPtr & column, size_t rows_to_read, - const InputStreamGetter & getter) + const InputStreamGetter & getter, + ISerialization::SubstreamsCache & cache) { try { @@ -158,6 +159,13 @@ void MergeTreeReaderCompact::readData( deserialize_settings.getter = getter; deserialize_settings.avg_value_size_hint = avg_value_size_hints[name]; + auto it = cache.find(name); + if (it != cache.end() && it->second != nullptr) + { + column = it->second; + return; + } + if (name_and_type.isSubcolumn()) { const auto & type_in_storage = name_and_type.getTypeInStorage(); @@ -181,6 +189,8 @@ void MergeTreeReaderCompact::readData( serialization->deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name], nullptr); } + cache.emplace(name, column); + size_t read_rows_in_column = column->size() - column_size_before_reading; if (read_rows_in_column != rows_to_read) throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h index 22eabd47930..1c6bd1474e3 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -44,7 +44,8 @@ protected: const NameAndTypePair & name_and_type, ColumnPtr & column, size_t rows_to_read, - const InputStreamGetter & getter); + const InputStreamGetter & getter, + ISerialization::SubstreamsCache & cache); void readPrefix( const NameAndTypePair & name_and_type, diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp index 2b2cf493bb5..004ba4db028 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp @@ -22,6 +22,10 @@ try checkNumberOfColumns(num_columns); createColumnsForReading(res_columns); + /// Use cache to avoid reading the column with the same name twice. + /// It may happen if there are empty array Nested in the part. + std::unordered_map caches; + while (read_rows < max_rows_to_read) { size_t rows_to_read = data_part_info_for_read->getIndexGranularity().getMarkRows(from_mark); @@ -32,6 +36,7 @@ try continue; auto & column = res_columns[pos]; + auto & cache = caches[columns_to_read[pos].name]; stream->adjustRightMark(current_task_last_mark); /// Must go before seek. stream->seekToMarkAndColumn(from_mark, *column_positions[pos]); @@ -52,7 +57,7 @@ try }; readPrefix(columns_to_read[pos], buffer_getter, buffer_getter_for_prefix, columns_for_offsets[pos]); - readData(columns_to_read[pos], column, rows_to_read, buffer_getter); + readData(columns_to_read[pos], column, rows_to_read, buffer_getter, cache); } ++from_mark; From 5c4f2c198573ba03b793e4610f4e9854a61a5543 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 6 Aug 2024 14:33:28 +0000 Subject: [PATCH 100/129] Fix DateTime64 parsing after constant folding --- src/Analyzer/ConstantNode.cpp | 22 ++++++++++++++++++- ...03217_datetime64_constant_to_ast.reference | 2 ++ .../03217_datetime64_constant_to_ast.sql | 6 +++++ 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03217_datetime64_constant_to_ast.reference create mode 100644 tests/queries/0_stateless/03217_datetime64_constant_to_ast.sql diff --git a/src/Analyzer/ConstantNode.cpp b/src/Analyzer/ConstantNode.cpp index 46c1f7fb1ed..c65090f5b55 100644 --- a/src/Analyzer/ConstantNode.cpp +++ b/src/Analyzer/ConstantNode.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -162,6 +163,7 @@ QueryTreeNodePtr ConstantNode::cloneImpl() const ASTPtr ConstantNode::toASTImpl(const ConvertToASTOptions & options) const { const auto & constant_value_literal = constant_value->getValue(); + const auto & constant_value_type = constant_value->getType(); auto constant_value_ast = std::make_shared(constant_value_literal); if (!options.add_cast_for_constants) @@ -169,7 +171,25 @@ ASTPtr ConstantNode::toASTImpl(const ConvertToASTOptions & options) const if (requiresCastCall()) { - auto constant_type_name_ast = std::make_shared(constant_value->getType()->getName()); + /** Value for DateTime64 is Decimal64, which is serialized as a string literal. + * If we serialize it as is, DateTime64 would be parsed from that string literal, which can be incorrect. + * For example, DateTime64 cannot be parsed from the short value, like '1', while it's a valid Decimal64 value. + * It could also lead to ambiguous parsing because we don't know if the string literal represents a date or a Decimal64 literal. + * For this reason, we use a string literal representing a date instead of a Decimal64 literal. + */ + if (WhichDataType(constant_value_type->getTypeId()).isDateTime64()) + { + const auto * date_time_type = typeid_cast(constant_value_type.get()); + DecimalField decimal_value; + if (constant_value_literal.tryGet>(decimal_value)) + { + WriteBufferFromOwnString ostr; + writeDateTimeText(decimal_value.getValue(), date_time_type->getScale(), ostr, date_time_type->getTimeZone()); + constant_value_ast = std::make_shared(ostr.str()); + } + } + + auto constant_type_name_ast = std::make_shared(constant_value_type->getName()); return makeASTFunction("_CAST", std::move(constant_value_ast), std::move(constant_type_name_ast)); } diff --git a/tests/queries/0_stateless/03217_datetime64_constant_to_ast.reference b/tests/queries/0_stateless/03217_datetime64_constant_to_ast.reference new file mode 100644 index 00000000000..c20baa0d261 --- /dev/null +++ b/tests/queries/0_stateless/03217_datetime64_constant_to_ast.reference @@ -0,0 +1,2 @@ +1970-01-01 00:00:01.000 +1970-01-01 00:00:01.000 diff --git a/tests/queries/0_stateless/03217_datetime64_constant_to_ast.sql b/tests/queries/0_stateless/03217_datetime64_constant_to_ast.sql new file mode 100644 index 00000000000..63334a511c7 --- /dev/null +++ b/tests/queries/0_stateless/03217_datetime64_constant_to_ast.sql @@ -0,0 +1,6 @@ + +SET session_timezone = 'UTC'; + +SELECT toDateTime64('1970-01-01 00:00:01', 3) +FROM remote('127.0.0.{1,2}', system, one) +; From f2591bd1a6401337de0e3bf8d5c1fa93fd1b9394 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Tue, 6 Aug 2024 16:55:04 +0200 Subject: [PATCH 101/129] CI: CiBuddy with channel dispatcher --- tests/ci/ci_buddy.py | 102 ++++++++++++++++++++++++++++--------- tests/ci/create_release.py | 4 +- 2 files changed, 81 insertions(+), 25 deletions(-) diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py index f0e73e925fe..07f318207a4 100644 --- a/tests/ci/ci_buddy.py +++ b/tests/ci/ci_buddy.py @@ -1,7 +1,7 @@ import argparse import json import os -from typing import Union, Dict +from typing import Union, Dict, List import boto3 import requests @@ -9,20 +9,44 @@ from botocore.exceptions import ClientError from pr_info import PRInfo from ci_config import CI +from ci_utils import WithIter + + +class Channels(metaclass=WithIter): + # Channel names must match json keys in ParameterStore + ALERTS = "alerts-channel" + INFO = "info-channel" + DRY_RUN = "dry-ryn-channel" + DEFAULT = "default" class CIBuddy: + Channels = Channels _HEADERS = {"Content-Type": "application/json"} def __init__(self, dry_run=False): self.repo = os.getenv("GITHUB_REPOSITORY", "") self.dry_run = dry_run res = self._get_webhooks() - self.test_channel = "" - self.dev_ci_channel = "" + self.channels = {} if res: - self.test_channel = json.loads(res)["test_channel"] - self.dev_ci_channel = json.loads(res)["ci_channel"] + channels = json.loads(res) + for channel in Channels: + if channel in channels: + self.channels[channel] = channels[channel] + + for channel in Channels: + if channel not in self.channels: + if Channels.DEFAULT in self.channels: + print( + f"ERROR: missing config for channel [{channel}] - will use default channel instead" + ) + self.channels[channel] = self.channels[Channels.DEFAULT] + else: + print( + f"ERROR: missing config for channel [{channel}] - will disable notification" + ) + self.channels[channel] = "" self.job_name = os.getenv("CHECK_NAME", "unknown") pr_info = PRInfo() self.pr_number = pr_info.number @@ -63,22 +87,33 @@ class CIBuddy: return json_string - def post(self, message, dry_run=None): - if dry_run is None: - dry_run = self.dry_run - print(f"Posting slack message, dry_run [{dry_run}]") - if dry_run: - url = self.test_channel + def post(self, message: str, channels: List[str]) -> None: + print(f"Posting slack message, dry_run [{self.dry_run}]") + if self.dry_run: + urls = [self.channels[Channels.DRY_RUN]] else: - url = self.dev_ci_channel + urls = [] + for channel in channels: + url = self.channels[channel] + if url: + urls.append(url) + else: + print(f"WARNING: no channel config for [{channel}] - skip") data = {"text": message} try: - requests.post(url, headers=self._HEADERS, data=json.dumps(data), timeout=10) + for url in urls: + requests.post( + url, headers=self._HEADERS, data=json.dumps(data), timeout=10 + ) except Exception as e: print(f"ERROR: Failed to post message, ex {e}") def _post_formatted( - self, title: str, body: Union[Dict, str], with_wf_link: bool + self, + title: str, + body: Union[Dict, str], + with_wf_link: bool, + channels: Union[List[str], str], ) -> None: message = title if isinstance(body, dict): @@ -96,31 +131,49 @@ class CIBuddy: run_id = os.getenv("GITHUB_RUN_ID", "") if with_wf_link and run_id: message += f" *workflow*: \n" - self.post(message) + self.post( + message, channels=[channels] if isinstance(channels, str) else channels + ) def post_info( - self, title: str, body: Union[Dict, str], with_wf_link: bool = True + self, + title: str, + body: Union[Dict, str], + with_wf_link: bool = True, + channels: Union[List[str], str] = Channels.INFO, ) -> None: title_extended = f":white_circle: *{title}*\n\n" - self._post_formatted(title_extended, body, with_wf_link) + self._post_formatted(title_extended, body, with_wf_link, channels=channels) def post_done( - self, title: str, body: Union[Dict, str], with_wf_link: bool = True + self, + title: str, + body: Union[Dict, str], + with_wf_link: bool = True, + channels: Union[List[str], str] = Channels.INFO, ) -> None: title_extended = f":white_check_mark: *{title}*\n\n" - self._post_formatted(title_extended, body, with_wf_link) + self._post_formatted(title_extended, body, with_wf_link, channels=channels) def post_warning( - self, title: str, body: Union[Dict, str], with_wf_link: bool = True + self, + title: str, + body: Union[Dict, str], + with_wf_link: bool = True, + channels: Union[List[str], str] = Channels.ALERTS, ) -> None: title_extended = f":warning: *{title}*\n\n" - self._post_formatted(title_extended, body, with_wf_link) + self._post_formatted(title_extended, body, with_wf_link, channels=channels) def post_critical( - self, title: str, body: Union[Dict, str], with_wf_link: bool = True + self, + title: str, + body: Union[Dict, str], + with_wf_link: bool = True, + channels: Union[List[str], str] = Channels.ALERTS, ) -> None: title_extended = f":black_circle: *{title}*\n\n" - self._post_formatted(title_extended, body, with_wf_link) + self._post_formatted(title_extended, body, with_wf_link, channels=channels) def post_job_error( self, @@ -129,6 +182,7 @@ class CIBuddy: with_instance_info: bool = True, with_wf_link: bool = True, critical: bool = False, + channel: Union[List[str], str] = Channels.ALERTS, ) -> None: instance_id, instance_type = "unknown", "unknown" if with_instance_info: @@ -159,7 +213,7 @@ class CIBuddy: run_id = os.getenv("GITHUB_RUN_ID", "") if with_wf_link and run_id: message += f" *workflow*: \n" - self.post(message) + self.post(message, channels=[channel] if isinstance(channel, str) else channel) def parse_args(): diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py index b4e08f29dbe..27eba273ce0 100755 --- a/tests/ci/create_release.py +++ b/tests/ci/create_release.py @@ -821,7 +821,9 @@ if __name__ == "__main__": else: title = "Failed: " + title CIBuddy(dry_run=args.dry_run).post_critical( - title, dataclasses.asdict(release_info) + title, + dataclasses.asdict(release_info), + channels=[CIBuddy.Channels.ALERTS, CIBuddy.Channels.INFO], ) if args.set_progress_started: From e1e298f14e34a519d59844417e31b7a532b0a157 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 6 Aug 2024 14:56:54 +0000 Subject: [PATCH 102/129] Automatic style fix --- tests/ci/changelog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/changelog.py b/tests/ci/changelog.py index fe47fe53a9e..39e426945d3 100755 --- a/tests/ci/changelog.py +++ b/tests/ci/changelog.py @@ -38,7 +38,7 @@ categories_preferred_order = ( "Experimental Feature", "Performance Improvement", "Improvement", - #"Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)", + # "Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)", "Bug Fix (user-visible misbehavior in an official stable release)", "Build/Testing/Packaging Improvement", "Other", @@ -295,7 +295,7 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri category, ) # Map "Critical Bug Fix" to "Bug fix" category for changelog - #and "Critical Bug Fix" not in category + # and "Critical Bug Fix" not in category ): category = "Bug Fix (user-visible misbehavior in an official stable release)" From bdfaffa9d74b0405ea619330215e6c71f0b6976a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 6 Aug 2024 16:59:23 +0200 Subject: [PATCH 103/129] tests: make test_distributed_inter_server_secret idempotent Signed-off-by: Azat Khuzhin --- .../test.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index 36d7e044f1c..d74cf97e5c6 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -46,6 +46,10 @@ users = pytest.mark.parametrize( ) +def generate_query_id(): + return str(uuid.uuid4()) + + def bootstrap(): for n in list(cluster.instances.values()): n.query("DROP TABLE IF EXISTS data") @@ -268,13 +272,13 @@ def test_secure_insert_buffer_async(): n1.query("SYSTEM RELOAD CONFIG") # ensure that SELECT creates new connection (we need separate table for # this, so that separate distributed pool will be used) - query_id = uuid.uuid4().hex + query_id = generate_query_id() n1.query("SELECT * FROM dist_secure_from_buffer", user="ro", query_id=query_id) assert n1.contains_in_log( "{" + query_id + "} Connection (n2:9000): Connecting." ) - query_id = uuid.uuid4().hex + query_id = generate_query_id() n1.query( "INSERT INTO dist_secure_buffer SELECT * FROM numbers(2)", query_id=query_id ) @@ -331,7 +335,7 @@ def test_secure_disagree_insert(): @users def test_user_insecure_cluster(user, password): - id_ = "query-dist_insecure-" + user + id_ = "query-dist_insecure-" + user + "-" + generate_query_id() n1.query(f"SELECT *, '{id_}' FROM dist_insecure", user=user, password=password) assert get_query_user_info(n1, id_)[0] == [ user, @@ -342,7 +346,7 @@ def test_user_insecure_cluster(user, password): @users def test_user_secure_cluster(user, password): - id_ = "query-dist_secure-" + user + id_ = "query-dist_secure-" + user + "-" + generate_query_id() n1.query(f"SELECT *, '{id_}' FROM dist_secure", user=user, password=password) assert get_query_user_info(n1, id_)[0] == [user, user] assert get_query_user_info(n2, id_)[0] == [user, user] @@ -350,7 +354,7 @@ def test_user_secure_cluster(user, password): @users def test_per_user_inline_settings_insecure_cluster(user, password): - id_ = "query-ddl-settings-dist_insecure-" + user + id_ = "query-ddl-settings-dist_insecure-" + user + "-" + generate_query_id() n1.query( f""" SELECT *, '{id_}' FROM dist_insecure @@ -367,7 +371,7 @@ def test_per_user_inline_settings_insecure_cluster(user, password): @users def test_per_user_inline_settings_secure_cluster(user, password): - id_ = "query-ddl-settings-dist_secure-" + user + id_ = "query-ddl-settings-dist_secure-" + user + "-" + generate_query_id() n1.query( f""" SELECT *, '{id_}' FROM dist_secure @@ -386,7 +390,7 @@ def test_per_user_inline_settings_secure_cluster(user, password): @users def test_per_user_protocol_settings_insecure_cluster(user, password): - id_ = "query-protocol-settings-dist_insecure-" + user + id_ = "query-protocol-settings-dist_insecure-" + user + "-" + generate_query_id() n1.query( f"SELECT *, '{id_}' FROM dist_insecure", user=user, @@ -402,7 +406,7 @@ def test_per_user_protocol_settings_insecure_cluster(user, password): @users def test_per_user_protocol_settings_secure_cluster(user, password): - id_ = "query-protocol-settings-dist_secure-" + user + id_ = "query-protocol-settings-dist_secure-" + user + "-" + generate_query_id() n1.query( f"SELECT *, '{id_}' FROM dist_secure", user=user, @@ -436,7 +440,7 @@ def test_secure_cluster_distributed_over_distributed_different_users_remote(): def test_secure_cluster_distributed_over_distributed_different_users_cluster(): - id_ = "cluster-user" + id_ = "cluster-user" + "-" + generate_query_id() n1.query( f"SELECT *, '{id_}' FROM cluster(secure, currentDatabase(), dist_secure)", user="nopass", From 47953da08dbc791d521b4e722c08af08b9072b89 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 6 Aug 2024 15:08:40 +0000 Subject: [PATCH 104/129] fix reading of size column from missed Nested in compact parts --- src/Storages/MergeTree/MergeTreeReaderCompact.cpp | 2 +- .../MergeTree/MergeTreeReaderCompactSingleBuffer.cpp | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index d49ad61feca..69dc2e4b2bb 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -189,7 +189,7 @@ void MergeTreeReaderCompact::readData( serialization->deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name], nullptr); } - cache.emplace(name, column); + cache[name] = column; size_t read_rows_in_column = column->size() - column_size_before_reading; if (read_rows_in_column != rows_to_read) diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp index 004ba4db028..649bcce1188 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp @@ -22,21 +22,20 @@ try checkNumberOfColumns(num_columns); createColumnsForReading(res_columns); - /// Use cache to avoid reading the column with the same name twice. - /// It may happen if there are empty array Nested in the part. - std::unordered_map caches; - while (read_rows < max_rows_to_read) { size_t rows_to_read = data_part_info_for_read->getIndexGranularity().getMarkRows(from_mark); + /// Use cache to avoid reading the column with the same name twice. + /// It may happen if there are empty array Nested in the part. + ISerialization::SubstreamsCache cache; + for (size_t pos = 0; pos < num_columns; ++pos) { if (!res_columns[pos]) continue; auto & column = res_columns[pos]; - auto & cache = caches[columns_to_read[pos].name]; stream->adjustRightMark(current_task_last_mark); /// Must go before seek. stream->seekToMarkAndColumn(from_mark, *column_positions[pos]); From 0bb8d1a8ede4906a1cdc75af2dbf1e190ff355fc Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 6 Aug 2024 17:10:34 +0200 Subject: [PATCH 105/129] optimize formatting by 3% --- .../functions/type-conversion-functions.md | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 77dd1628fe4..a2b6e496319 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -3063,7 +3063,7 @@ Supported arguments: - Values of type (U)Int8/16/32/64/128/256. - String representations of (U)Int8/16/32/128/256. - Values of type Float32/64, including `NaN` and `Inf`. -- String representations of Float32/64, including `NaN` and `Inf`. +- String representations of Float32/64, including `NaN` and `Inf` (case-insensitive). Unsupported arguments: - String representations of binary and hexadecimal values, e.g. `SELECT toFloat32('0xc0fe');`. @@ -3081,7 +3081,7 @@ SELECT toFloat32(42.7), toFloat32('42.7'), toFloat32('NaN') -FORMAT vertical; +FORMAT Vertical; ``` Result: @@ -3128,11 +3128,11 @@ Unsupported arguments (return `0`): Query: -``` sql +```sql SELECT toFloat32OrZero('42.7'), toFloat32OrZero('abc') -FORMAT vertical; +FORMAT Vertical; ``` Result: @@ -3178,11 +3178,11 @@ Unsupported arguments (return `\N`): Query: -``` sql +```sql SELECT toFloat32OrNull('42.7'), toFloat32OrNull('abc') -FORMAT vertical; +FORMAT Vertical; ``` Result: @@ -3219,8 +3219,8 @@ toFloat32OrDefault(expr[, default]) Supported arguments: - Values of type (U)Int8/16/32/64/128/256. - String representations of (U)Int8/16/32/128/256. -- Values of type Float32/64. -- String representations of Float32/64. +- Values of type Float32/64, including `NaN` and `Inf`. +- String representations of Float32/64, including `NaN` and `Inf` (case-insensitive). Arguments for which the default value is returned: - String representations of binary and hexadecimal values, e.g. `SELECT toFloat32OrDefault('0xc0fe', CAST('0', 'Float32'));`. @@ -3233,11 +3233,11 @@ Arguments for which the default value is returned: Query: -``` sql +```sql SELECT toFloat32OrDefault('8', CAST('0', 'Float32')), toFloat32OrDefault('abc', CAST('0', 'Float32')) -FORMAT vertical; +FORMAT Vertical; ``` Result: @@ -3273,7 +3273,7 @@ Supported arguments: - Values of type (U)Int8/16/32/64/128/256. - String representations of (U)Int8/16/32/128/256. - Values of type Float32/64, including `NaN` and `Inf`. -- String representations of type Float32/64, including `NaN` and `Inf`. +- String representations of type Float32/64, including `NaN` and `Inf` (case-insensitive). Unsupported arguments: - String representations of binary and hexadecimal values, e.g. `SELECT toFloat64('0xc0fe');`. @@ -3291,7 +3291,7 @@ SELECT toFloat64(42.7), toFloat64('42.7'), toFloat64('NaN') -FORMAT vertical; +FORMAT Vertical; ``` Result: @@ -3338,11 +3338,11 @@ Unsupported arguments (return `0`): Query: -``` sql +```sql SELECT toFloat64OrZero('42.7'), toFloat64OrZero('abc') -FORMAT vertical; +FORMAT Vertical; ``` Result: @@ -3388,11 +3388,11 @@ Unsupported arguments (return `\N`): Query: -``` sql +```sql SELECT toFloat64OrNull('42.7'), toFloat64OrNull('abc') -FORMAT vertical; +FORMAT Vertical; ``` Result: @@ -3429,8 +3429,8 @@ toFloat64OrDefault(expr[, default]) Supported arguments: - Values of type (U)Int8/16/32/64/128/256. - String representations of (U)Int8/16/32/128/256. -- Values of type Float32/64. -- String representations of Float32/64. +- Values of type Float32/64, including `NaN` and `Inf`. +- String representations of Float32/64, including `NaN` and `Inf` (case-insensitive). Arguments for which the default value is returned: - String representations of binary and hexadecimal values, e.g. `SELECT toFloat64OrDefault('0xc0fe', CAST('0', 'Float64'));`. @@ -3443,11 +3443,11 @@ Arguments for which the default value is returned: Query: -``` sql +```sql SELECT toFloat64OrDefault('8', CAST('0', 'Float64')), toFloat64OrDefault('abc', CAST('0', 'Float64')) -FORMAT vertical; +FORMAT Vertical; ``` Result: From 40e763dd8b79b20a56b9a5cbff9571d7d9ed8869 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Tue, 6 Aug 2024 17:17:02 +0200 Subject: [PATCH 106/129] more logs to debug logical error from async inserts --- src/Storages/MergeTree/MergeTreeDataWriter.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index ee3ac4207cc..cb02f1cf5f2 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -240,6 +240,14 @@ std::vector scatterAsyncInsertInfoBySelector(AsyncInsertInfo ++offset_idx; } } + if (offset_idx != async_insert_info->offsets.size()) + { + LOG_ERROR( + getLogger("MergeTreeDataWriter"), + "ChunkInfo of async insert offsets doesn't match the selector size {}. Offsets content is ({})", + selector.size(), fmt::join(async_insert_info->offsets.begin(), async_insert_info->offsets.end(), ",")); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected error for async deduplicated insert, please check error logs"); + } return result; } @@ -317,8 +325,10 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts( { if (async_insert_info_with_partition[i] == nullptr) { - LOG_ERROR(getLogger("MergeTreeDataWriter"), "The {}th element in async_insert_info_with_partition is nullptr. There are totally {} partitions in the insert. Selector content is {}", - i, partitions_count, fmt::join(selector.begin(), selector.end(), ",")); + LOG_ERROR( + getLogger("MergeTreeDataWriter"), + "The {}th element in async_insert_info_with_partition is nullptr. There are totally {} partitions in the insert. Selector content is ({}). Offsets content is ({})", + i, partitions_count, fmt::join(selector.begin(), selector.end(), ","), fmt::join(async_insert_info->offsets.begin(), async_insert_info->offsets.end(), ",")); throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected error for async deduplicated insert, please check error logs"); } result[i].offsets = std::move(async_insert_info_with_partition[i]->offsets); From 2d93910bd601f7a4f8cc9385a2d191ce806453e3 Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Tue, 6 Aug 2024 17:44:29 +0200 Subject: [PATCH 107/129] added tags no-fasttest --- tests/queries/0_stateless/03215_parquet_index.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/03215_parquet_index.sql b/tests/queries/0_stateless/03215_parquet_index.sql index 0afccdf6f3b..5766f40f021 100644 --- a/tests/queries/0_stateless/03215_parquet_index.sql +++ b/tests/queries/0_stateless/03215_parquet_index.sql @@ -1,3 +1,5 @@ +-- Tags: no-fasttest + -- default settings. DROP TABLE IF EXISTS test_parquet; CREATE TABLE test_parquet (col1 int, col2 String) ENGINE=File(Parquet); From 1e631472f0aabcb1b856942fb7409c5c010315a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 6 Aug 2024 18:07:49 +0200 Subject: [PATCH 108/129] Make 01685_ssd_cache_dictionary_complex_key parallelizable --- .../01685_ssd_cache_dictionary_complex_key.sh | 81 +++++++++---------- 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/tests/queries/0_stateless/01685_ssd_cache_dictionary_complex_key.sh b/tests/queries/0_stateless/01685_ssd_cache_dictionary_complex_key.sh index 5583a9dd5e7..55061b9a643 100755 --- a/tests/queries/0_stateless/01685_ssd_cache_dictionary_complex_key.sh +++ b/tests/queries/0_stateless/01685_ssd_cache_dictionary_complex_key.sh @@ -1,15 +1,11 @@ #!/usr/bin/env bash -# Tags: no-parallel CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -n --query=" - DROP DATABASE IF EXISTS 01685_database_for_cache_dictionary; - CREATE DATABASE 01685_database_for_cache_dictionary; - - CREATE TABLE 01685_database_for_cache_dictionary.complex_key_simple_attributes_source_table + CREATE TABLE complex_key_simple_attributes_source_table ( id UInt64, id_key String, @@ -18,11 +14,11 @@ $CLICKHOUSE_CLIENT -n --query=" ) ENGINE = TinyLog; - INSERT INTO 01685_database_for_cache_dictionary.complex_key_simple_attributes_source_table VALUES(0, 'id_key_0', 'value_0', 'value_second_0'); - INSERT INTO 01685_database_for_cache_dictionary.complex_key_simple_attributes_source_table VALUES(1, 'id_key_1', 'value_1', 'value_second_1'); - INSERT INTO 01685_database_for_cache_dictionary.complex_key_simple_attributes_source_table VALUES(2, 'id_key_2', 'value_2', 'value_second_2'); + INSERT INTO complex_key_simple_attributes_source_table VALUES(0, 'id_key_0', 'value_0', 'value_second_0'); + INSERT INTO complex_key_simple_attributes_source_table VALUES(1, 'id_key_1', 'value_1', 'value_second_1'); + INSERT INTO complex_key_simple_attributes_source_table VALUES(2, 'id_key_2', 'value_2', 'value_second_2'); - CREATE DICTIONARY 01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes + CREATE DICTIONARY cache_dictionary_complex_key_simple_attributes ( id UInt64, id_key String, @@ -30,32 +26,32 @@ $CLICKHOUSE_CLIENT -n --query=" value_second String DEFAULT 'value_second_default' ) PRIMARY KEY id, id_key - SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'complex_key_simple_attributes_source_table' DB '01685_database_for_cache_dictionary')) + SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'complex_key_simple_attributes_source_table' DB '${CLICKHOUSE_DATABASE}')) LIFETIME(MIN 1 MAX 1000) - LAYOUT(COMPLEX_KEY_SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 8192 PATH '$USER_FILES_PATH/0d')); + LAYOUT(COMPLEX_KEY_SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 8192 PATH '$USER_FILES_PATH/${CLICKHOUSE_DATABASE}_dic')); SELECT 'Dictionary cache_dictionary_complex_key_simple_attributes'; SELECT 'dictGet existing value'; - SELECT dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first, - dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 3; + SELECT dictGet('cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first, + dictGet('cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 3; SELECT 'dictGet with non existing value'; - SELECT dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first, - dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 4; + SELECT dictGet('cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first, + dictGet('cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 4; SELECT 'dictGetOrDefault existing value'; - SELECT dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first, - dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 3; + SELECT dictGetOrDefault('cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first, + dictGetOrDefault('cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 3; SELECT 'dictGetOrDefault non existing value'; - SELECT dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first, - dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 4; + SELECT dictGetOrDefault('cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first, + dictGetOrDefault('cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 4; SELECT 'dictHas'; - SELECT dictHas('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', (number, concat('id_key_', toString(number)))) FROM system.numbers LIMIT 4; + SELECT dictHas('cache_dictionary_complex_key_simple_attributes', (number, concat('id_key_', toString(number)))) FROM system.numbers LIMIT 4; SELECT 'select all values as input stream'; - SELECT * FROM 01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes ORDER BY id; + SELECT * FROM cache_dictionary_complex_key_simple_attributes ORDER BY id; - DROP DICTIONARY 01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes; - DROP TABLE 01685_database_for_cache_dictionary.complex_key_simple_attributes_source_table; + DROP DICTIONARY cache_dictionary_complex_key_simple_attributes; + DROP TABLE complex_key_simple_attributes_source_table; - CREATE TABLE 01685_database_for_cache_dictionary.complex_key_complex_attributes_source_table + CREATE TABLE complex_key_complex_attributes_source_table ( id UInt64, id_key String, @@ -64,11 +60,11 @@ $CLICKHOUSE_CLIENT -n --query=" ) ENGINE = TinyLog; - INSERT INTO 01685_database_for_cache_dictionary.complex_key_complex_attributes_source_table VALUES(0, 'id_key_0', 'value_0', 'value_second_0'); - INSERT INTO 01685_database_for_cache_dictionary.complex_key_complex_attributes_source_table VALUES(1, 'id_key_1', 'value_1', NULL); - INSERT INTO 01685_database_for_cache_dictionary.complex_key_complex_attributes_source_table VALUES(2, 'id_key_2', 'value_2', 'value_second_2'); + INSERT INTO complex_key_complex_attributes_source_table VALUES(0, 'id_key_0', 'value_0', 'value_second_0'); + INSERT INTO complex_key_complex_attributes_source_table VALUES(1, 'id_key_1', 'value_1', NULL); + INSERT INTO complex_key_complex_attributes_source_table VALUES(2, 'id_key_2', 'value_2', 'value_second_2'); - CREATE DICTIONARY 01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes + CREATE DICTIONARY cache_dictionary_complex_key_complex_attributes ( id UInt64, id_key String, @@ -77,29 +73,28 @@ $CLICKHOUSE_CLIENT -n --query=" value_second Nullable(String) DEFAULT 'value_second_default' ) PRIMARY KEY id, id_key - SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'complex_key_complex_attributes_source_table' DB '01685_database_for_cache_dictionary')) + SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'complex_key_complex_attributes_source_table' DB '${CLICKHOUSE_DATABASE}')) LIFETIME(MIN 1 MAX 1000) LAYOUT(COMPLEX_KEY_SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 8192 PATH '$USER_FILES_PATH/1d')); SELECT 'Dictionary cache_dictionary_complex_key_complex_attributes'; SELECT 'dictGet existing value'; - SELECT dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first, - dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 3; + SELECT dictGet('cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first, + dictGet('cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 3; SELECT 'dictGet with non existing value'; - SELECT dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first, - dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 4; + SELECT dictGet('cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first, + dictGet('cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 4; SELECT 'dictGetOrDefault existing value'; - SELECT dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first, - dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 3; + SELECT dictGetOrDefault('cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first, + dictGetOrDefault('cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 3; SELECT 'dictGetOrDefault non existing value'; - SELECT dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first, - dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 4; + SELECT dictGetOrDefault('cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first, + dictGetOrDefault('cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 4; SELECT 'dictHas'; - SELECT dictHas('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', (number, concat('id_key_', toString(number)))) FROM system.numbers LIMIT 4; + SELECT dictHas('cache_dictionary_complex_key_complex_attributes', (number, concat('id_key_', toString(number)))) FROM system.numbers LIMIT 4; SELECT 'select all values as input stream'; - SELECT * FROM 01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes ORDER BY id; + SELECT * FROM cache_dictionary_complex_key_complex_attributes ORDER BY id; - DROP DICTIONARY 01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes; - DROP TABLE 01685_database_for_cache_dictionary.complex_key_complex_attributes_source_table; - - DROP DATABASE 01685_database_for_cache_dictionary;" + DROP DICTIONARY cache_dictionary_complex_key_complex_attributes; + DROP TABLE complex_key_complex_attributes_source_table; +" From 38d891c6befdacfa835005b522f104e9428a270b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 6 Aug 2024 18:18:09 +0200 Subject: [PATCH 109/129] Make 02015_column_default_dict_get_identifier parallelizable --- ...015_column_default_dict_get_identifier.sql | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/tests/queries/0_stateless/02015_column_default_dict_get_identifier.sql b/tests/queries/0_stateless/02015_column_default_dict_get_identifier.sql index 24a3b631388..046d0c42180 100644 --- a/tests/queries/0_stateless/02015_column_default_dict_get_identifier.sql +++ b/tests/queries/0_stateless/02015_column_default_dict_get_identifier.sql @@ -1,9 +1,6 @@ -- Tags: no-parallel -DROP DATABASE IF EXISTS 02015_db; -CREATE DATABASE 02015_db; - -CREATE TABLE 02015_db.test_table +CREATE TABLE test_table ( key_column UInt64, data_column_1 UInt64, @@ -12,9 +9,9 @@ CREATE TABLE 02015_db.test_table ENGINE = MergeTree ORDER BY key_column; -INSERT INTO 02015_db.test_table VALUES (0, 0, 0); +INSERT INTO test_table VALUES (0, 0, 0); -CREATE DICTIONARY 02015_db.test_dictionary +CREATE DICTIONARY test_dictionary ( key_column UInt64 DEFAULT 0, data_column_1 UInt64 DEFAULT 1, @@ -22,20 +19,18 @@ CREATE DICTIONARY 02015_db.test_dictionary ) PRIMARY KEY key_column LAYOUT(DIRECT()) -SOURCE(CLICKHOUSE(DB '02015_db' TABLE 'test_table')); +SOURCE(CLICKHOUSE(DB currentDatabase() TABLE 'test_table')); -CREATE TABLE 02015_db.test_table_default +CREATE TABLE test_table_default ( - data_1 DEFAULT dictGetUInt64('02015_db.test_dictionary', 'data_column_1', toUInt64(0)), - data_2 DEFAULT dictGet(02015_db.test_dictionary, 'data_column_2', toUInt64(0)) + data_1 DEFAULT dictGetUInt64('test_dictionary', 'data_column_1', toUInt64(0)), + data_2 DEFAULT dictGet(test_dictionary, 'data_column_2', toUInt64(0)) ) ENGINE=TinyLog; -INSERT INTO 02015_db.test_table_default(data_1) VALUES (5); -SELECT * FROM 02015_db.test_table_default; +INSERT INTO test_table_default(data_1) VALUES (5); +SELECT * FROM test_table_default; -DROP TABLE 02015_db.test_table_default; -DROP DICTIONARY 02015_db.test_dictionary; -DROP TABLE 02015_db.test_table; - -DROP DATABASE 02015_db; +DROP TABLE test_table_default; +DROP DICTIONARY test_dictionary; +DROP TABLE test_table; From 425fb0f485b1936ebfa46f0ae9db1e7ea134272f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 6 Aug 2024 18:20:09 +0200 Subject: [PATCH 110/129] Make 01600_multiple_left_join_with_aliases parallelizable --- .../01600_multiple_left_join_with_aliases.sql | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/tests/queries/0_stateless/01600_multiple_left_join_with_aliases.sql b/tests/queries/0_stateless/01600_multiple_left_join_with_aliases.sql index ae296e18560..2945622fbed 100644 --- a/tests/queries/0_stateless/01600_multiple_left_join_with_aliases.sql +++ b/tests/queries/0_stateless/01600_multiple_left_join_with_aliases.sql @@ -1,9 +1,4 @@ --- Tags: no-parallel - -drop database if exists test_01600; -create database test_01600; - -CREATE TABLE test_01600.base +CREATE TABLE base ( `id` UInt64, `id2` UInt64, @@ -14,7 +9,7 @@ ENGINE=MergeTree() PARTITION BY d ORDER BY (id,id2,d); -CREATE TABLE test_01600.derived1 +CREATE TABLE derived1 ( `id1` UInt64, `d1` UInt64, @@ -25,7 +20,7 @@ PARTITION BY d1 ORDER BY (id1, d1) ; -CREATE TABLE test_01600.derived2 +CREATE TABLE derived2 ( `id2` UInt64, `d2` UInt64, @@ -36,19 +31,17 @@ PARTITION BY d2 ORDER BY (id2, d2) ; -select +select base.id as `base.id`, derived2.value2 as `derived2.value2`, derived1.value1 as `derived1.value1` -from test_01600.base as base -left join test_01600.derived2 as derived2 on base.id2 = derived2.id2 -left join test_01600.derived1 as derived1 on base.id = derived1.id1; +from base as base +left join derived2 as derived2 on base.id2 = derived2.id2 +left join derived1 as derived1 on base.id = derived1.id1; SELECT base.id AS `base.id`, derived1.value1 AS `derived1.value1` -FROM test_01600.base AS base -LEFT JOIN test_01600.derived1 AS derived1 ON base.id = derived1.id1; - -drop database test_01600; +FROM base AS base +LEFT JOIN derived1 AS derived1 ON base.id = derived1.id1; From 078e4ca36db15a4f3bab27141d24fe00f89a5a1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 6 Aug 2024 18:21:58 +0200 Subject: [PATCH 111/129] Make 02950_dictionary_ssd_cache_short_circuit parallelizable --- ...2950_dictionary_ssd_cache_short_circuit.sh | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/tests/queries/0_stateless/02950_dictionary_ssd_cache_short_circuit.sh b/tests/queries/0_stateless/02950_dictionary_ssd_cache_short_circuit.sh index 3d2fe5d664d..daa9c571a5d 100755 --- a/tests/queries/0_stateless/02950_dictionary_ssd_cache_short_circuit.sh +++ b/tests/queries/0_stateless/02950_dictionary_ssd_cache_short_circuit.sh @@ -1,15 +1,11 @@ #!/usr/bin/env bash -# Tags: no-parallel CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -n --query=" - DROP DATABASE IF EXISTS 02950_database_for_ssd_cache_dictionary; - CREATE DATABASE 02950_database_for_ssd_cache_dictionary; - - CREATE TABLE 02950_database_for_ssd_cache_dictionary.source_table + CREATE TABLE source_table ( id UInt64, v1 String, @@ -18,9 +14,9 @@ $CLICKHOUSE_CLIENT -n --query=" ) ENGINE = TinyLog; - INSERT INTO 02950_database_for_ssd_cache_dictionary.source_table VALUES (0, 'zero', 'zero', 0), (1, 'one', NULL, 1); + INSERT INTO source_table VALUES (0, 'zero', 'zero', 0), (1, 'one', NULL, 1); - CREATE DICTIONARY 02950_database_for_ssd_cache_dictionary.ssd_cache_dictionary + CREATE DICTIONARY ssd_cache_dictionary ( id UInt64, v1 String, @@ -30,12 +26,11 @@ $CLICKHOUSE_CLIENT -n --query=" PRIMARY KEY id SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'source_table')) LIFETIME(MIN 1 MAX 1000) - LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 8192 PATH '$CLICKHOUSE_USER_FILES/0d')); + LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 8192 PATH '$CLICKHOUSE_USER_FILES/${CLICKHOUSE_DATABASE}_ssd_dic')); - SELECT dictGetOrDefault('02950_database_for_ssd_cache_dictionary.ssd_cache_dictionary', ('v1', 'v2'), 0, (intDiv(1, id), intDiv(1, id))) FROM 02950_database_for_ssd_cache_dictionary.source_table; - SELECT dictGetOrDefault('02950_database_for_ssd_cache_dictionary.ssd_cache_dictionary', 'v2', id+1, intDiv(NULL, id)) FROM 02950_database_for_ssd_cache_dictionary.source_table; - SELECT dictGetOrDefault('02950_database_for_ssd_cache_dictionary.ssd_cache_dictionary', 'v3', id+1, intDiv(NULL, id)) FROM 02950_database_for_ssd_cache_dictionary.source_table; + SELECT dictGetOrDefault('ssd_cache_dictionary', ('v1', 'v2'), 0, (intDiv(1, id), intDiv(1, id))) FROM source_table; + SELECT dictGetOrDefault('ssd_cache_dictionary', 'v2', id+1, intDiv(NULL, id)) FROM source_table; + SELECT dictGetOrDefault('ssd_cache_dictionary', 'v3', id+1, intDiv(NULL, id)) FROM source_table; - DROP DICTIONARY 02950_database_for_ssd_cache_dictionary.ssd_cache_dictionary; - DROP TABLE 02950_database_for_ssd_cache_dictionary.source_table; - DROP DATABASE 02950_database_for_ssd_cache_dictionary;" + DROP DICTIONARY ssd_cache_dictionary; + DROP TABLE source_table;" From c58643897a95d1e58bf25d65dfaad52aa0eddefc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 6 Aug 2024 18:26:31 +0200 Subject: [PATCH 112/129] Make 02002_row_level_filter_bug parallelizable --- .../0_stateless/02002_row_level_filter_bug.sh | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/tests/queries/0_stateless/02002_row_level_filter_bug.sh b/tests/queries/0_stateless/02002_row_level_filter_bug.sh index d15a26f48f5..557aa738217 100755 --- a/tests/queries/0_stateless/02002_row_level_filter_bug.sh +++ b/tests/queries/0_stateless/02002_row_level_filter_bug.sh @@ -1,11 +1,12 @@ #!/usr/bin/env bash -# Tags: no-parallel -# Tag no-parallel: create user CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +TEST_ROLE="${CLICKHOUSE_DATABASE}_role" +TEST_USER="${CLICKHOUSE_DATABASE}_user" +TEST_POLICY="${CLICKHOUSE_DATABASE}_policy" $CLICKHOUSE_CLIENT --query "drop table if exists test_table" $CLICKHOUSE_CLIENT --query "CREATE TABLE test_table @@ -37,26 +38,26 @@ arrayJoin(['AWD','ZZZ']) as team, arrayJoin([3183,3106,0,3130,3108,3126,3109,3107,3182,3180,3129,3128,3125,3266]) as a FROM numbers(600);" -$CLICKHOUSE_CLIENT --query "DROP ROLE IF exists AWD;" -$CLICKHOUSE_CLIENT --query "create role AWD;" -$CLICKHOUSE_CLIENT --query "REVOKE ALL ON *.* FROM AWD;" +$CLICKHOUSE_CLIENT --query "DROP ROLE IF EXISTS ${TEST_ROLE};" +$CLICKHOUSE_CLIENT --query "create role ${TEST_ROLE};" +$CLICKHOUSE_CLIENT --query "REVOKE ALL ON *.* FROM ${TEST_ROLE};" -$CLICKHOUSE_CLIENT --query "DROP USER IF EXISTS AWD_user;" -$CLICKHOUSE_CLIENT --query "CREATE USER AWD_user IDENTIFIED WITH plaintext_password BY 'AWD_pwd' DEFAULT ROLE AWD;" +$CLICKHOUSE_CLIENT --query "DROP USER IF EXISTS ${TEST_USER};" +$CLICKHOUSE_CLIENT --query "CREATE USER ${TEST_USER} IDENTIFIED WITH plaintext_password BY 'AWD_pwd' DEFAULT ROLE ${TEST_ROLE};" -$CLICKHOUSE_CLIENT --query "GRANT SELECT ON test_table TO AWD;" +$CLICKHOUSE_CLIENT --query "GRANT SELECT ON test_table TO ${TEST_ROLE};" -$CLICKHOUSE_CLIENT --query "DROP ROW POLICY IF EXISTS ttt_bu_test_table_AWD ON test_table;" -$CLICKHOUSE_CLIENT --query "CREATE ROW POLICY ttt_bu_test_table_AWD ON test_table FOR SELECT USING team = 'AWD' TO AWD;" +$CLICKHOUSE_CLIENT --query "DROP ROW POLICY IF EXISTS ${TEST_POLICY} ON test_table;" +$CLICKHOUSE_CLIENT --query "CREATE ROW POLICY ${TEST_POLICY} ON test_table FOR SELECT USING team = 'AWD' TO ${TEST_ROLE};" -$CLICKHOUSE_CLIENT --user=AWD_user --password=AWD_pwd --query " +$CLICKHOUSE_CLIENT --user=${TEST_USER} --password=AWD_pwd --query " SELECT count() AS count FROM test_table WHERE t_date = '2021-07-15' AND c = 'aur' AND a=3130; " -$CLICKHOUSE_CLIENT --user=AWD_user --password=AWD_pwd --query " +$CLICKHOUSE_CLIENT --user=${TEST_USER} --password=AWD_pwd --query " SELECT team, a, @@ -70,8 +71,12 @@ GROUP BY t_date; " -$CLICKHOUSE_CLIENT --user=AWD_user --password=AWD_pwd --query " +$CLICKHOUSE_CLIENT --user=${TEST_USER} --password=AWD_pwd --query " SELECT count() AS count FROM test_table WHERE (t_date = '2021-07-15') AND (c = 'aur') AND (a = 313) " + +$CLICKHOUSE_CLIENT --query "DROP ROLE IF EXISTS ${TEST_ROLE};" +$CLICKHOUSE_CLIENT --query "DROP USER IF EXISTS ${TEST_USER};" +$CLICKHOUSE_CLIENT --query "DROP ROW POLICY IF EXISTS ${TEST_POLICY} ON test_table;" From 57b6d461418d4ad8e2d5cb579d5671abc077117e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 6 Aug 2024 18:28:01 +0200 Subject: [PATCH 113/129] Make 01764_table_function_dictionary parallelizable --- tests/queries/0_stateless/01764_table_function_dictionary.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01764_table_function_dictionary.sql b/tests/queries/0_stateless/01764_table_function_dictionary.sql index 76e7213b367..e37f8d2a290 100644 --- a/tests/queries/0_stateless/01764_table_function_dictionary.sql +++ b/tests/queries/0_stateless/01764_table_function_dictionary.sql @@ -1,5 +1,3 @@ --- Tags: no-parallel - DROP TABLE IF EXISTS table_function_dictionary_source_table; CREATE TABLE table_function_dictionary_source_table ( @@ -18,7 +16,7 @@ CREATE DICTIONARY table_function_dictionary_test_dictionary value UInt64 DEFAULT 0 ) PRIMARY KEY id -SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_function_dictionary_source_table')) +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' DATABASE currentDatabase() TABLE 'table_function_dictionary_source_table')) LAYOUT(DIRECT()); SELECT * FROM dictionary('table_function_dictionary_test_dictionary'); From f0fdba3bbeb8687e38f89d8379e70cfd39db1252 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Tue, 6 Aug 2024 19:02:49 +0200 Subject: [PATCH 114/129] CI: Strict job timeout 1.5h for tests, 2h for builds --- tests/ci/ci_config.py | 1 - tests/ci/ci_definitions.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 7a19eb6f827..f578cd8b559 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -510,7 +510,6 @@ class CI: JobNames.LIBFUZZER_TEST: JobConfig( required_builds=[BuildNames.FUZZERS], run_by_label=Tags.libFuzzer, - timeout=10800, run_command='libfuzzer_test_check.py "$CHECK_NAME"', runner_type=Runners.STYLE_CHECKER, ), diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py index 48847b0d7a6..69e7ed259d5 100644 --- a/tests/ci/ci_definitions.py +++ b/tests/ci/ci_definitions.py @@ -331,7 +331,7 @@ class JobConfig: # will be triggered for the job if omitted in CI workflow yml run_command: str = "" # job timeout, seconds - timeout: Optional[int] = None + timeout: Optional[int] = 5400 # sets number of batches for a multi-batch job num_batches: int = 1 # label that enables job in CI, if set digest isn't used @@ -420,7 +420,6 @@ class CommonJobConfigs: ), run_command='functional_test_check.py "$CHECK_NAME"', runner_type=Runners.FUNC_TESTER, - timeout=9000, ) STATEFUL_TEST = JobConfig( job_name_keyword="stateful", @@ -531,7 +530,6 @@ class CommonJobConfigs: docker=["clickhouse/sqllogic-test"], ), run_command="sqllogic_test.py", - timeout=10800, release_only=True, runner_type=Runners.FUNC_TESTER, ) @@ -543,7 +541,6 @@ class CommonJobConfigs: docker=["clickhouse/sqltest"], ), run_command="sqltest.py", - timeout=10800, release_only=True, runner_type=Runners.FUZZER_UNIT_TESTER, ) @@ -613,6 +610,7 @@ class CommonJobConfigs: docker=["clickhouse/binary-builder"], git_submodules=True, ), + timeout=7200, run_command="build_check.py $BUILD_NAME", runner_type=Runners.BUILDER, ) From 9c92c26edcb8ac83191f8a814196aa05a5600730 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 6 Aug 2024 17:51:26 +0000 Subject: [PATCH 115/129] Simplify code. --- src/Storages/MergeTree/MergeTreeData.cpp | 9 +++--- .../MergeTree/registerStorageMergeTree.cpp | 13 +++++++++ src/Storages/StorageFactory.cpp | 28 ------------------- 3 files changed, 17 insertions(+), 33 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 98e1cddcf4b..dbb3e39c12e 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3216,13 +3216,12 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context /// Block the case of alter table add projection for special merge trees. if (std::any_of(commands.begin(), commands.end(), [](const AlterCommand & c) { return c.type == AlterCommand::ADD_PROJECTION; })) { - const std::unordered_set allowed_storages{"MergeTree", "ReplicatedMergeTree", "SharedMergeTree"}; - if (auto storage_name = getName(); !allowed_storages.contains(storage_name) + if (merging_params.mode != MergingParams::Mode::Ordinary && settings_from_storage->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, - "Projection is fully supported in (Replictaed, Shared)MergeTree, but also allowed in non-throw mode with other" - " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode." - " Current storage name is {}.", storage_name); + "Projection is fully supported in {} with deduplicate_merge_projection_mode = throw. " + "Use 'drop' or 'rebuild' option of deduplicate_merge_projection_mode.", + getName()); } commands.apply(new_metadata, local_context); diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 3f0603f6900..b7887c35590 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -34,6 +34,7 @@ namespace ErrorCodes extern const int UNKNOWN_STORAGE; extern const int NO_REPLICA_NAME_GIVEN; extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; + extern const int SUPPORT_IS_DISABLED; } @@ -829,6 +830,18 @@ static StoragePtr create(const StorageFactory::Arguments & args) "Floating point partition key is not supported: {}", metadata.partition_key.column_names[i]); } + if (metadata.hasProjections() && args.mode == LoadingStrictnessLevel::CREATE) + { + /// Now let's handle the merge tree family. Note we only handle in the mode of CREATE due to backward compatibility. + /// Otherwise, it would fail to start in the case of existing projections with special mergetree. + if (merging_params.mode != MergeTreeData::MergingParams::Mode::Ordinary + && storage_settings->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "Projection is fully supported in {}MergeTree with deduplicate_merge_projection_mode = throw. " + "Use 'drop' or 'rebuild' option of deduplicate_merge_projection_mode.", + merging_params.getModeName()); + } + if (arg_num != arg_cnt) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong number of engine arguments."); diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 557f53a9ada..25cb81fa5fa 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -20,7 +20,6 @@ namespace ErrorCodes extern const int FUNCTION_CANNOT_HAVE_PARAMETERS; extern const int BAD_ARGUMENTS; extern const int DATA_TYPE_CANNOT_BE_USED_IN_TABLES; - extern const int SUPPORT_IS_DISABLED; } @@ -201,33 +200,6 @@ StoragePtr StorageFactory::get( check_feature( "projections", [](StorageFeatures features) { return features.supports_projections; }); - - /// Now let's handle the merge tree family. Note we only handle in the mode of CREATE due to backward compatibility. - /// Otherwise, it would fail to start in the case of existing projections with special mergetree. - chassert(query.storage->engine); - const std::unordered_set allowed_engines{"MergeTree", "ReplicatedMergeTree", "SharedMergeTree"}; - if (auto engine_name(query.storage->engine->name); mode == LoadingStrictnessLevel::CREATE - && !allowed_engines.contains(engine_name)) - { - /// default throw mode in deduplicate_merge_projection_mode - bool projection_allowed = false; - if (auto * setting = query.storage->settings; setting != nullptr) - { - for (const auto & change : setting->changes) - { - if (change.name == "deduplicate_merge_projection_mode" && change.value != Field("throw")) - { - projection_allowed = true; - break; - } - } - } - if (!projection_allowed) - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, - "Projection is fully supported in (Replictaed, Shared)MergeTree, but also allowed in non-throw mode with other" - " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode." - " Current storage name is {}.", engine_name); - } } } } From c74460b47e23ced4562d6f5123d61042c42d2896 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 6 Aug 2024 17:53:23 +0000 Subject: [PATCH 116/129] Cleanup. --- src/Storages/StorageFactory.cpp | 2 -- .../0_stateless/03206_projection_merge_special_mergetree.sql | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 25cb81fa5fa..060b271d8f4 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -196,11 +196,9 @@ StoragePtr StorageFactory::get( [](StorageFeatures features) { return features.supports_skipping_indices; }); if (query.columns_list && query.columns_list->projections && !query.columns_list->projections->children.empty()) - { check_feature( "projections", [](StorageFeatures features) { return features.supports_projections; }); - } } } diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql index d3448138396..82684f754b6 100644 --- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql +++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql @@ -103,4 +103,4 @@ SELECT FROM system.projection_parts WHERE (database = currentDatabase()) AND (`table` = 'tp') AND (active = 1); -DROP TABLE tp; \ No newline at end of file +DROP TABLE tp; From 302bd5fdc6f077acf05a17c5e700cb44b2e609ab Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 6 Aug 2024 18:37:26 +0000 Subject: [PATCH 117/129] Remove usearch submodule --- .gitmodules | 3 --- contrib/usearch | 1 - 2 files changed, 4 deletions(-) delete mode 160000 contrib/usearch diff --git a/.gitmodules b/.gitmodules index 7e0b4df4ad1..c4c93822711 100644 --- a/.gitmodules +++ b/.gitmodules @@ -339,9 +339,6 @@ [submodule "contrib/incbin"] path = contrib/incbin url = https://github.com/graphitemaster/incbin.git -[submodule "contrib/usearch"] - path = contrib/usearch - url = https://github.com/unum-cloud/usearch.git [submodule "contrib/SimSIMD"] path = contrib/SimSIMD url = https://github.com/ashvardanian/SimSIMD.git diff --git a/contrib/usearch b/contrib/usearch deleted file mode 160000 index 955c6f9c11a..00000000000 --- a/contrib/usearch +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 955c6f9c11adfd89c912e0d1643d160b4e9e543f From 7a5b30d955b85a164097b4c7dd5ce957a47059e5 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 6 Aug 2024 18:39:18 +0000 Subject: [PATCH 118/129] Re-add forked usearch repo --- .gitmodules | 3 +++ contrib/usearch | 1 + 2 files changed, 4 insertions(+) create mode 160000 contrib/usearch diff --git a/.gitmodules b/.gitmodules index c4c93822711..7fdfb1103c5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -339,6 +339,9 @@ [submodule "contrib/incbin"] path = contrib/incbin url = https://github.com/graphitemaster/incbin.git +[submodule "contrib/usearch"] + path = contrib/usearch + url = https://github.com/ClickHouse/usearch.git [submodule "contrib/SimSIMD"] path = contrib/SimSIMD url = https://github.com/ashvardanian/SimSIMD.git diff --git a/contrib/usearch b/contrib/usearch new file mode 160000 index 00000000000..955c6f9c11a --- /dev/null +++ b/contrib/usearch @@ -0,0 +1 @@ +Subproject commit 955c6f9c11adfd89c912e0d1643d160b4e9e543f From 9594a9baffb125e21b28cd421b511d26febfc900 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 6 Aug 2024 18:43:44 +0000 Subject: [PATCH 119/129] Fix memory corruption in usearch --- contrib/usearch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/usearch b/contrib/usearch index 955c6f9c11a..30810452bec 160000 --- a/contrib/usearch +++ b/contrib/usearch @@ -1 +1 @@ -Subproject commit 955c6f9c11adfd89c912e0d1643d160b4e9e543f +Subproject commit 30810452bec5d3d3aa0931bb5d761e2f09aa6356 From 36b6adbb30efdc0a96a274bdcbaeec2d961a8bed Mon Sep 17 00:00:00 2001 From: "Zhukova, Maria" Date: Tue, 6 Aug 2024 12:15:23 -0700 Subject: [PATCH 120/129] qpl-cmake: Better wording on ISA-L copy and removed non-required linking --- contrib/qpl-cmake/CMakeLists.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/contrib/qpl-cmake/CMakeLists.txt b/contrib/qpl-cmake/CMakeLists.txt index b2f263252c2..e62612cff5a 100644 --- a/contrib/qpl-cmake/CMakeLists.txt +++ b/contrib/qpl-cmake/CMakeLists.txt @@ -24,7 +24,9 @@ message(STATUS "Intel QPL version: ${QPL_VERSION}") # which are then combined into static or shared qpl. # Output ch_contrib::qpl by linking with 8 library targets. -# Note, qpl submodule comes with its own version of isal that is not compatible with upstream isal (e.g., ch_contrib::isal). +# Note, QPL has integrated a customized version of ISA-L to meet specific needs. +# This version has been significantly modified and there are no plans to maintain compatibility with the upstream version +# or upgrade the current copy. ## cmake/CompileOptions.cmake and automatic wrappers generation @@ -733,10 +735,6 @@ target_compile_definitions(_qpl target_link_libraries(_qpl PRIVATE ch_contrib::accel-config) -# C++ filesystem library requires additional linking for older GNU/Clang -target_link_libraries(_qpl PRIVATE $<$,$,9.1>>:stdc++fs>) -target_link_libraries(_qpl PRIVATE $<$,$,9.0>>:c++fs>) - target_include_directories(_qpl SYSTEM BEFORE PUBLIC "${QPL_PROJECT_DIR}/include" PUBLIC ${UUID_DIR}) From 139f5e55d34092969e49e4fd191404cac0ddd05b Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 6 Aug 2024 20:31:41 +0000 Subject: [PATCH 121/129] Unflake 02099_tsv_raw_format.sh --- ...rence => 02099_tsv_raw_format_1.reference} | 2 -- ...aw_format.sh => 02099_tsv_raw_format_1.sh} | 13 ------------- .../02099_tsv_raw_format_2.reference | 2 ++ .../0_stateless/02099_tsv_raw_format_2.sh | 19 +++++++++++++++++++ 4 files changed, 21 insertions(+), 15 deletions(-) rename tests/queries/0_stateless/{02099_tsv_raw_format.reference => 02099_tsv_raw_format_1.reference} (94%) rename tests/queries/0_stateless/{02099_tsv_raw_format.sh => 02099_tsv_raw_format_1.sh} (69%) create mode 100644 tests/queries/0_stateless/02099_tsv_raw_format_2.reference create mode 100755 tests/queries/0_stateless/02099_tsv_raw_format_2.sh diff --git a/tests/queries/0_stateless/02099_tsv_raw_format.reference b/tests/queries/0_stateless/02099_tsv_raw_format_1.reference similarity index 94% rename from tests/queries/0_stateless/02099_tsv_raw_format.reference rename to tests/queries/0_stateless/02099_tsv_raw_format_1.reference index de46cf8dff7..3ac175e51f6 100644 --- a/tests/queries/0_stateless/02099_tsv_raw_format.reference +++ b/tests/queries/0_stateless/02099_tsv_raw_format_1.reference @@ -109,5 +109,3 @@ UInt64 String Date 2 \N nSome text -b1cad4eb4be08a40387c9de70d02fcc2 - -b1cad4eb4be08a40387c9de70d02fcc2 - diff --git a/tests/queries/0_stateless/02099_tsv_raw_format.sh b/tests/queries/0_stateless/02099_tsv_raw_format_1.sh similarity index 69% rename from tests/queries/0_stateless/02099_tsv_raw_format.sh rename to tests/queries/0_stateless/02099_tsv_raw_format_1.sh index a69c96ab613..a3468f46ca0 100755 --- a/tests/queries/0_stateless/02099_tsv_raw_format.sh +++ b/tests/queries/0_stateless/02099_tsv_raw_format_1.sh @@ -46,16 +46,3 @@ echo 'nSome text' | $CLICKHOUSE_CLIENT -q "INSERT INTO test_nullable_string_0209 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_nullable_string_02099" $CLICKHOUSE_CLIENT -q "DROP TABLE test_nullable_string_02099" - - -$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_parallel_parsing_02099" -$CLICKHOUSE_CLIENT -q "CREATE TABLE test_parallel_parsing_02099 (x UInt64, a Array(UInt64), s String) ENGINE=Memory()"; -$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(1000000) FORMAT TSVRaw" | $CLICKHOUSE_CLIENT --input_format_parallel_parsing=0 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw" -$CLICKHOUSE_CLIENT -q "SELECT * FROM test_parallel_parsing_02099 ORDER BY x" | md5sum - -$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE test_parallel_parsing_02099" - -$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(1000000) FORMAT TSVRaw" | $CLICKHOUSE_CLIENT --input_format_parallel_parsing=1 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw" -$CLICKHOUSE_CLIENT -q "SELECT * FROM test_parallel_parsing_02099 ORDER BY x" | md5sum - -$CLICKHOUSE_CLIENT -q "DROP TABLE test_parallel_parsing_02099" diff --git a/tests/queries/0_stateless/02099_tsv_raw_format_2.reference b/tests/queries/0_stateless/02099_tsv_raw_format_2.reference new file mode 100644 index 00000000000..4682749c21f --- /dev/null +++ b/tests/queries/0_stateless/02099_tsv_raw_format_2.reference @@ -0,0 +1,2 @@ +c8ff17885084035ea1aebd95fee2efb6 - +c8ff17885084035ea1aebd95fee2efb6 - diff --git a/tests/queries/0_stateless/02099_tsv_raw_format_2.sh b/tests/queries/0_stateless/02099_tsv_raw_format_2.sh new file mode 100755 index 00000000000..d6034a0616f --- /dev/null +++ b/tests/queries/0_stateless/02099_tsv_raw_format_2.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Tags: long + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_parallel_parsing_02099" +$CLICKHOUSE_CLIENT -q "CREATE TABLE test_parallel_parsing_02099 (x UInt64, a Array(UInt64), s String) ENGINE=Memory()"; +$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(100000) FORMAT TSVRaw" | $CLICKHOUSE_CLIENT --input_format_parallel_parsing=0 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw" +$CLICKHOUSE_CLIENT -q "SELECT * FROM test_parallel_parsing_02099 ORDER BY x" | md5sum + +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE test_parallel_parsing_02099" + +$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(100000) FORMAT TSVRaw" | $CLICKHOUSE_CLIENT --input_format_parallel_parsing=1 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw" +$CLICKHOUSE_CLIENT -q "SELECT * FROM test_parallel_parsing_02099 ORDER BY x" | md5sum + +$CLICKHOUSE_CLIENT -q "DROP TABLE test_parallel_parsing_02099" + From ead9dc42d8a75ac99a0e4f538764cba206ad59a7 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 7 Aug 2024 08:31:49 +0000 Subject: [PATCH 122/129] Make it worse --- ...02310_clickhouse_local_INSERT_progress_profile_events.expect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect b/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect index d5b2a278180..5c731ea6d89 100755 --- a/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect +++ b/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect @@ -1,5 +1,5 @@ #!/usr/bin/expect -f -# Tags: no-debug, no-tsan, no-msan, no-asan, no-ubsan, no-s3-storage +# Tags: no-debug, no-tsan, no-msan, no-asan, no-ubsan, no-s3-storage, no-cpu-aarch64 # ^ it can be slower than 60 seconds # This is the regression for the concurrent access in ProgressIndication, From de41ffa18f5c2d3533fab3222aa401c193a0baac Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Wed, 7 Aug 2024 12:06:14 +0200 Subject: [PATCH 123/129] fix docks for groupConcat function documentation had wrong usage of a function. Examples of usage - https://github.com/ClickHouse/ClickHouse/blob/763952bf36d0b55f1b33ff11c693267574aa9666/tests/queries/0_stateless/03156_group_concat.sql --- .../aggregate-functions/reference/groupconcat.md | 6 +++--- tests/instructions/easy_tasks_sorted_ru.md | 4 ---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md b/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md index 072252de8c9..bfa6160bbcc 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md +++ b/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md @@ -10,7 +10,7 @@ Calculates a concatenated string from a group of strings, optionally separated b **Syntax** ``` sql -groupConcat(expression [, delimiter] [, limit]); +groupConcat[(delimiter [, limit])](expression); ``` **Arguments** @@ -20,7 +20,7 @@ groupConcat(expression [, delimiter] [, limit]); - `limit` — A positive [integer](../../../sql-reference/data-types/int-uint.md) specifying the maximum number of elements to concatenate. If more elements are present, excess elements are ignored. This parameter is optional. :::note -If delimiter is specified without limit, it must be the first parameter following the expression. If both delimiter and limit are specified, delimiter must precede limit. +If delimiter is specified without limit, it must be the first parameter. If both delimiter and limit are specified, delimiter must precede limit. ::: **Returned value** @@ -61,7 +61,7 @@ This concatenates all names into one continuous string without any separator. Query: ``` sql -SELECT groupConcat(Name, ', ', 2) FROM Employees; +SELECT groupConcat(', ')(Name) FROM Employees; ``` Result: diff --git a/tests/instructions/easy_tasks_sorted_ru.md b/tests/instructions/easy_tasks_sorted_ru.md index fbd86ebf08f..e0607126ecc 100644 --- a/tests/instructions/easy_tasks_sorted_ru.md +++ b/tests/instructions/easy_tasks_sorted_ru.md @@ -97,10 +97,6 @@ Upd: сделали по-другому: теперь всё безопасно. Возвращает инкрементальное число для повторно встречающихся значений key. -## Агрегатная функция groupConcat. - -`groupConcat(x, ',')` - собрать из переданных значений x строку, разделённую запятыми. - ## Функции DATE_ADD, DATE_SUB как синонимы для совместимости с SQL. https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_date-add From a9735f470c195dadbfffe02545b0979bfa9bd778 Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Wed, 7 Aug 2024 12:09:26 +0200 Subject: [PATCH 124/129] squash! fix docks for groupConcat function documentation had wrong usage of a function. Examples of usage - https://github.com/ClickHouse/ClickHouse/blob/763952bf36d0b55f1b33ff11c693267574aa9666/tests/queries/0_stateless/03156_group_concat.sql --- .../sql-reference/aggregate-functions/reference/groupconcat.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md b/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md index bfa6160bbcc..6a24aa244bf 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md +++ b/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md @@ -78,7 +78,7 @@ This output shows the names separated by a comma followed by a space. Query: ``` sql -SELECT groupConcat(Name, ', ', 2) FROM Employees; +SELECT groupConcat(', ', 2)(Name) FROM Employees; ``` Result: From e8bf5129c03dd88712829e9e187145e248ba4f04 Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Wed, 7 Aug 2024 12:44:31 +0200 Subject: [PATCH 125/129] fix docks for clickhouse-keeper-client starting 24.7 paths are not accepted as bare strings, only as string literals - https://github.com/ClickHouse/ClickHouse/pull/65494 --- .../utilities/clickhouse-keeper-client.md | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/en/operations/utilities/clickhouse-keeper-client.md b/docs/en/operations/utilities/clickhouse-keeper-client.md index 6407c66783b..a66ecbc1372 100644 --- a/docs/en/operations/utilities/clickhouse-keeper-client.md +++ b/docs/en/operations/utilities/clickhouse-keeper-client.md @@ -28,39 +28,39 @@ A client application to interact with clickhouse-keeper by its native protocol. Connected to ZooKeeper at [::1]:9181 with session_id 137 / :) ls keeper foo bar -/ :) cd keeper +/ :) cd 'keeper' /keeper :) ls api_version -/keeper :) cd api_version +/keeper :) cd 'api_version' /keeper/api_version :) ls -/keeper/api_version :) cd xyz +/keeper/api_version :) cd 'xyz' Path /keeper/api_version/xyz does not exist /keeper/api_version :) cd ../../ / :) ls keeper foo bar -/ :) get keeper/api_version +/ :) get 'keeper/api_version' 2 ``` ## Commands {#clickhouse-keeper-client-commands} -- `ls [path]` -- Lists the nodes for the given path (default: cwd) -- `cd [path]` -- Changes the working path (default `.`) -- `exists ` -- Returns `1` if node exists, `0` otherwise -- `set [version]` -- Updates the node's value. Only updates if version matches (default: -1) -- `create [mode]` -- Creates new node with the set value -- `touch ` -- Creates new node with an empty string as value. Doesn't throw an exception if the node already exists -- `get ` -- Returns the node's value -- `rm [version]` -- Removes the node only if version matches (default: -1) -- `rmr ` -- Recursively deletes path. Confirmation required +- `ls '[path]'` -- Lists the nodes for the given path (default: cwd) +- `cd '[path]'` -- Changes the working path (default `.`) +- `exists ''` -- Returns `1` if node exists, `0` otherwise +- `set '' [version]` -- Updates the node's value. Only updates if version matches (default: -1) +- `create '' [mode]` -- Creates new node with the set value +- `touch ''` -- Creates new node with an empty string as value. Doesn't throw an exception if the node already exists +- `get ''` -- Returns the node's value +- `rm '' [version]` -- Removes the node only if version matches (default: -1) +- `rmr ''` -- Recursively deletes path. Confirmation required - `flwc ` -- Executes four-letter-word command - `help` -- Prints this message -- `get_direct_children_number [path]` -- Get numbers of direct children nodes under a specific path -- `get_all_children_number [path]` -- Get all numbers of children nodes under a specific path -- `get_stat [path]` -- Returns the node's stat (default `.`) -- `find_super_nodes [path]` -- Finds nodes with number of children larger than some threshold for the given path (default `.`) +- `get_direct_children_number '[path]'` -- Get numbers of direct children nodes under a specific path +- `get_all_children_number '[path]'` -- Get all numbers of children nodes under a specific path +- `get_stat '[path]'` -- Returns the node's stat (default `.`) +- `find_super_nodes '[path]'` -- Finds nodes with number of children larger than some threshold for the given path (default `.`) - `delete_stale_backups` -- Deletes ClickHouse nodes used for backups that are now inactive - `find_big_family [path] [n]` -- Returns the top n nodes with the biggest family in the subtree (default path = `.` and n = 10) -- `sync ` -- Synchronizes node between processes and leader +- `sync ''` -- Synchronizes node between processes and leader - `reconfig "" [version]` -- Reconfigure Keeper cluster. See https://clickhouse.com/docs/en/guides/sre/keeper/clickhouse-keeper#reconfiguration From c39cdcffaff5917510b27f457e35a63dddeed0b5 Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Wed, 7 Aug 2024 14:03:53 +0200 Subject: [PATCH 126/129] docs for lightweight_mutation_projection_mode option starting with 24.7 we can lightweight delete in tables with projections. Fixed docs stating it is not possible --- docs/en/sql-reference/statements/delete.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/delete.md b/docs/en/sql-reference/statements/delete.md index a52b7204c30..8745a06c124 100644 --- a/docs/en/sql-reference/statements/delete.md +++ b/docs/en/sql-reference/statements/delete.md @@ -36,9 +36,10 @@ If you anticipate frequent deletes, consider using a [custom partitioning key](/ ## Limitations of lightweight `DELETE` -### Lightweight `DELETE`s do not work with projections +### Lightweight `DELETE`s with projections -Currently, `DELETE` does not work for tables with projections. This is because rows in a projection may be affected by a `DELETE` operation and may require the projection to be rebuilt, negatively affecting `DELETE` performance. +By default, `DELETE` does not work for tables with projections. This is because rows in a projection may be affected by a `DELETE` operation and may require the projection to be rebuilt, negatively affecting `DELETE` performance. +However, there is an option to change this behavior. By changing setting `lightweight_mutation_projection_mode = 'drop'` will work with projections. ## Performance considerations when using lightweight `DELETE` From 37a6bd25f6d6dd2bfe3913639a7650c313642fb5 Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Wed, 7 Aug 2024 14:06:02 +0200 Subject: [PATCH 127/129] squash! docs for lightweight_mutation_projection_mode option starting with 24.7 we can lightweight delete in tables with projections. Fixed docs stating it is not possible --- docs/en/sql-reference/statements/delete.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/delete.md b/docs/en/sql-reference/statements/delete.md index 8745a06c124..88a9c933519 100644 --- a/docs/en/sql-reference/statements/delete.md +++ b/docs/en/sql-reference/statements/delete.md @@ -39,7 +39,7 @@ If you anticipate frequent deletes, consider using a [custom partitioning key](/ ### Lightweight `DELETE`s with projections By default, `DELETE` does not work for tables with projections. This is because rows in a projection may be affected by a `DELETE` operation and may require the projection to be rebuilt, negatively affecting `DELETE` performance. -However, there is an option to change this behavior. By changing setting `lightweight_mutation_projection_mode = 'drop'` will work with projections. +However, there is an option to change this behavior. By changing setting `lightweight_mutation_projection_mode = 'drop'`, deletes will work with projections. ## Performance considerations when using lightweight `DELETE` From ecba21bfe78fabf76eb7207f40e3fe53ecb70eaf Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 7 Aug 2024 15:23:19 +0200 Subject: [PATCH 128/129] Fix documentation for memory overcommit --- docs/en/operations/settings/settings.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 2b11c836fc1..968481062e9 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -4629,8 +4629,8 @@ Default Value: 5. ## memory_overcommit_ratio_denominator {#memory_overcommit_ratio_denominator} -It represents soft memory limit in case when hard limit is reached on user level. -This value is used to compute overcommit ratio for the query. +It represents the soft memory limit when the hard limit is reached on the global level. +This value is used to compute the overcommit ratio for the query. Zero means skip the query. Read more about [memory overcommit](memory-overcommit.md). @@ -4646,8 +4646,8 @@ Default value: `5000000`. ## memory_overcommit_ratio_denominator_for_user {#memory_overcommit_ratio_denominator_for_user} -It represents soft memory limit in case when hard limit is reached on global level. -This value is used to compute overcommit ratio for the query. +It represents the soft memory limit when the hard limit is reached on the user level. +This value is used to compute the overcommit ratio for the query. Zero means skip the query. Read more about [memory overcommit](memory-overcommit.md). From 9a812fc53502cbd3bb69964d8e8b1afb186e6b46 Mon Sep 17 00:00:00 2001 From: "Max K." Date: Wed, 7 Aug 2024 15:55:03 +0200 Subject: [PATCH 129/129] Revert "CI: Strict job timeout 1.5h for tests, 2h for builds" --- tests/ci/ci_config.py | 1 + tests/ci/ci_definitions.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index f578cd8b559..7a19eb6f827 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -510,6 +510,7 @@ class CI: JobNames.LIBFUZZER_TEST: JobConfig( required_builds=[BuildNames.FUZZERS], run_by_label=Tags.libFuzzer, + timeout=10800, run_command='libfuzzer_test_check.py "$CHECK_NAME"', runner_type=Runners.STYLE_CHECKER, ), diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py index 69e7ed259d5..48847b0d7a6 100644 --- a/tests/ci/ci_definitions.py +++ b/tests/ci/ci_definitions.py @@ -331,7 +331,7 @@ class JobConfig: # will be triggered for the job if omitted in CI workflow yml run_command: str = "" # job timeout, seconds - timeout: Optional[int] = 5400 + timeout: Optional[int] = None # sets number of batches for a multi-batch job num_batches: int = 1 # label that enables job in CI, if set digest isn't used @@ -420,6 +420,7 @@ class CommonJobConfigs: ), run_command='functional_test_check.py "$CHECK_NAME"', runner_type=Runners.FUNC_TESTER, + timeout=9000, ) STATEFUL_TEST = JobConfig( job_name_keyword="stateful", @@ -530,6 +531,7 @@ class CommonJobConfigs: docker=["clickhouse/sqllogic-test"], ), run_command="sqllogic_test.py", + timeout=10800, release_only=True, runner_type=Runners.FUNC_TESTER, ) @@ -541,6 +543,7 @@ class CommonJobConfigs: docker=["clickhouse/sqltest"], ), run_command="sqltest.py", + timeout=10800, release_only=True, runner_type=Runners.FUZZER_UNIT_TESTER, ) @@ -610,7 +613,6 @@ class CommonJobConfigs: docker=["clickhouse/binary-builder"], git_submodules=True, ), - timeout=7200, run_command="build_check.py $BUILD_NAME", runner_type=Runners.BUILDER, )