From 2c055480d622d0ccc05b65c9c0252b36b66f7eca Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 24 Oct 2023 14:52:47 +0000 Subject: [PATCH 001/192] Remove unnecessary flag --- src/Processors/Sources/RemoteSource.cpp | 6 ++---- src/Processors/Sources/RemoteSource.h | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp index 74ab3649068..6ca5e611713 100644 --- a/src/Processors/Sources/RemoteSource.cpp +++ b/src/Processors/Sources/RemoteSource.cpp @@ -42,7 +42,7 @@ void RemoteSource::setStorageLimits(const std::shared_ptr RemoteSource::tryGenerate() { /// onCancel() will do the cancel if the query was sent. - if (was_query_canceled) + if (isCancelled()) return {}; if (!was_query_sent) @@ -169,7 +169,6 @@ std::optional RemoteSource::tryGenerate() void RemoteSource::onCancel() { - was_query_canceled = true; query_executor->cancel(); } @@ -177,7 +176,6 @@ void RemoteSource::onUpdatePorts() { if (getPort().isFinished()) { - was_query_canceled = true; query_executor->finish(); } } diff --git a/src/Processors/Sources/RemoteSource.h b/src/Processors/Sources/RemoteSource.h index da39b5d0046..dbfa0156331 100644 --- a/src/Processors/Sources/RemoteSource.h +++ b/src/Processors/Sources/RemoteSource.h @@ -39,7 +39,6 @@ protected: void onCancel() override; private: - std::atomic was_query_canceled = false; bool was_query_sent = false; bool add_aggregation_info = false; RemoteQueryExecutorPtr query_executor; From 940d099e84d92eaaacaa96682c5a94b26f7a782c Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Fri, 27 Oct 2023 16:50:34 -0700 Subject: [PATCH 002/192] Set correct max_block_size value in docs --- docs/en/operations/settings/settings.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index ccf290c8e20..60eda45ab22 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -731,11 +731,13 @@ Default value: LZ4. ## max_block_size {#setting-max_block_size} -In ClickHouse, data is processed by blocks (sets of column parts). The internal processing cycles for a single block are efficient enough, but there are noticeable expenditures on each block. The `max_block_size` setting is a recommendation for what size of the block (in a count of rows) to load from tables. The block size shouldn’t be too small, so that the expenditures on each block are still noticeable, but not too large so that the query with LIMIT that is completed after the first block is processed quickly. The goal is to avoid consuming too much memory when extracting a large number of columns in multiple threads and to preserve at least some cache locality. +In ClickHouse, data is processed by blocks, which are sets of column parts. The internal processing cycles for a single block are efficient but there are noticeable costs when processing each block. -Default value: 65,536. +The `max_block_size` setting indicates the recommended maximum number of rows to include in a single block when loading data from tables. Blocks the size of `max_block_size` are not always loaded from the table: if ClickHouse determines that less data needs to be retrieved, a smaller block is processed. -Blocks the size of `max_block_size` are not always loaded from the table. If it is obvious that less data needs to be retrieved, a smaller block is processed. +The block size should not be too small to avoid noticeable costs when processing each block. It should also not be too large to ensure that queries with a LIMIT clause execute quickly after processing the first block. When setting `max_block_size`, the goal should be to avoid consuming too much memory when extracting a large number of columns in multiple threads and to preserve at least some cache locality. + +Default value: `65,409` ## preferred_block_size_bytes {#preferred-block-size-bytes} From 70e3dd808cc3f087504892d18a9e61eb6f948151 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 29 Oct 2023 02:07:24 +0100 Subject: [PATCH 003/192] Granular code coverage with introspection --- CMakeLists.txt | 9 -- base/base/CMakeLists.txt | 2 + base/base/coverage.cpp | 106 ++++++++++++++++++- base/base/coverage.h | 6 ++ base/glibc-compatibility/memcpy/memcpy.cpp | 1 + base/glibc-compatibility/memcpy/memcpy.h | 2 +- cmake/sanitize.cmake | 18 ++++ contrib/CMakeLists.txt | 9 -- contrib/google-protobuf-cmake/CMakeLists.txt | 32 ------ contrib/libcxx-cmake/CMakeLists.txt | 2 - programs/CMakeLists.txt | 2 + src/CMakeLists.txt | 5 +- src/Functions/coverage.cpp | 91 ++++++++++++++++ src/Interpreters/InterpreterSystemQuery.cpp | 8 ++ src/Parsers/ASTSystemQuery.h | 1 + src/Parsers/ParserSystemQuery.cpp | 16 +-- 16 files changed, 244 insertions(+), 66 deletions(-) create mode 100644 src/Functions/coverage.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a5b94efefc5..d259b105a0a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -286,9 +286,6 @@ set (CMAKE_C_STANDARD 11) set (CMAKE_C_EXTENSIONS ON) # required by most contribs written in C set (CMAKE_C_STANDARD_REQUIRED ON) -# Compiler-specific coverage flags e.g. -fcoverage-mapping -option(WITH_COVERAGE "Profile the resulting binary/binaries" OFF) - if (COMPILER_CLANG) # Enable C++14 sized global deallocation functions. It should be enabled by setting -std=c++14 but I'm not sure. # See https://reviews.llvm.org/D112921 @@ -304,12 +301,6 @@ if (COMPILER_CLANG) set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries") set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}") endif() - - if (WITH_COVERAGE) - set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-instr-generate -fcoverage-mapping") - # If we want to disable coverage for specific translation units - set(WITHOUT_COVERAGE "-fno-profile-instr-generate -fno-coverage-mapping") - endif() endif () set (COMPILER_FLAGS "${COMPILER_FLAGS}") diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt index 8ab3c8a0711..f9bf413a6c8 100644 --- a/base/base/CMakeLists.txt +++ b/base/base/CMakeLists.txt @@ -1,3 +1,5 @@ +add_compile_options($<$,$>:${COVERAGE_FLAGS}>) + if (USE_CLANG_TIDY) set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () diff --git a/base/base/coverage.cpp b/base/base/coverage.cpp index 1027638be3d..60eb6fcac72 100644 --- a/base/base/coverage.cpp +++ b/base/base/coverage.cpp @@ -1,11 +1,15 @@ #include "coverage.h" -#if WITH_COVERAGE - #pragma GCC diagnostic ignored "-Wreserved-identifier" -# include -# include + +/// WITH_COVERAGE enables the default implementation of code coverage, +/// that dumps a map to the filesystem. + +#if WITH_COVERAGE + +#include +#include # if defined(__clang__) @@ -31,3 +35,97 @@ void dumpCoverageReportIfPossible() #endif } + + +/// SANITIZE_COVERAGE enables code instrumentation, +/// but leaves the callbacks implementation to us, +/// which we use to calculate coverage on a per-test basis +/// and to write it to system tables. + +#if defined(SANITIZE_COVERAGE) + +namespace +{ + bool initialized = false; + + uint32_t * guards_start = nullptr; + uint32_t * guards_end = nullptr; + size_t coverage_array_size = 0; + + uintptr_t * coverage_array = nullptr; +} + +extern "C" +{ + +/// This is called at least once for every DSO for initialization. +/// But we will use it only for the main DSO. +void __sanitizer_cov_trace_pc_guard_init(uint32_t * start, uint32_t * stop) +{ + if (initialized) + return; + initialized = true; + + /// The function can be called multiple times, but we need to initialize only once. + if (start == stop || *start) + return; + + guards_start = start; + guards_end = stop; + coverage_array_size = stop - start; + + /// Note: we will leak this. + coverage_array = static_cast(malloc(sizeof(uintptr_t) * coverage_array_size)); + + resetCoverage(); +} + +/// This is called at every basic block / edge, etc. +void __sanitizer_cov_trace_pc_guard(uint32_t * guard) +{ + /// Duplicate the guard check. + if (!*guard) + return; + *guard = 0; + + /// If you set *guard to 0 this code will not be called again for this edge. + /// Now we can get the PC and do whatever you want: + /// - store it somewhere or symbolize it and print right away. + /// The values of `*guard` are as you set them in + /// __sanitizer_cov_trace_pc_guard_init and so you can make them consecutive + /// and use them to dereference an array or a bit vector. + void * pc = __builtin_return_address(0); + + coverage_array[guard - guards_start] = reinterpret_cast(pc); +} + +} + +__attribute__((no_sanitize("coverage"))) std::span getCoverage() +{ + return {coverage_array, coverage_array_size}; +} + +__attribute__((no_sanitize("coverage"))) void resetCoverage() +{ + memset(coverage_array, 0, coverage_array_size * sizeof(*coverage_array)); + + /// The guard defines whether the __sanitizer_cov_trace_pc_guard should be called. + /// For example, you can unset it after first invocation to prevent excessive work. + /// Initially set all the guards to 1 to enable callbacks. + for (uint32_t * x = guards_start; x < guards_end; ++x) + *x = 1; +} + +#else + +std::span getCoverage() +{ + return {}; +} + +void resetCoverage() +{ +} + +#endif diff --git a/base/base/coverage.h b/base/base/coverage.h index 4a57528b0ce..b6664bec223 100644 --- a/base/base/coverage.h +++ b/base/base/coverage.h @@ -1,5 +1,8 @@ #pragma once +#include +#include + /// Flush coverage report to file, depending on coverage system /// proposed by compiler (llvm for clang and gcov for gcc). /// @@ -7,3 +10,6 @@ /// Thread safe (use exclusive lock). /// Idempotent, may be called multiple times. void dumpCoverageReportIfPossible(); + +std::span getCoverage(); +void resetCoverage(); diff --git a/base/glibc-compatibility/memcpy/memcpy.cpp b/base/glibc-compatibility/memcpy/memcpy.cpp index ec43a2c3649..8bab35934d3 100644 --- a/base/glibc-compatibility/memcpy/memcpy.cpp +++ b/base/glibc-compatibility/memcpy/memcpy.cpp @@ -1,5 +1,6 @@ #include "memcpy.h" +__attribute__((no_sanitize("coverage"))) extern "C" void * memcpy(void * __restrict dst, const void * __restrict src, size_t size) { return inline_memcpy(dst, src, size); diff --git a/base/glibc-compatibility/memcpy/memcpy.h b/base/glibc-compatibility/memcpy/memcpy.h index 0930dfb5c67..86439dda061 100644 --- a/base/glibc-compatibility/memcpy/memcpy.h +++ b/base/glibc-compatibility/memcpy/memcpy.h @@ -93,7 +93,7 @@ * See https://habr.com/en/company/yandex/blog/457612/ */ - +__attribute__((no_sanitize("coverage"))) static inline void * inline_memcpy(void * __restrict dst_, const void * __restrict src_, size_t size) { /// We will use pointer arithmetic, so char pointer will be used. diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index f17283774eb..0c901f1aa36 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -58,3 +58,21 @@ if (SANITIZE) message (FATAL_ERROR "Unknown sanitizer type: ${SANITIZE}") endif () endif() + +# Default coverage instrumentation (dumping the coverage map on exit) +option(WITH_COVERAGE "Instrumentation for code coverage with default implementation" OFF) + +if (WITH_COVERAGE) + message (INFORMATION "Enabled instrumentation for code coverage") + set(COVERAGE_FLAGS "-fprofile-instr-generate -fcoverage-mapping") +endif() + +option (SANITIZE_COVERAGE "Instrumentation for code coverage with custom callbacks" OFF) + +if (SANITIZE_COVERAGE) + message (INFORMATION "Enabled instrumentation for code coverage") + add_definitions(-DSANITIZE_COVERAGE=1) + set (COVERAGE_FLAGS "-fsanitize-coverage=trace-pc-guard") +endif() + +set (WITHOUT_COVERAGE_FLAGS "-fno-profile-instr-generate -fno-coverage-mapping -fno-sanitize-coverage=trace-pc-guard") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 390b0241e7d..fa97e59eefc 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -3,15 +3,6 @@ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w") -if (WITH_COVERAGE) - set (WITHOUT_COVERAGE_LIST ${WITHOUT_COVERAGE}) - separate_arguments(WITHOUT_COVERAGE_LIST) - # disable coverage for contib files and build with optimisations - if (COMPILER_CLANG) - add_compile_options(-O3 -DNDEBUG -finline-functions -finline-hint-functions ${WITHOUT_COVERAGE_LIST}) - endif() -endif() - if (SANITIZE STREQUAL "undefined") # 3rd-party libraries usually not intended to work with UBSan. add_compile_options(-fno-sanitize=undefined) diff --git a/contrib/google-protobuf-cmake/CMakeLists.txt b/contrib/google-protobuf-cmake/CMakeLists.txt index 268f0fbe0e4..fbb7d6ea018 100644 --- a/contrib/google-protobuf-cmake/CMakeLists.txt +++ b/contrib/google-protobuf-cmake/CMakeLists.txt @@ -278,38 +278,6 @@ else () COMMAND_ECHO STDOUT) endif () -# add_custom_command ( -# OUTPUT ${PROTOC_BUILD_DIR} -# COMMAND mkdir -p ${PROTOC_BUILD_DIR}) -# -# add_custom_command ( -# OUTPUT "${PROTOC_BUILD_DIR}/CMakeCache.txt" -# -# COMMAND ${CMAKE_COMMAND} -# -G"${CMAKE_GENERATOR}" -# -DCMAKE_MAKE_PROGRAM="${CMAKE_MAKE_PROGRAM}" -# -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}" -# -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" -# -Dprotobuf_BUILD_TESTS=0 -# -Dprotobuf_BUILD_CONFORMANCE=0 -# -Dprotobuf_BUILD_EXAMPLES=0 -# -Dprotobuf_BUILD_PROTOC_BINARIES=1 -# "${protobuf_source_dir}/cmake" -# -# DEPENDS "${PROTOC_BUILD_DIR}" -# WORKING_DIRECTORY "${PROTOC_BUILD_DIR}" -# COMMENT "Configuring 'protoc' for host architecture." -# USES_TERMINAL) -# -# add_custom_command ( -# OUTPUT "${PROTOC_BUILD_DIR}/protoc" -# COMMAND ${CMAKE_COMMAND} --build "${PROTOC_BUILD_DIR}" -# DEPENDS "${PROTOC_BUILD_DIR}/CMakeCache.txt" -# COMMENT "Building 'protoc' for host architecture." -# USES_TERMINAL) -# -# add_custom_target (protoc-host DEPENDS "${PROTOC_BUILD_DIR}/protoc") - add_executable(protoc IMPORTED GLOBAL) set_target_properties (protoc PROPERTIES IMPORTED_LOCATION "${PROTOC_BUILD_DIR}/protoc") add_dependencies(protoc "${PROTOC_BUILD_DIR}/protoc") diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt index b7e59e2c9a3..c77d5d8319e 100644 --- a/contrib/libcxx-cmake/CMakeLists.txt +++ b/contrib/libcxx-cmake/CMakeLists.txt @@ -1,5 +1,3 @@ -include(CheckCXXCompilerFlag) - set(LIBCXX_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm-project/libcxx") set(SRCS diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index eb4a898d472..fce6894ed11 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -1,3 +1,5 @@ +add_compile_options($<$,$>:${COVERAGE_FLAGS}>) + if (USE_CLANG_TIDY) set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d2985665db3..f88a6cff6c0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,3 +1,5 @@ +add_compile_options($<$,$>:${COVERAGE_FLAGS}>) + if (USE_INCLUDE_WHAT_YOU_USE) set (CMAKE_CXX_INCLUDE_WHAT_YOU_USE ${IWYU_PATH}) endif () @@ -282,7 +284,8 @@ set_source_files_properties( Common/Elf.cpp Common/Dwarf.cpp Common/SymbolIndex.cpp - PROPERTIES COMPILE_FLAGS "-O2 ${WITHOUT_COVERAGE}") + Common/ThreadFuzzer.cpp + PROPERTIES COMPILE_FLAGS "-O2 ${WITHOUT_COVERAGE_FLAGS}") target_link_libraries (clickhouse_common_io PRIVATE diff --git a/src/Functions/coverage.cpp b/src/Functions/coverage.cpp new file mode 100644 index 00000000000..1825e6aa826 --- /dev/null +++ b/src/Functions/coverage.cpp @@ -0,0 +1,91 @@ +#if defined(SANITIZE_COVERAGE) + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ + +namespace +{ + +/** If ClickHouse is build with coverage instrumentation, returns an array + * of currently accumulated unique code addresses. + */ +class FunctionCoverage : public IFunction +{ +public: + static constexpr auto name = "coverage"; + + String getName() const override + { + return name; + } + + explicit FunctionCoverage() + { + } + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override + { + return false; + } + + size_t getNumberOfArguments() const override + { + return 0; + } + + bool isDeterministic() const override + { + return false; + } + + DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override + { + return std::make_shared(std::make_shared()); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName &, const DataTypePtr &, size_t input_rows_count) const override + { + auto coverage_table = getCoverage(); + + auto column_addresses = ColumnUInt64::create(); + auto & data = column_addresses->getData(); + + for (auto ptr : coverage_table) + if (ptr) + data.push_back(ptr); + + auto column_array = ColumnArray::create( + std::move(column_addresses), + ColumnArray::ColumnOffsets::create(1, data.size())); + + return ColumnConst::create(std::move(column_array), input_rows_count); + } +}; + +} + +REGISTER_FUNCTION(Coverage) +{ + factory.registerFunction(); +} + +} + +#endif diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 07a1ae7d170..4e1d32bd3cb 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -60,6 +60,7 @@ #include #include #include +#include #include #include #include @@ -690,6 +691,12 @@ BlockIO InterpreterSystemQuery::execute() FailPointInjection::disableFailPoint(query.fail_point_name); break; } + case Type::RESET_COVERAGE: + { + getContext()->checkAccess(AccessType::SYSTEM); + resetCoverage(); + break; + } default: throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown type of SYSTEM query"); } @@ -1299,6 +1306,7 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() case Type::START_THREAD_FUZZER: case Type::ENABLE_FAILPOINT: case Type::DISABLE_FAILPOINT: + case Type::RESET_COVERAGE: case Type::UNKNOWN: case Type::END: break; } diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index cc06e0fdcb5..5f7ba5be330 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -86,6 +86,7 @@ public: START_PULLING_REPLICATION_LOG, STOP_CLEANUP, START_CLEANUP, + RESET_COVERAGE, END }; diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index a26fdc1396b..f0fc38d6adb 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -453,14 +453,14 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & } case Type::DROP_FORMAT_SCHEMA_CACHE: { - if (ParserKeyword{"FOR"}.ignore(pos, expected)) - { - if (ParserKeyword{"Protobuf"}.ignore(pos, expected)) - res->schema_cache_format = "Protobuf"; - else - return false; - } - break; + if (ParserKeyword{"FOR"}.ignore(pos, expected)) + { + if (ParserKeyword{"Protobuf"}.ignore(pos, expected)) + res->schema_cache_format = "Protobuf"; + else + return false; + } + break; } case Type::UNFREEZE: { From 56de2333f9c69097e57ec2134f6270271f1d5b3b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 29 Oct 2023 16:55:47 +0100 Subject: [PATCH 004/192] Add warning --- programs/server/Server.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index af460ccc7d9..854168a2041 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -666,6 +666,10 @@ try global_context->addWarningMessage("Server was built with sanitizer. It will work slowly."); #endif +#if defined(SANITIZE_COVERAGE) || WITH_COVERAGE + global_context->addWarningMessage("Server was built with code coverage. It will work slowly."); +#endif + const size_t physical_server_memory = getMemoryAmount(); LOG_INFO(log, "Available RAM: {}; physical cores: {}; logical cores: {}.", From 8e0f48738710f2715f04006db9ddfb0d76c0a865 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 29 Oct 2023 17:21:45 +0100 Subject: [PATCH 005/192] Initial support in clickhouse-test --- cmake/sanitize.cmake | 8 +++++++- tests/clickhouse-test | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index 0c901f1aa36..7d25a85ef62 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -71,7 +71,13 @@ option (SANITIZE_COVERAGE "Instrumentation for code coverage with custom callbac if (SANITIZE_COVERAGE) message (INFORMATION "Enabled instrumentation for code coverage") - add_definitions(-DSANITIZE_COVERAGE=1) + + # We set this define for whole build to indicate that at least some parts are compiled with coverage. + # And to expose it in system.build_options. + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSANITIZE_COVERAGE=1") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSANITIZE_COVERAGE=1") + + # But the actual coverage will be enabled on per-library basis: for ClickHouse code, but not for 3rd-party. set (COVERAGE_FLAGS "-fsanitize-coverage=trace-pc-guard") endif() diff --git a/tests/clickhouse-test b/tests/clickhouse-test index cab7d7e79ff..2a4ed865dd5 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1173,6 +1173,16 @@ class TestCase: description_full += result.reason.value description_full += result.description + + if BuildFlags.SANITIZE_COVERAGE in args.build_flags: + coverage = clickhouse_execute( + args, + f"SELECT length(coverage())", + retry_error_codes=True, + ).decode() + + description_full += f" Coverage: {coverage}" + description_full += "\n" if result.status == TestStatus.FAIL and self.testcase_args: @@ -1872,6 +1882,7 @@ class BuildFlags: UNDEFINED = "ubsan" MEMORY = "msan" DEBUG = "debug" + SANITIZE_COVERAGE = "sanitize-coverage" RELEASE = "release" ORDINARY_DATABASE = "ordinary-database" POLYMORPHIC_PARTS = "polymorphic-parts" @@ -1891,6 +1902,8 @@ def collect_build_flags(args): result.append(BuildFlags.UNDEFINED) elif b"-fsanitize=memory" in value: result.append(BuildFlags.MEMORY) + elif b"-DSANITIZE_COVERAGE=1" in value: + result.append(BuildFlags.SANITIZE_COVERAGE) value = clickhouse_execute( args, "SELECT value FROM system.build_options WHERE name = 'BUILD_TYPE'" @@ -2072,6 +2085,8 @@ def reportCoverageFor(args, what, query, permissive=False): return True +# This is high-level coverage on per-component basis (functions, data types, etc.) +# Don't be confused with the code coverage. def reportCoverage(args): clickhouse_execute(args, "SYSTEM FLUSH LOGS") From 4288cb3b7895df917f982e03d0d0b55029ecc5cc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 29 Oct 2023 17:43:01 +0100 Subject: [PATCH 006/192] Make clickhouse-test to calculate coverage on a per-test basis --- tests/clickhouse-test | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 2a4ed865dd5..e827a596ada 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1175,6 +1175,12 @@ class TestCase: description_full += result.description if BuildFlags.SANITIZE_COVERAGE in args.build_flags: + clickhouse_execute( + args, + f"INSERT INTO system.coverage SELECT '{self.case}', coverage()", + retry_error_codes=True, + ) + coverage = clickhouse_execute( args, f"SELECT length(coverage())", @@ -1241,6 +1247,14 @@ class TestCase: + pattern ) + # We want to calculate per-test code coverage. That's why we reset it before each test. + if BuildFlags.SANITIZE_COVERAGE in args.build_flags: + clickhouse_execute( + args, + "SYSTEM RESET COVERAGE", + retry_error_codes=True, + ) + command = pattern.format(**params) proc = Popen(command, shell=True, env=os.environ) @@ -2349,6 +2363,18 @@ def main(args): print(f"Failed to create databases for tests: {e}") server_died.set() + if BuildFlags.SANITIZE_COVERAGE in args.build_flags: + clickhouse_execute( + args, + """ + CREATE TABLE IF NOT EXISTS system.coverage + ( + test_name String, + coverage Array(UInt64) + ) ENGINE = MergeTree ORDER BY test_name; + """, + ) + total_tests_run = 0 for suite in sorted(os.listdir(base_dir), key=suite_key_func): From 8e6a7fdff09430378a6b13e87ded874524327e3b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 29 Oct 2023 23:43:36 +0100 Subject: [PATCH 007/192] Fix f-string --- CMakeLists.txt | 7 ++++--- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- tests/clickhouse-test | 8 +++++++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d259b105a0a..d19bb521c70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -187,9 +187,10 @@ if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") endif () endif() -if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" - OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" - OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") +if (NOT (SANITIZE_COVERAGE OR WITH_COVERAGE) + AND (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" + OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" + OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL")) set (OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT ON) else() set (OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT OFF) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 069ed20c730..705972da8f4 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8795,7 +8795,7 @@ void StorageReplicatedMergeTree::createTableSharedID() const else if (code == Coordination::Error::ZNONODE) /// table completely dropped, we can choose any id we want { id = toString(UUIDHelpers::Nil); - LOG_DEBUG(log, "Table was completely drop, we can use anything as ID (will use {})", id); + LOG_DEBUG(log, "Table was completely dropped, and we can use anything as ID (will use {})", id); } else if (code != Coordination::Error::ZOK) { diff --git a/tests/clickhouse-test b/tests/clickhouse-test index e827a596ada..e5659e8fca4 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1183,7 +1183,7 @@ class TestCase: coverage = clickhouse_execute( args, - f"SELECT length(coverage())", + "SELECT length(coverage())", retry_error_codes=True, ).decode() @@ -2375,6 +2375,12 @@ def main(args): """, ) + # Coverage collected at the system startup before running any tests: + clickhouse_execute( + args, + "INSERT INTO system.coverage SELECT '', coverage()", + ) + total_tests_run = 0 for suite in sorted(os.listdir(base_dir), key=suite_key_func): From ea6cb1ad0c95f194519c863bb29302e8829669a3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 30 Oct 2023 01:04:50 +0100 Subject: [PATCH 008/192] Maybe better --- tests/clickhouse-test | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index e5659e8fca4..36846a4aeb1 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1177,7 +1177,7 @@ class TestCase: if BuildFlags.SANITIZE_COVERAGE in args.build_flags: clickhouse_execute( args, - f"INSERT INTO system.coverage SELECT '{self.case}', coverage()", + f"INSERT INTO system.coverage SELECT now(), '{self.case}', coverage()", retry_error_codes=True, ) @@ -2369,6 +2369,7 @@ def main(args): """ CREATE TABLE IF NOT EXISTS system.coverage ( + time DateTime, test_name String, coverage Array(UInt64) ) ENGINE = MergeTree ORDER BY test_name; @@ -2378,7 +2379,7 @@ def main(args): # Coverage collected at the system startup before running any tests: clickhouse_execute( args, - "INSERT INTO system.coverage SELECT '', coverage()", + "INSERT INTO system.coverage SELECT now(), '', coverage()", ) total_tests_run = 0 From ccf5003442eff0b60cafad3faa489fc2c7ff1aa0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 30 Oct 2023 03:20:04 +0100 Subject: [PATCH 009/192] Maybe smaller binary --- src/Functions/geometryConverters.h | 5 +++++ src/Functions/polygonArea.cpp | 4 ++++ src/Functions/polygonConvexHull.cpp | 5 ++++- src/Functions/polygonPerimeter.cpp | 5 +++++ src/Functions/polygonsDistance.cpp | 5 ++++- src/Functions/polygonsEquals.cpp | 7 ++++++- src/Functions/polygonsIntersection.cpp | 5 ++++- src/Functions/polygonsSymDifference.cpp | 4 ++++ src/Functions/polygonsUnion.cpp | 10 ++++------ src/Functions/polygonsWithin.cpp | 3 +++ src/Functions/readWkt.cpp | 4 ++++ src/Functions/svg.cpp | 5 +++++ src/Functions/wkt.cpp | 6 ++++++ 13 files changed, 58 insertions(+), 10 deletions(-) diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 97162fa9dd0..dba984b4184 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -28,6 +28,9 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +namespace +{ + template using Ring = boost::geometry::model::ring; @@ -371,3 +374,5 @@ static void callOnTwoGeometryDataTypes(DataTypePtr left_type, DataTypePtr right_ } } + +} diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index e49a4eb9fb3..1c4ef9f79a3 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -26,6 +26,9 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +namespace +{ + template class FunctionPolygonArea : public IFunction { @@ -99,6 +102,7 @@ const char * FunctionPolygonArea::name = "polygonAreaCartesian"; template <> const char * FunctionPolygonArea::name = "polygonAreaSpherical"; +} REGISTER_FUNCTION(PolygonArea) { diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index d7fca45bd1e..921c0700ca7 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -25,6 +25,9 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +namespace +{ + template class FunctionPolygonConvexHull : public IFunction { @@ -94,10 +97,10 @@ public: } }; - template <> const char * FunctionPolygonConvexHull::name = "polygonConvexHullCartesian"; +} REGISTER_FUNCTION(PolygonConvexHull) { diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index 2d89d4e4f5b..85645118f84 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -17,13 +17,17 @@ #include #include + namespace DB { + namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +namespace +{ template class FunctionPolygonPerimeter : public IFunction @@ -97,6 +101,7 @@ const char * FunctionPolygonPerimeter::name = "polygonPerimeterC template <> const char * FunctionPolygonPerimeter::name = "polygonPerimeterSpherical"; +} REGISTER_FUNCTION(PolygonPerimeter) { diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index d2c58105eae..d6c7d799b5e 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -27,6 +27,9 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +namespace +{ + template class FunctionPolygonsDistance : public IFunction { @@ -108,6 +111,7 @@ const char * FunctionPolygonsDistance::name = "polygonsDistanceC template <> const char * FunctionPolygonsDistance::name = "polygonsDistanceSpherical"; +} REGISTER_FUNCTION(PolygonsDistance) { @@ -115,5 +119,4 @@ REGISTER_FUNCTION(PolygonsDistance) factory.registerFunction>(); } - } diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index 3c80ae1e4c5..bdc4f18042c 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -19,13 +19,18 @@ #include #include + namespace DB { + namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +namespace +{ + template class FunctionPolygonsEquals : public IFunction { @@ -103,10 +108,10 @@ public: } }; - template <> const char * FunctionPolygonsEquals::name = "polygonsEqualsCartesian"; +} REGISTER_FUNCTION(PolygonsEquals) { diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 84e5fe0d4b7..5777f438a19 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -26,6 +26,9 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +namespace +{ + template class FunctionPolygonsIntersection : public IFunction { @@ -107,13 +110,13 @@ public: } }; - template <> const char * FunctionPolygonsIntersection::name = "polygonsIntersectionCartesian"; template <> const char * FunctionPolygonsIntersection::name = "polygonsIntersectionSpherical"; +} REGISTER_FUNCTION(PolygonsIntersection) { diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index ceb39547427..785a8f76ba6 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -25,6 +25,8 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +namespace +{ template class FunctionPolygonsSymDifference : public IFunction @@ -109,6 +111,8 @@ const char * FunctionPolygonsSymDifference::name = "polygonsSymD template <> const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceSpherical"; +} + REGISTER_FUNCTION(PolygonsSymDifference) { factory.registerFunction>(); diff --git a/src/Functions/polygonsUnion.cpp b/src/Functions/polygonsUnion.cpp index 4a604d0f810..a31d223ea8c 100644 --- a/src/Functions/polygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -3,19 +3,14 @@ #include #include -#include #include -#include #include -#include -#include #include -#include #include -#include + namespace DB { @@ -25,6 +20,8 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +namespace +{ template class FunctionPolygonsUnion : public IFunction @@ -112,6 +109,7 @@ const char * FunctionPolygonsUnion::name = "polygonsUnionCartesi template <> const char * FunctionPolygonsUnion::name = "polygonsUnionSpherical"; +} REGISTER_FUNCTION(PolygonsUnion) { diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 1b094f42060..bf4db1cf9f8 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -27,6 +27,8 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +namespace +{ template class FunctionPolygonsWithin : public IFunction @@ -111,6 +113,7 @@ const char * FunctionPolygonsWithin::name = "polygonsWithinCarte template <> const char * FunctionPolygonsWithin::name = "polygonsWithinSpherical"; +} REGISTER_FUNCTION(PolygonsWithin) { diff --git a/src/Functions/readWkt.cpp b/src/Functions/readWkt.cpp index ec20cdf3723..8dff297bcb1 100644 --- a/src/Functions/readWkt.cpp +++ b/src/Functions/readWkt.cpp @@ -16,6 +16,8 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +namespace +{ template class FunctionReadWKT : public IFunction @@ -95,6 +97,8 @@ struct ReadWKTMultiPolygonNameHolder static constexpr const char * name = "readWKTMultiPolygon"; }; +} + REGISTER_FUNCTION(ReadWKT) { factory.registerFunction, ReadWKTPointNameHolder>>(); diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index f8f85216b3f..550fe29cfc4 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -16,6 +16,9 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } +namespace +{ + class FunctionSvg : public IFunction { public: @@ -94,6 +97,8 @@ public: } }; +} + REGISTER_FUNCTION(Svg) { factory.registerFunction(); diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index fc9ef75a1e2..afcfabd0bf4 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -9,6 +9,9 @@ namespace DB { +namespace +{ + class FunctionWkt : public IFunction { public: @@ -52,6 +55,7 @@ public: for (size_t i = 0; i < input_rows_count; ++i) { std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + str.exceptions(std::ios::failbit); str << boost::geometry::wkt(figures[i]); std::string serialized = str.str(); res_column->insertData(serialized.c_str(), serialized.size()); @@ -68,6 +72,8 @@ public: } }; +} + REGISTER_FUNCTION(Wkt) { factory.registerFunction(); From aaca32f6a7f5c84ce9be36f2d7864c3f80a56b4f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 30 Oct 2023 03:22:49 +0100 Subject: [PATCH 010/192] Add function coverageAll --- base/base/coverage.cpp | 46 +++++++++++++++++++++++++++++++++----- base/base/coverage.h | 12 +++++++++- cmake/sanitize.cmake | 4 ++-- src/Functions/coverage.cpp | 27 ++++++++++++---------- 4 files changed, 68 insertions(+), 21 deletions(-) diff --git a/base/base/coverage.cpp b/base/base/coverage.cpp index 60eb6fcac72..4af6a279af9 100644 --- a/base/base/coverage.cpp +++ b/base/base/coverage.cpp @@ -46,13 +46,17 @@ void dumpCoverageReportIfPossible() namespace { - bool initialized = false; + bool pc_guards_initialized = false; + bool pc_table_initialized = false; uint32_t * guards_start = nullptr; uint32_t * guards_end = nullptr; - size_t coverage_array_size = 0; uintptr_t * coverage_array = nullptr; + size_t coverage_array_size = 0; + + uintptr_t * all_addresses_array = nullptr; + size_t all_addresses_array_size = 0; } extern "C" @@ -62,9 +66,9 @@ extern "C" /// But we will use it only for the main DSO. void __sanitizer_cov_trace_pc_guard_init(uint32_t * start, uint32_t * stop) { - if (initialized) + if (pc_guards_initialized) return; - initialized = true; + pc_guards_initialized = true; /// The function can be called multiple times, but we need to initialize only once. if (start == stop || *start) @@ -80,6 +84,23 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t * start, uint32_t * stop) resetCoverage(); } +/// This is called at least once for every DSO for initialization +/// and provides information about all instrumented addresses. +void __sanitizer_cov_pcs_init(const uintptr_t * pcs_begin, const uintptr_t * pcs_end) +{ + if (pc_table_initialized) + return; + pc_table_initialized = true; + + all_addresses_array = static_cast(malloc(sizeof(uintptr_t) * coverage_array_size)); + all_addresses_array_size = pcs_end - pcs_begin; + + /// They are not a real pointers, but also contain a flag in the most significant bit, + /// in which we are not interested for now. Reset it. + for (size_t i = 0; i < all_addresses_array_size; ++i) + all_addresses_array[i] = pcs_begin[i] & 0x7FFFFFFFFFFFFFFFULL; +} + /// This is called at every basic block / edge, etc. void __sanitizer_cov_trace_pc_guard(uint32_t * guard) { @@ -101,12 +122,20 @@ void __sanitizer_cov_trace_pc_guard(uint32_t * guard) } -__attribute__((no_sanitize("coverage"))) std::span getCoverage() +__attribute__((no_sanitize("coverage"))) +std::span getCoverage() { return {coverage_array, coverage_array_size}; } -__attribute__((no_sanitize("coverage"))) void resetCoverage() +__attribute__((no_sanitize("coverage"))) +std::span getAllInstrumentedAddresses() +{ + return {all_addresses_array, all_addresses_array_size}; +} + +__attribute__((no_sanitize("coverage"))) +void resetCoverage() { memset(coverage_array, 0, coverage_array_size * sizeof(*coverage_array)); @@ -124,6 +153,11 @@ std::span getCoverage() return {}; } +std::span getAllInstrumentedAddresses() +{ + return {}; +} + void resetCoverage() { } diff --git a/base/base/coverage.h b/base/base/coverage.h index b6664bec223..f75ed2d3553 100644 --- a/base/base/coverage.h +++ b/base/base/coverage.h @@ -11,5 +11,15 @@ /// Idempotent, may be called multiple times. void dumpCoverageReportIfPossible(); -std::span getCoverage(); +/// This is effective if SANITIZE_COVERAGE is enabled at build time. +/// Get accumulated unique program addresses of the instrumented parts of the code, +/// seen so far after program startup or after previous reset. +/// The returned span will be represented as a sparse map, containing mostly zeros, which you should filter away. +std::span getCoverage(); + +/// Get all instrumented addresses that could be in the coverage. +std::span getAllInstrumentedAddresses(); + +/// Reset the accumulated coverage. +/// This is useful to compare coverage of different tests, including differential coverage. void resetCoverage(); diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index 7d25a85ef62..3f7a8498059 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -78,7 +78,7 @@ if (SANITIZE_COVERAGE) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSANITIZE_COVERAGE=1") # But the actual coverage will be enabled on per-library basis: for ClickHouse code, but not for 3rd-party. - set (COVERAGE_FLAGS "-fsanitize-coverage=trace-pc-guard") + set (COVERAGE_FLAGS "-fsanitize-coverage=trace-pc-guard,pc-table") endif() -set (WITHOUT_COVERAGE_FLAGS "-fno-profile-instr-generate -fno-coverage-mapping -fno-sanitize-coverage=trace-pc-guard") +set (WITHOUT_COVERAGE_FLAGS "-fno-profile-instr-generate -fno-coverage-mapping -fno-sanitize-coverage=trace-pc-guard,pc-table") diff --git a/src/Functions/coverage.cpp b/src/Functions/coverage.cpp index 1825e6aa826..8a62469fa54 100644 --- a/src/Functions/coverage.cpp +++ b/src/Functions/coverage.cpp @@ -18,28 +18,30 @@ namespace DB namespace { +enum class Kind +{ + Current, + All +}; + /** If ClickHouse is build with coverage instrumentation, returns an array - * of currently accumulated unique code addresses. + * of currently accumulated (`coverage`) / all possible (`coverageAll`) unique code addresses. */ class FunctionCoverage : public IFunction { -public: - static constexpr auto name = "coverage"; +private: + Kind kind; +public: String getName() const override { - return name; + return kind == Kind::Current ? "coverage" : "coverageAll"; } - explicit FunctionCoverage() + explicit FunctionCoverage(Kind kind_) : kind(kind_) { } - static FunctionPtr create(ContextPtr) - { - return std::make_shared(); - } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; @@ -62,7 +64,7 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName &, const DataTypePtr &, size_t input_rows_count) const override { - auto coverage_table = getCoverage(); + auto coverage_table = kind == Kind::Current ? getCoverage() : getAllInstrumentedAddresses(); auto column_addresses = ColumnUInt64::create(); auto & data = column_addresses->getData(); @@ -83,7 +85,8 @@ public: REGISTER_FUNCTION(Coverage) { - factory.registerFunction(); + factory.registerFunction("coverage", [](ContextPtr){ return std::make_unique(std::make_shared(Kind::Current)); }); + factory.registerFunction("coverageAll", [](ContextPtr){ return std::make_unique(std::make_shared(Kind::All)); }); } } From ad4bde6b8bcafcb9c87454cbeeb0448533279e07 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 30 Oct 2023 03:54:11 +0100 Subject: [PATCH 011/192] Fix build --- base/base/coverage.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/base/coverage.cpp b/base/base/coverage.cpp index 4af6a279af9..d70c3bcd82b 100644 --- a/base/base/coverage.cpp +++ b/base/base/coverage.cpp @@ -148,7 +148,7 @@ void resetCoverage() #else -std::span getCoverage() +std::span getCoverage() { return {}; } From f8e209ebd26f278ed582adf0aab8f786be8bb591 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 30 Oct 2023 13:45:18 +0300 Subject: [PATCH 012/192] WindowTransform decrease amount of virtual function calls --- src/Processors/Transforms/WindowTransform.cpp | 19 ++++++++++++------- src/Processors/Transforms/WindowTransform.h | 3 +++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp index 9565a073f48..df6246510bd 100644 --- a/src/Processors/Transforms/WindowTransform.cpp +++ b/src/Processors/Transforms/WindowTransform.cpp @@ -257,6 +257,7 @@ WindowTransform::WindowTransform(const Block & input_header_, window_description.frame = *custom_default_frame; } + workspace.is_aggregate_function_state = workspace.aggregate_function->isState(); workspace.aggregate_function_state.reset( aggregate_function->sizeOfData(), aggregate_function->alignOfData()); @@ -957,10 +958,7 @@ void WindowTransform::updateAggregationState() auto * columns = ws.argument_columns.data(); // Removing arena.get() from the loop makes it faster somehow... auto * arena_ptr = arena.get(); - for (auto row = first_row; row < past_the_end_row; ++row) - { - a->add(buf, columns, row, arena_ptr); - } + a->addBatchSinglePlaceFromInterval(first_row, past_the_end_row, buf, columns, arena_ptr); } } } @@ -987,9 +985,16 @@ void WindowTransform::writeOutCurrentRow() // FIXME does it also allocate the result on the arena? // We'll have to pass it out with blocks then... - /// We should use insertMergeResultInto to insert result into ColumnAggregateFunction - /// correctly if result contains AggregateFunction's states - a->insertMergeResultInto(buf, *result_column, arena.get()); + if (ws.is_aggregate_function_state) + { + /// We should use insertMergeResultInto to insert result into ColumnAggregateFunction + /// correctly if result contains AggregateFunction's states + a->insertMergeResultInto(buf, *result_column, arena.get()); + } + else + { + a->insertResultInto(buf, *result_column, arena.get()); + } } } } diff --git a/src/Processors/Transforms/WindowTransform.h b/src/Processors/Transforms/WindowTransform.h index de3e82d15ee..347c2516230 100644 --- a/src/Processors/Transforms/WindowTransform.h +++ b/src/Processors/Transforms/WindowTransform.h @@ -26,6 +26,9 @@ struct WindowFunctionWorkspace { AggregateFunctionPtr aggregate_function; + // Cached value of aggregate function isState virtual method + bool is_aggregate_function_state = false; + // This field is set for pure window functions. When set, we ignore the // window_function.aggregate_function, and work through this interface // instead. From 52a3d37ebe6008fc4301d369f6309587b32e648b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 27 Oct 2023 16:59:14 +0200 Subject: [PATCH 013/192] Try reducing number of different images --- .../integration/test_backward_compatibility/test.py | 2 +- .../test_aggregate_fixed_key.py | 2 +- .../test_convert_ordinary.py | 2 +- .../test_cte_distributed.py | 2 +- .../test_insert_profile_events.py | 2 +- .../test_memory_bound_aggregation.py | 4 ++-- .../test_normalized_count_comparison.py | 2 +- .../test_select_aggregate_alias_column.py | 2 +- .../test_vertical_merges_from_compact_parts.py | 2 +- .../test_default_compression_codec/test.py | 2 +- tests/integration/test_disk_over_web_server/test.py | 2 +- .../test_distributed_backward_compatability/test.py | 2 +- .../test.py | 2 +- .../test_distributed_inter_server_secret/test.py | 2 +- .../test_groupBitmapAnd_on_distributed/test.py | 2 +- tests/integration/test_old_versions/test.py | 2 +- tests/integration/test_polymorphic_parts/test.py | 2 +- .../test_replicated_merge_tree_compatibility/test.py | 4 ++-- tests/integration/test_replicating_constants/test.py | 2 +- tests/integration/test_ttl_replicated/test.py | 6 +++--- tests/integration/test_version_update/test.py | 12 ++++++------ .../test_version_update_after_mutation/test.py | 6 +++--- 22 files changed, 33 insertions(+), 33 deletions(-) diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py index 6f21b184a95..847483f2b9b 100644 --- a/tests/integration/test_backward_compatibility/test.py +++ b/tests/integration/test_backward_compatibility/test.py @@ -7,7 +7,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=True, image="yandex/clickhouse-server", - tag="19.17.8.54", + tag="19.16.9.37", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py index cf258987cbf..94bc1d3bfc9 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py @@ -7,7 +7,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=True, image="yandex/clickhouse-server", - tag="21.3", + tag="20.8.11.17", with_installed_binary=True, allow_analyzer=False, ) diff --git a/tests/integration/test_backward_compatibility/test_convert_ordinary.py b/tests/integration/test_backward_compatibility/test_convert_ordinary.py index 36facdd59b1..034a68e0f30 100644 --- a/tests/integration/test_backward_compatibility/test_convert_ordinary.py +++ b/tests/integration/test_backward_compatibility/test_convert_ordinary.py @@ -5,7 +5,7 @@ cluster = ClickHouseCluster(__file__) node = cluster.add_instance( "node", image="yandex/clickhouse-server", - tag="19.17.8.54", + tag="19.16.9.37", stay_alive=True, with_zookeeper=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_cte_distributed.py b/tests/integration/test_backward_compatibility/test_cte_distributed.py index c68468aad75..d47ae3aa255 100644 --- a/tests/integration/test_backward_compatibility/test_cte_distributed.py +++ b/tests/integration/test_backward_compatibility/test_cte_distributed.py @@ -8,7 +8,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, image="yandex/clickhouse-server", - tag="21.7.3.14", + tag="21.6", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_backward_compatibility/test_insert_profile_events.py b/tests/integration/test_backward_compatibility/test_insert_profile_events.py index 8564c6b5952..d38bece7855 100644 --- a/tests/integration/test_backward_compatibility/test_insert_profile_events.py +++ b/tests/integration/test_backward_compatibility/test_insert_profile_events.py @@ -11,7 +11,7 @@ upstream_node = cluster.add_instance("upstream_node", allow_analyzer=False) old_node = cluster.add_instance( "old_node", image="clickhouse/clickhouse-server", - tag="22.5.1.2079", + tag="22.6", with_installed_binary=True, allow_analyzer=False, ) diff --git a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py index 96b41c81384..5261a279a4f 100644 --- a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py @@ -7,7 +7,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=False, image="yandex/clickhouse-server", - tag="21.1", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, allow_analyzer=False, @@ -16,7 +16,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, image="yandex/clickhouse-server", - tag="21.1", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py b/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py index 3cd708d5029..cf7a25e8dc1 100644 --- a/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py +++ b/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py @@ -8,7 +8,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, image="yandex/clickhouse-server", - tag="21.7.2.7", + tag="21.6", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py index 7e10b6ab430..ec1d7fedac5 100644 --- a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py +++ b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py @@ -8,7 +8,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, image="yandex/clickhouse-server", - tag="21.7.2.7", + tag="21.6", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py index 9c9d1a4d312..e0a9b5ebad6 100644 --- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py +++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py @@ -7,7 +7,7 @@ cluster = ClickHouseCluster(__file__) node_old = cluster.add_instance( "node1", image="clickhouse/clickhouse-server", - tag="22.8", + tag="22.6", stay_alive=True, with_installed_binary=True, with_zookeeper=True, diff --git a/tests/integration/test_default_compression_codec/test.py b/tests/integration/test_default_compression_codec/test.py index 82d5eb04d2a..db116ff42f3 100644 --- a/tests/integration/test_default_compression_codec/test.py +++ b/tests/integration/test_default_compression_codec/test.py @@ -29,7 +29,7 @@ node3 = cluster.add_instance( "node3", main_configs=["configs/default_compression.xml", "configs/wide_parts_only.xml"], image="yandex/clickhouse-server", - tag="20.3.16", + tag="19.16.9.37", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_disk_over_web_server/test.py b/tests/integration/test_disk_over_web_server/test.py index 7695d235425..a71fdeff302 100644 --- a/tests/integration/test_disk_over_web_server/test.py +++ b/tests/integration/test_disk_over_web_server/test.py @@ -38,7 +38,7 @@ def cluster(): stay_alive=True, with_installed_binary=True, image="clickhouse/clickhouse-server", - tag="22.8.14.53", + tag="22.6", allow_analyzer=False, ) diff --git a/tests/integration/test_distributed_backward_compatability/test.py b/tests/integration/test_distributed_backward_compatability/test.py index c48a7ad1fa1..319a4c08e60 100644 --- a/tests/integration/test_distributed_backward_compatability/test.py +++ b/tests/integration/test_distributed_backward_compatability/test.py @@ -8,7 +8,7 @@ node_old = cluster.add_instance( "node1", main_configs=["configs/remote_servers.xml"], image="yandex/clickhouse-server", - tag="20.8.9.6", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_distributed_insert_backward_compatibility/test.py b/tests/integration/test_distributed_insert_backward_compatibility/test.py index 1e566d5e2da..7cfea61ffff 100644 --- a/tests/integration/test_distributed_insert_backward_compatibility/test.py +++ b/tests/integration/test_distributed_insert_backward_compatibility/test.py @@ -11,7 +11,7 @@ node_dist = cluster.add_instance( "node2", main_configs=["configs/remote_servers.xml"], image="yandex/clickhouse-server", - tag="21.11.9.1", + tag="21.6", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index 1aeaddcf3c5..62beeee80e1 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -31,7 +31,7 @@ backward = make_instance( "configs/remote_servers_backward.xml", image="clickhouse/clickhouse-server", # version without DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 - tag="23.2.3", + tag="22.6", with_installed_binary=True, allow_analyzer=False, ) diff --git a/tests/integration/test_groupBitmapAnd_on_distributed/test.py b/tests/integration/test_groupBitmapAnd_on_distributed/test.py index 8cf7e0fb2c1..5d3dda8ecf2 100644 --- a/tests/integration/test_groupBitmapAnd_on_distributed/test.py +++ b/tests/integration/test_groupBitmapAnd_on_distributed/test.py @@ -26,7 +26,7 @@ node4 = cluster.add_instance( "node4", main_configs=["configs/clusters.xml"], image="yandex/clickhouse-server", - tag="21.5", + tag="21.6", with_zookeeper=True, allow_analyzer=False, ) diff --git a/tests/integration/test_old_versions/test.py b/tests/integration/test_old_versions/test.py index aff07c53114..b59bfcc4f6b 100644 --- a/tests/integration/test_old_versions/test.py +++ b/tests/integration/test_old_versions/test.py @@ -55,7 +55,7 @@ node19_13 = cluster.add_instance( node19_16 = cluster.add_instance( "node19_16", image="yandex/clickhouse-server", - tag="19.16.2.2", + tag="19.16.9.37", with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"], allow_analyzer=False, diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index debb509de90..ba9b5ec6cac 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -360,7 +360,7 @@ node7 = cluster.add_instance( user_configs=["configs_old/users.d/not_optimize_count.xml"], with_zookeeper=True, image="yandex/clickhouse-server", - tag="19.17.8.54", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_replicated_merge_tree_compatibility/test.py b/tests/integration/test_replicated_merge_tree_compatibility/test.py index c30a0d86c98..32a44aa65b9 100644 --- a/tests/integration/test_replicated_merge_tree_compatibility/test.py +++ b/tests/integration/test_replicated_merge_tree_compatibility/test.py @@ -6,7 +6,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.12.4.5", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, allow_analyzer=False, @@ -15,7 +15,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.12.4.5", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_replicating_constants/test.py b/tests/integration/test_replicating_constants/test.py index 00781e473c7..9669e890cd3 100644 --- a/tests/integration/test_replicating_constants/test.py +++ b/tests/integration/test_replicating_constants/test.py @@ -9,7 +9,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=True, image="yandex/clickhouse-server", - tag="19.1.14", + tag="19.16.9.37", with_installed_binary=True, allow_analyzer=False, ) diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index 117ebe37dd2..29ce2b3dc8d 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -17,7 +17,7 @@ node4 = cluster.add_instance( "node4", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.12.4.5", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, main_configs=[ @@ -30,7 +30,7 @@ node5 = cluster.add_instance( "node5", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.12.4.5", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, main_configs=[ @@ -42,7 +42,7 @@ node6 = cluster.add_instance( "node6", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.12.4.5", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, main_configs=[ diff --git a/tests/integration/test_version_update/test.py b/tests/integration/test_version_update/test.py index b8fa3e7ebb4..a752960bc76 100644 --- a/tests/integration/test_version_update/test.py +++ b/tests/integration/test_version_update/test.py @@ -12,18 +12,18 @@ node2 = cluster.add_instance( "node2", with_zookeeper=True, image="yandex/clickhouse-server", - tag="21.2", + tag="20.8.11.17", with_installed_binary=True, stay_alive=True, allow_analyzer=False, ) -# Use differents nodes because if there is node.restart_from_latest_version(), then in later tests +# Use different nodes because if there is node.restart_from_latest_version(), then in later tests # it will be with latest version, but shouldn't, order of tests in CI is shuffled. node3 = cluster.add_instance( "node3", image="yandex/clickhouse-server", - tag="21.5", + tag="21.6", with_installed_binary=True, stay_alive=True, allow_analyzer=False, @@ -31,7 +31,7 @@ node3 = cluster.add_instance( node4 = cluster.add_instance( "node4", image="yandex/clickhouse-server", - tag="21.5", + tag="21.6", with_installed_binary=True, stay_alive=True, allow_analyzer=False, @@ -39,7 +39,7 @@ node4 = cluster.add_instance( node5 = cluster.add_instance( "node5", image="yandex/clickhouse-server", - tag="21.5", + tag="21.6", with_installed_binary=True, stay_alive=True, allow_analyzer=False, @@ -47,7 +47,7 @@ node5 = cluster.add_instance( node6 = cluster.add_instance( "node6", image="yandex/clickhouse-server", - tag="21.5", + tag="21.6", with_installed_binary=True, stay_alive=True, allow_analyzer=False, diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index f3ae190ee46..9fb396b1c14 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -10,7 +10,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.4.9.110", + tag="20.8.11.17", with_installed_binary=True, stay_alive=True, main_configs=[ @@ -22,7 +22,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.4.9.110", + tag="20.8.11.17", with_installed_binary=True, stay_alive=True, main_configs=[ @@ -34,7 +34,7 @@ node3 = cluster.add_instance( "node3", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.4.9.110", + tag="20.8.11.17", with_installed_binary=True, stay_alive=True, main_configs=[ From 957671bf744cd173676f5be0f8ca14d0f03118df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 31 Oct 2023 18:06:28 +0100 Subject: [PATCH 014/192] Adapt to work with releases without DROP SYNC --- tests/integration/test_ttl_replicated/test.py | 222 ++++++++---------- 1 file changed, 102 insertions(+), 120 deletions(-) diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index 29ce2b3dc8d..119a211ae45 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -66,47 +66,41 @@ def started_cluster(): cluster.shutdown() -def drop_table(nodes, table_name): - for node in nodes: - node.query("DROP TABLE IF EXISTS {} SYNC".format(table_name)) - - # Column TTL works only with wide parts, because it's very expensive to apply it for compact parts def test_ttl_columns(started_cluster): - drop_table([node1, node2], "test_ttl") + table_name = f"test_ttl_{node1.name}_{node2.name}" for node in [node1, node2]: node.query( """ - CREATE TABLE test_ttl(date DateTime, id UInt32, a Int32 TTL date + INTERVAL 1 DAY, b Int32 TTL date + INTERVAL 1 MONTH) + CREATE TABLE {table_name}(date DateTime, id UInt32, a Int32 TTL date + INTERVAL 1 DAY, b Int32 TTL date + INTERVAL 1 MONTH) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_columns', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) SETTINGS merge_with_ttl_timeout=0, min_bytes_for_wide_part=0, max_merge_selecting_sleep_ms=6000; """.format( - replica=node.name + table_name=table_name, replica=node.name ) ) node1.query( - "INSERT INTO test_ttl VALUES (toDateTime('2000-10-10 00:00:00'), 1, 1, 3)" + f"INSERT INTO {table_name} VALUES (toDateTime('2000-10-10 00:00:00'), 1, 1, 3)" ) node1.query( - "INSERT INTO test_ttl VALUES (toDateTime('2000-10-11 10:00:00'), 2, 2, 4)" + f"INSERT INTO {table_name} VALUES (toDateTime('2000-10-11 10:00:00'), 2, 2, 4)" ) time.sleep(1) # sleep to allow use ttl merge selector for second time - node1.query("OPTIMIZE TABLE test_ttl FINAL") + node1.query(f"OPTIMIZE TABLE {table_name} FINAL") expected = "1\t0\t0\n2\t0\t0\n" - assert TSV(node1.query("SELECT id, a, b FROM test_ttl ORDER BY id")) == TSV( + assert TSV(node1.query(f"SELECT id, a, b FROM {table_name} ORDER BY id")) == TSV( expected ) - assert TSV(node2.query("SELECT id, a, b FROM test_ttl ORDER BY id")) == TSV( + assert TSV(node2.query(f"SELECT id, a, b FROM {table_name} ORDER BY id")) == TSV( expected ) def test_merge_with_ttl_timeout(started_cluster): - table = "test_merge_with_ttl_timeout" - drop_table([node1, node2], table) + table = f"test_merge_with_ttl_timeout_{node1.name}_{node2.name}" for node in [node1, node2]: node.query( """ @@ -157,11 +151,11 @@ def test_merge_with_ttl_timeout(started_cluster): def test_ttl_many_columns(started_cluster): - drop_table([node1, node2], "test_ttl_2") + table = f"test_ttl_2{node1.name}_{node2.name}" for node in [node1, node2]: node.query( """ - CREATE TABLE test_ttl_2(date DateTime, id UInt32, + CREATE TABLE {table}(date DateTime, id UInt32, a Int32 TTL date, _idx Int32 TTL date, _offset Int32 TTL date, @@ -169,44 +163,40 @@ def test_ttl_many_columns(started_cluster): ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_2', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) SETTINGS merge_with_ttl_timeout=0, max_merge_selecting_sleep_ms=6000; """.format( - replica=node.name + table=table, replica=node.name ) ) - node1.query("SYSTEM STOP TTL MERGES test_ttl_2") - node2.query("SYSTEM STOP TTL MERGES test_ttl_2") + node1.query(f"SYSTEM STOP TTL MERGES {table}") + node2.query(f"SYSTEM STOP TTL MERGES {table}") node1.query( - "INSERT INTO test_ttl_2 VALUES (toDateTime('2000-10-10 00:00:00'), 1, 2, 3, 4, 5)" + f"INSERT INTO {table} VALUES (toDateTime('2000-10-10 00:00:00'), 1, 2, 3, 4, 5)" ) node1.query( - "INSERT INTO test_ttl_2 VALUES (toDateTime('2100-10-10 10:00:00'), 6, 7, 8, 9, 10)" + f"INSERT INTO {table} VALUES (toDateTime('2100-10-10 10:00:00'), 6, 7, 8, 9, 10)" ) - node2.query("SYSTEM SYNC REPLICA test_ttl_2", timeout=5) + node2.query(f"SYSTEM SYNC REPLICA {table}", timeout=5) # Check that part will appear in result of merge - node1.query("SYSTEM STOP FETCHES test_ttl_2") - node2.query("SYSTEM STOP FETCHES test_ttl_2") + node1.query(f"SYSTEM STOP FETCHES {table}") + node2.query(f"SYSTEM STOP FETCHES {table}") - node1.query("SYSTEM START TTL MERGES test_ttl_2") - node2.query("SYSTEM START TTL MERGES test_ttl_2") + node1.query(f"SYSTEM START TTL MERGES {table}") + node2.query(f"SYSTEM START TTL MERGES {table}") time.sleep(1) # sleep to allow use ttl merge selector for second time - node1.query("OPTIMIZE TABLE test_ttl_2 FINAL", timeout=5) + node1.query(f"OPTIMIZE TABLE {table} FINAL", timeout=5) - node2.query("SYSTEM SYNC REPLICA test_ttl_2", timeout=5) + node2.query(f"SYSTEM SYNC REPLICA {table}", timeout=5) expected = "1\t0\t0\t0\t0\n6\t7\t8\t9\t10\n" assert TSV( - node1.query( - "SELECT id, a, _idx, _offset, _partition FROM test_ttl_2 ORDER BY id" - ) + node1.query(f"SELECT id, a, _idx, _offset, _partition FROM {table} ORDER BY id") ) == TSV(expected) assert TSV( - node2.query( - "SELECT id, a, _idx, _offset, _partition FROM test_ttl_2 ORDER BY id" - ) + node2.query(f"SELECT id, a, _idx, _offset, _partition FROM {table} ORDER BY id") ) == TSV(expected) @@ -218,107 +208,107 @@ def test_ttl_many_columns(started_cluster): ], ) def test_ttl_table(started_cluster, delete_suffix): - drop_table([node1, node2], "test_ttl") + table = f"test_ttl_table_{delete_suffix}_{node1.name}_{node2.name}" for node in [node1, node2]: node.query( """ - CREATE TABLE test_ttl(date DateTime, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') + CREATE TABLE {table}(date DateTime, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 1 DAY {delete_suffix} SETTINGS merge_with_ttl_timeout=0, max_merge_selecting_sleep_ms=6000; """.format( - replica=node.name, delete_suffix=delete_suffix + table=table, replica=node.name, delete_suffix=delete_suffix ) ) - node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-10 00:00:00'), 1)") - node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-11 10:00:00'), 2)") + node1.query(f"INSERT INTO {table} VALUES (toDateTime('2000-10-10 00:00:00'), 1)") + node1.query(f"INSERT INTO {table} VALUES (toDateTime('2000-10-11 10:00:00'), 2)") time.sleep(1) # sleep to allow use ttl merge selector for second time - node1.query("OPTIMIZE TABLE test_ttl FINAL") + node1.query(f"OPTIMIZE TABLE {table} FINAL") - assert TSV(node1.query("SELECT * FROM test_ttl")) == TSV("") - assert TSV(node2.query("SELECT * FROM test_ttl")) == TSV("") + assert TSV(node1.query(f"SELECT * FROM {table}")) == TSV("") + assert TSV(node2.query(f"SELECT * FROM {table}")) == TSV("") def test_modify_ttl(started_cluster): - drop_table([node1, node2], "test_ttl") + table = f"test_modify_ttl_{node1.name}_{node2.name}" for node in [node1, node2]: node.query( """ - CREATE TABLE test_ttl(d DateTime, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_modify', '{replica}') + CREATE TABLE {table}(d DateTime, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}', '{replica}') ORDER BY id """.format( - replica=node.name + table=table, replica=node.name ) ) node1.query( - "INSERT INTO test_ttl VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)" + f"INSERT INTO {table} VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)" ) - node2.query("SYSTEM SYNC REPLICA test_ttl", timeout=20) + node2.query(f"SYSTEM SYNC REPLICA {table}", timeout=20) node1.query( - "ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 4 HOUR SETTINGS replication_alter_partitions_sync = 2" + f"ALTER TABLE {table} MODIFY TTL d + INTERVAL 4 HOUR SETTINGS replication_alter_partitions_sync = 2" ) - assert node2.query("SELECT id FROM test_ttl") == "2\n3\n" + assert node2.query(f"SELECT id FROM {table}") == "2\n3\n" node2.query( - "ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 2 HOUR SETTINGS replication_alter_partitions_sync = 2" + f"ALTER TABLE {table} MODIFY TTL d + INTERVAL 2 HOUR SETTINGS replication_alter_partitions_sync = 2" ) - assert node1.query("SELECT id FROM test_ttl") == "3\n" + assert node1.query(f"SELECT id FROM {table}") == "3\n" node1.query( - "ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 30 MINUTE SETTINGS replication_alter_partitions_sync = 2" + f"ALTER TABLE {table} MODIFY TTL d + INTERVAL 30 MINUTE SETTINGS replication_alter_partitions_sync = 2" ) - assert node2.query("SELECT id FROM test_ttl") == "" + assert node2.query(f"SELECT id FROM {table}") == "" def test_modify_column_ttl(started_cluster): - drop_table([node1, node2], "test_ttl") + table = f"test_modify_column_ttl_{node1.name}_{node2.name}" for node in [node1, node2]: node.query( """ - CREATE TABLE test_ttl(d DateTime, id UInt32 DEFAULT 42) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_column', '{replica}') + CREATE TABLE {table}(d DateTime, id UInt32 DEFAULT 42) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}', '{replica}') ORDER BY d """.format( - replica=node.name + table=table, replica=node.name ) ) node1.query( - "INSERT INTO test_ttl VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)" + f"INSERT INTO {table} VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)" ) - node2.query("SYSTEM SYNC REPLICA test_ttl", timeout=20) + node2.query(f"SYSTEM SYNC REPLICA {table}", timeout=20) node1.query( - "ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 4 HOUR SETTINGS replication_alter_partitions_sync = 2" + f"ALTER TABLE {table} MODIFY COLUMN id UInt32 TTL d + INTERVAL 4 HOUR SETTINGS replication_alter_partitions_sync = 2" ) - assert node2.query("SELECT id FROM test_ttl") == "42\n2\n3\n" + assert node2.query(f"SELECT id FROM {table}") == "42\n2\n3\n" node1.query( - "ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 2 HOUR SETTINGS replication_alter_partitions_sync = 2" + f"ALTER TABLE {table} MODIFY COLUMN id UInt32 TTL d + INTERVAL 2 HOUR SETTINGS replication_alter_partitions_sync = 2" ) - assert node1.query("SELECT id FROM test_ttl") == "42\n42\n3\n" + assert node1.query(f"SELECT id FROM {table}") == "42\n42\n3\n" node1.query( - "ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 30 MINUTE SETTINGS replication_alter_partitions_sync = 2" + f"ALTER TABLE {table} MODIFY COLUMN id UInt32 TTL d + INTERVAL 30 MINUTE SETTINGS replication_alter_partitions_sync = 2" ) - assert node2.query("SELECT id FROM test_ttl") == "42\n42\n42\n" + assert node2.query(f"SELECT id FROM {table}") == "42\n42\n42\n" def test_ttl_double_delete_rule_returns_error(started_cluster): - drop_table([node1, node2], "test_ttl") + table = "test_ttl_double_delete_rule_returns_error" try: node1.query( """ - CREATE TABLE test_ttl(date DateTime, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_double_delete', '{replica}') + CREATE TABLE {table}(date DateTime, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 1 DAY, date + INTERVAL 2 DAY SETTINGS merge_with_ttl_timeout=0, max_merge_selecting_sleep_ms=6000 """.format( - replica=node1.name + table=table, replica=node1.name ) ) assert False @@ -364,7 +354,6 @@ def test_ttl_alter_delete(started_cluster, name, engine): for a table that has TTL delete expression defined but no explicit storage policy assigned. """ - drop_table([node1], name) node1.query( """ @@ -426,7 +415,6 @@ def test_ttl_alter_delete(started_cluster, name, engine): def test_ttl_empty_parts(started_cluster): - drop_table([node1, node2], "test_ttl_empty_parts") for node in [node1, node2]: node.query( """ @@ -519,65 +507,59 @@ def test_ttl_empty_parts(started_cluster): [(node1, node2, 0), (node3, node4, 1), (node5, node6, 2)], ) def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): - drop_table([node_left, node_right], "test_ttl_delete") - drop_table([node_left, node_right], "test_ttl_group_by") - drop_table([node_left, node_right], "test_ttl_where") - + table = f"test_ttl_compatibility_{node_left.name}_{node_right.name}_{num_run}" for node in [node_left, node_right]: node.query( """ - CREATE TABLE test_ttl_delete(date DateTime, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_delete_{suff}', '{replica}') + CREATE TABLE {table}_delete(date DateTime, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}_delete', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 3 SECOND - SETTINGS max_number_of_merges_with_ttl_in_pool=100, max_replicated_merges_with_ttl_in_queue=100 """.format( - suff=num_run, replica=node.name + table=table, replica=node.name ) ) node.query( """ - CREATE TABLE test_ttl_group_by(date DateTime, id UInt32, val UInt64) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_group_by_{suff}', '{replica}') + CREATE TABLE {table}_group_by(date DateTime, id UInt32, val UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}_group_by', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 3 SECOND GROUP BY id SET val = sum(val) - SETTINGS max_number_of_merges_with_ttl_in_pool=100, max_replicated_merges_with_ttl_in_queue=100 """.format( - suff=num_run, replica=node.name + table=table, replica=node.name ) ) node.query( """ - CREATE TABLE test_ttl_where(date DateTime, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_where_{suff}', '{replica}') + CREATE TABLE {table}_where(date DateTime, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}_where', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 3 SECOND DELETE WHERE id % 2 = 1 - SETTINGS max_number_of_merges_with_ttl_in_pool=100, max_replicated_merges_with_ttl_in_queue=100 """.format( - suff=num_run, replica=node.name + table=table, replica=node.name ) ) - node_left.query("INSERT INTO test_ttl_delete VALUES (now(), 1)") + node_left.query(f"INSERT INTO {table}_delete VALUES (now(), 1)") node_left.query( - "INSERT INTO test_ttl_delete VALUES (toDateTime('2100-10-11 10:00:00'), 2)" + f"INSERT INTO {table}_delete VALUES (toDateTime('2100-10-11 10:00:00'), 2)" ) - node_right.query("INSERT INTO test_ttl_delete VALUES (now(), 3)") + node_right.query(f"INSERT INTO {table}_delete VALUES (now(), 3)") node_right.query( - "INSERT INTO test_ttl_delete VALUES (toDateTime('2100-10-11 10:00:00'), 4)" + f"INSERT INTO {table}_delete VALUES (toDateTime('2100-10-11 10:00:00'), 4)" ) - node_left.query("INSERT INTO test_ttl_group_by VALUES (now(), 0, 1)") - node_left.query("INSERT INTO test_ttl_group_by VALUES (now(), 0, 2)") - node_right.query("INSERT INTO test_ttl_group_by VALUES (now(), 0, 3)") - node_right.query("INSERT INTO test_ttl_group_by VALUES (now(), 0, 4)") + node_left.query(f"INSERT INTO {table}_group_by VALUES (now(), 0, 1)") + node_left.query(f"INSERT INTO {table}_group_by VALUES (now(), 0, 2)") + node_right.query(f"INSERT INTO {table}_group_by VALUES (now(), 0, 3)") + node_right.query(f"INSERT INTO {table}_group_by VALUES (now(), 0, 4)") - node_left.query("INSERT INTO test_ttl_where VALUES (now(), 1)") - node_left.query("INSERT INTO test_ttl_where VALUES (now(), 2)") - node_right.query("INSERT INTO test_ttl_where VALUES (now(), 3)") - node_right.query("INSERT INTO test_ttl_where VALUES (now(), 4)") + node_left.query(f"INSERT INTO {table}_where VALUES (now(), 1)") + node_left.query(f"INSERT INTO {table}_where VALUES (now(), 2)") + node_right.query(f"INSERT INTO {table}_where VALUES (now(), 3)") + node_right.query(f"INSERT INTO {table}_where VALUES (now(), 4)") if node_left.with_installed_binary: node_left.restart_with_latest_version() @@ -588,13 +570,13 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): time.sleep(5) # Wait for TTL # after restart table can be in readonly mode - exec_query_with_retry(node_right, "OPTIMIZE TABLE test_ttl_delete FINAL") - node_right.query("OPTIMIZE TABLE test_ttl_group_by FINAL") - node_right.query("OPTIMIZE TABLE test_ttl_where FINAL") + exec_query_with_retry(node_right, f"OPTIMIZE TABLE {table}_delete FINAL") + node_right.query(f"OPTIMIZE TABLE {table}_group_by FINAL") + node_right.query(f"OPTIMIZE TABLE {table}_where FINAL") - exec_query_with_retry(node_left, "OPTIMIZE TABLE test_ttl_delete FINAL") - node_left.query("OPTIMIZE TABLE test_ttl_group_by FINAL", timeout=20) - node_left.query("OPTIMIZE TABLE test_ttl_where FINAL", timeout=20) + exec_query_with_retry(node_left, f"OPTIMIZE TABLE {table}_delete FINAL") + node_left.query(f"OPTIMIZE TABLE {table}_group_by FINAL", timeout=20) + node_left.query(f"OPTIMIZE TABLE {table}_where FINAL", timeout=20) # After OPTIMIZE TABLE, it is not guaranteed that everything is merged. # Possible scenario (for test_ttl_group_by): @@ -605,19 +587,19 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): # 4. OPTIMIZE FINAL does nothing, cause there is an entry for 0_3 # # So, let's also sync replicas for node_right (for now). - exec_query_with_retry(node_right, "SYSTEM SYNC REPLICA test_ttl_delete") - node_right.query("SYSTEM SYNC REPLICA test_ttl_group_by", timeout=20) - node_right.query("SYSTEM SYNC REPLICA test_ttl_where", timeout=20) + exec_query_with_retry(node_right, f"SYSTEM SYNC REPLICA {table}_delete") + node_right.query(f"SYSTEM SYNC REPLICA {table}_group_by", timeout=20) + node_right.query(f"SYSTEM SYNC REPLICA {table}_where", timeout=20) - exec_query_with_retry(node_left, "SYSTEM SYNC REPLICA test_ttl_delete") - node_left.query("SYSTEM SYNC REPLICA test_ttl_group_by", timeout=20) - node_left.query("SYSTEM SYNC REPLICA test_ttl_where", timeout=20) + exec_query_with_retry(node_left, f"SYSTEM SYNC REPLICA {table}_delete") + node_left.query(f"SYSTEM SYNC REPLICA {table}_group_by", timeout=20) + node_left.query(f"SYSTEM SYNC REPLICA {table}_where", timeout=20) - assert node_left.query("SELECT id FROM test_ttl_delete ORDER BY id") == "2\n4\n" - assert node_right.query("SELECT id FROM test_ttl_delete ORDER BY id") == "2\n4\n" + assert node_left.query(f"SELECT id FROM {table}_delete ORDER BY id") == "2\n4\n" + assert node_right.query(f"SELECT id FROM {table}_delete ORDER BY id") == "2\n4\n" - assert node_left.query("SELECT val FROM test_ttl_group_by ORDER BY id") == "10\n" - assert node_right.query("SELECT val FROM test_ttl_group_by ORDER BY id") == "10\n" + assert node_left.query(f"SELECT val FROM {table}_group_by ORDER BY id") == "10\n" + assert node_right.query(f"SELECT val FROM {table}_group_by ORDER BY id") == "10\n" - assert node_left.query("SELECT id FROM test_ttl_where ORDER BY id") == "2\n4\n" - assert node_right.query("SELECT id FROM test_ttl_where ORDER BY id") == "2\n4\n" + assert node_left.query(f"SELECT id FROM {table}_where ORDER BY id") == "2\n4\n" + assert node_right.query(f"SELECT id FROM {table}_where ORDER BY id") == "2\n4\n" From f2f84fe6b7f49ae3bad1ed6f8c19c608f73d53d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 31 Oct 2023 18:24:33 +0100 Subject: [PATCH 015/192] Adapt version changes --- .../test_vertical_merges_from_compact_parts.py | 2 +- tests/integration/test_default_compression_codec/test.py | 2 +- .../integration/test_version_update_after_mutation/test.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py index e0a9b5ebad6..9c9d1a4d312 100644 --- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py +++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py @@ -7,7 +7,7 @@ cluster = ClickHouseCluster(__file__) node_old = cluster.add_instance( "node1", image="clickhouse/clickhouse-server", - tag="22.6", + tag="22.8", stay_alive=True, with_installed_binary=True, with_zookeeper=True, diff --git a/tests/integration/test_default_compression_codec/test.py b/tests/integration/test_default_compression_codec/test.py index db116ff42f3..ffe22c62325 100644 --- a/tests/integration/test_default_compression_codec/test.py +++ b/tests/integration/test_default_compression_codec/test.py @@ -27,7 +27,7 @@ node2 = cluster.add_instance( ) node3 = cluster.add_instance( "node3", - main_configs=["configs/default_compression.xml", "configs/wide_parts_only.xml"], + main_configs=["configs/default_compression.xml"], image="yandex/clickhouse-server", tag="19.16.9.37", stay_alive=True, diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index 9fb396b1c14..4e84b4c10ca 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -72,8 +72,8 @@ def test_mutate_and_upgrade(start_cluster): node1.query("DETACH TABLE mt") # stop being leader node1.query("SYSTEM FLUSH LOGS") node2.query("SYSTEM FLUSH LOGS") - node1.restart_with_latest_version(signal=9, fix_metadata=True) - node2.restart_with_latest_version(signal=9, fix_metadata=True) + node1.restart_with_latest_version(signal=9, fix_metadata=False) + node2.restart_with_latest_version(signal=9, fix_metadata=False) # After hard restart table can be in readonly mode exec_query_with_retry( @@ -129,7 +129,7 @@ def test_upgrade_while_mutation(start_cluster): # (We could be in process of creating some system table, which will leave empty directory on restart, # so when we start moving system tables from ordinary to atomic db, it will complain about some undeleted files) node3.query("SYSTEM FLUSH LOGS") - node3.restart_with_latest_version(signal=9, fix_metadata=True) + node3.restart_with_latest_version(signal=9, fix_metadata=False) # checks for readonly exec_query_with_retry(node3, "OPTIMIZE TABLE mt1", sleep_time=5, retry_count=60) From 83689c2a04b60288cbeda25d2c57762180273c29 Mon Sep 17 00:00:00 2001 From: flynn Date: Sat, 4 Nov 2023 14:35:39 +0000 Subject: [PATCH 016/192] Support create and materialized index in the same alter query --- src/Interpreters/InterpreterAlterQuery.cpp | 36 +++++++++++-------- ..._add_index_and_materialize_index.reference | 0 .../02911_add_index_and_materialize_index.sql | 16 +++++++++ 3 files changed, 38 insertions(+), 14 deletions(-) create mode 100644 tests/queries/0_stateless/02911_add_index_and_materialize_index.reference create mode 100644 tests/queries/0_stateless/02911_add_index_and_materialize_index.sql diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index f851607000c..c9a1bd17a46 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -128,10 +128,6 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) } else if (auto mut_command = MutationCommand::parse(command_ast)) { - if (mut_command->type == MutationCommand::MATERIALIZE_TTL && !metadata_snapshot->hasAnyTTL()) - throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot MATERIALIZE TTL as there is no TTL set for table {}", - table->getStorageID().getNameForLogs()); - if (mut_command->type == MutationCommand::UPDATE || mut_command->type == MutationCommand::DELETE) { /// TODO: add a check for result query size. @@ -162,8 +158,30 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) "to execute ALTERs of different types (replicated and non replicated) in single query"); } + if (!alter_commands.empty()) + { + auto alter_lock = table->lockForAlter(getContext()->getSettingsRef().lock_acquire_timeout); + StorageInMemoryMetadata metadata = table->getInMemoryMetadata(); + alter_commands.validate(table, getContext()); + alter_commands.prepare(metadata); + table->checkAlterIsPossible(alter_commands, getContext()); + table->alter(alter_commands, getContext(), alter_lock); + } + + /// Get newest metadata_snapshot after execute ALTER command, in order to + /// support like materialize index in the same ALTER query that creates it. + metadata_snapshot = table->getInMemoryMetadataPtr(); + if (mutation_commands.hasNonEmptyMutationCommands()) { + for (const auto & command : mutation_commands) + { + /// Check it after alter finished, so we can add TTL and materialize TTL in the same ALTER query. + if (command.type == MutationCommand::MATERIALIZE_TTL && !metadata_snapshot->hasAnyTTL()) + throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot MATERIALIZE TTL as there is no TTL set for table {}", + table->getStorageID().getNameForLogs()); + + } table->checkMutationIsPossible(mutation_commands, getContext()->getSettingsRef()); MutationsInterpreter::Settings settings(false); MutationsInterpreter(table, metadata_snapshot, mutation_commands, getContext(), settings).validate(); @@ -178,16 +196,6 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) res.pipeline = QueryPipeline(std::move(partition_commands_pipe)); } - if (!alter_commands.empty()) - { - auto alter_lock = table->lockForAlter(getContext()->getSettingsRef().lock_acquire_timeout); - StorageInMemoryMetadata metadata = table->getInMemoryMetadata(); - alter_commands.validate(table, getContext()); - alter_commands.prepare(metadata); - table->checkAlterIsPossible(alter_commands, getContext()); - table->alter(alter_commands, getContext(), alter_lock); - } - return res; } diff --git a/tests/queries/0_stateless/02911_add_index_and_materialize_index.reference b/tests/queries/0_stateless/02911_add_index_and_materialize_index.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02911_add_index_and_materialize_index.sql b/tests/queries/0_stateless/02911_add_index_and_materialize_index.sql new file mode 100644 index 00000000000..57b144a3a8d --- /dev/null +++ b/tests/queries/0_stateless/02911_add_index_and_materialize_index.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS index_test; + +CREATE TABLE index_test +( + x UInt32, + y UInt32, + z UInt32 +) ENGINE = MergeTree order by x; + +ALTER TABLE index_test + ADD INDEX i_x mortonDecode(2, z).1 TYPE minmax GRANULARITY 1, + ADD INDEX i_y mortonDecode(2, z).2 TYPE minmax GRANULARITY 1, + MATERIALIZE INDEX i_x, + MATERIALIZE INDEX i_y; + +drop table index_test; From d8b44dadd5c28a16e627c4c815996af75c37036e Mon Sep 17 00:00:00 2001 From: flynn Date: Sat, 4 Nov 2023 16:16:55 +0000 Subject: [PATCH 017/192] update test --- .../0_stateless/02911_add_index_and_materialize_index.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02911_add_index_and_materialize_index.sql b/tests/queries/0_stateless/02911_add_index_and_materialize_index.sql index 57b144a3a8d..f8785ec9a38 100644 --- a/tests/queries/0_stateless/02911_add_index_and_materialize_index.sql +++ b/tests/queries/0_stateless/02911_add_index_and_materialize_index.sql @@ -1,3 +1,5 @@ +-- Tags: no-replicated-database + DROP TABLE IF EXISTS index_test; CREATE TABLE index_test From f5e439d9dfe32a679f84a6720b461c5a3c5e3a4a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 4 Nov 2023 21:53:05 +0100 Subject: [PATCH 018/192] Add an option to enable or disable coverage collection in clickhouse-test --- tests/clickhouse-test | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 36846a4aeb1..debbb5116da 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1174,7 +1174,7 @@ class TestCase: description_full += result.description - if BuildFlags.SANITIZE_COVERAGE in args.build_flags: + if args.collect_per_test_coverage and BuildFlags.SANITIZE_COVERAGE in args.build_flags: clickhouse_execute( args, f"INSERT INTO system.coverage SELECT now(), '{self.case}', coverage()", @@ -1248,7 +1248,7 @@ class TestCase: ) # We want to calculate per-test code coverage. That's why we reset it before each test. - if BuildFlags.SANITIZE_COVERAGE in args.build_flags: + if args.collect_per_test_coverage and BuildFlags.SANITIZE_COVERAGE in args.build_flags: clickhouse_execute( args, "SYSTEM RESET COVERAGE", @@ -2363,7 +2363,7 @@ def main(args): print(f"Failed to create databases for tests: {e}") server_died.set() - if BuildFlags.SANITIZE_COVERAGE in args.build_flags: + if args.collect_per_test_coverage and BuildFlags.SANITIZE_COVERAGE in args.build_flags: clickhouse_execute( args, """ @@ -2726,6 +2726,12 @@ def parse_args(): default=False, help="Check what high-level server components were covered by tests", ) + parser.add_argument( + "--collect-per-test-coverage", + action="store_true", + default=False, + help="Create `system.coverage` table on the server and collect information about low-level code coverage on a per test basis there", + ) parser.add_argument( "--report-logs-stats", action="store_true", From 501d2106473f7963d10eb37554c42cbab7d5fe4d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 4 Nov 2023 21:04:39 +0000 Subject: [PATCH 019/192] Automatic style fix --- tests/clickhouse-test | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index debbb5116da..a1a270ec033 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1174,7 +1174,10 @@ class TestCase: description_full += result.description - if args.collect_per_test_coverage and BuildFlags.SANITIZE_COVERAGE in args.build_flags: + if ( + args.collect_per_test_coverage + and BuildFlags.SANITIZE_COVERAGE in args.build_flags + ): clickhouse_execute( args, f"INSERT INTO system.coverage SELECT now(), '{self.case}', coverage()", @@ -1248,7 +1251,10 @@ class TestCase: ) # We want to calculate per-test code coverage. That's why we reset it before each test. - if args.collect_per_test_coverage and BuildFlags.SANITIZE_COVERAGE in args.build_flags: + if ( + args.collect_per_test_coverage + and BuildFlags.SANITIZE_COVERAGE in args.build_flags + ): clickhouse_execute( args, "SYSTEM RESET COVERAGE", @@ -2363,7 +2369,10 @@ def main(args): print(f"Failed to create databases for tests: {e}") server_died.set() - if args.collect_per_test_coverage and BuildFlags.SANITIZE_COVERAGE in args.build_flags: + if ( + args.collect_per_test_coverage + and BuildFlags.SANITIZE_COVERAGE in args.build_flags + ): clickhouse_execute( args, """ From cc5179078c3c01bc797732edfabf852befcfaf2f Mon Sep 17 00:00:00 2001 From: flynn Date: Sun, 5 Nov 2023 02:29:15 +0000 Subject: [PATCH 020/192] remove unused code Fix --- src/Interpreters/InterpreterAlterQuery.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index c9a1bd17a46..54b4334eda9 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -105,7 +105,6 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) if (table->isStaticStorage()) throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is read-only"); auto table_lock = table->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); - auto metadata_snapshot = table->getInMemoryMetadataPtr(); /// Add default database to table identifiers that we can encounter in e.g. default expressions, mutation expression, etc. AddDefaultDatabaseVisitor visitor(getContext(), table_id.getDatabaseName()); @@ -170,7 +169,7 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) /// Get newest metadata_snapshot after execute ALTER command, in order to /// support like materialize index in the same ALTER query that creates it. - metadata_snapshot = table->getInMemoryMetadataPtr(); + auto metadata_snapshot = table->getInMemoryMetadataPtr(); if (mutation_commands.hasNonEmptyMutationCommands()) { From e4400ec24c1a7949638f80642f8510978dc6bbea Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 7 Nov 2023 13:33:02 -0300 Subject: [PATCH 021/192] add transition from reading key to reading quoted key when double quotes are found --- src/Functions/keyvaluepair/impl/StateHandlerImpl.h | 5 +++++ ..._extract_key_value_pairs_multiple_input.reference | 12 ++++++++++++ .../02499_extract_key_value_pairs_multiple_input.sql | 12 ++++++++++++ 3 files changed, 29 insertions(+) diff --git a/src/Functions/keyvaluepair/impl/StateHandlerImpl.h b/src/Functions/keyvaluepair/impl/StateHandlerImpl.h index 7fc3ba54833..b4fd91ec3c9 100644 --- a/src/Functions/keyvaluepair/impl/StateHandlerImpl.h +++ b/src/Functions/keyvaluepair/impl/StateHandlerImpl.h @@ -106,6 +106,11 @@ public: { return {next_pos, State::WAITING_KEY}; } + else if (isQuotingCharacter(*p)) + { + // +1 to skip quoting character + return {next_pos, State::READING_QUOTED_KEY}; + } pos = next_pos; } diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference index d0cf9ff680b..f646583bbd3 100644 --- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference +++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference @@ -345,6 +345,18 @@ WITH SELECT x; {'argument1':'1','argument2':'2','char':'=','char2':'=','formula':'1+2=3','result':'3','string':'foo=bar'} +-- https://github.com/ClickHouse/ClickHouse/issues/56357 +WITH + extractKeyValuePairs('{"a":"1", "b":"2"}') as s_map, + CAST( + arrayMap( + (x) -> (x, s_map[x]), arraySort(mapKeys(s_map)) + ), + 'Map(String,String)' + ) AS x +SELECT + x; +{'a':'1','b':'2'} -- check str_to_map alias (it is case-insensitive) WITH sTr_tO_mAp('name:neymar, age:31 team:psg,nationality:brazil') AS s_map, diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql index 804ff4ce880..9277ba6d7ec 100644 --- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql +++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql @@ -481,6 +481,18 @@ WITH SELECT x; +-- https://github.com/ClickHouse/ClickHouse/issues/56357 +WITH + extractKeyValuePairs('{"a":"1", "b":"2"}') as s_map, + CAST( + arrayMap( + (x) -> (x, s_map[x]), arraySort(mapKeys(s_map)) + ), + 'Map(String,String)' + ) AS x +SELECT + x; + -- check str_to_map alias (it is case-insensitive) WITH sTr_tO_mAp('name:neymar, age:31 team:psg,nationality:brazil') AS s_map, From 28ca29fda24350a954dc0747a822161e74992a44 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 7 Nov 2023 14:31:34 -0300 Subject: [PATCH 022/192] remove stale comment --- src/Functions/keyvaluepair/impl/StateHandlerImpl.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/keyvaluepair/impl/StateHandlerImpl.h b/src/Functions/keyvaluepair/impl/StateHandlerImpl.h index b4fd91ec3c9..687d8d95d42 100644 --- a/src/Functions/keyvaluepair/impl/StateHandlerImpl.h +++ b/src/Functions/keyvaluepair/impl/StateHandlerImpl.h @@ -108,7 +108,6 @@ public: } else if (isQuotingCharacter(*p)) { - // +1 to skip quoting character return {next_pos, State::READING_QUOTED_KEY}; } From f619f73f284039c4e3bfebeda2228dd5a799d6e3 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 29 Aug 2023 11:53:32 +0000 Subject: [PATCH 023/192] Fix incorrect header in grace hash join and filter pushdown --- src/QueryPipeline/QueryPipelineBuilder.cpp | 12 +++++------- .../02861_filter_pushdown_const_bug.reference | 2 ++ .../0_stateless/02861_filter_pushdown_const_bug.sql | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/QueryPipeline/QueryPipelineBuilder.cpp b/src/QueryPipeline/QueryPipelineBuilder.cpp index f9726339872..f13d1c56d7f 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.cpp +++ b/src/QueryPipeline/QueryPipelineBuilder.cpp @@ -483,8 +483,6 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesRightLe Block left_header = left->getHeader(); - Block joined_header = JoiningTransform::transformHeader(left_header, join); - for (size_t i = 0; i < num_streams; ++i) { auto joining = std::make_shared( @@ -496,9 +494,9 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesRightLe { // Process delayed joined blocks when all JoiningTransform are finished. auto delayed = std::make_shared( - joined_header, - [left_header, joined_header, max_block_size, join]() - { return join->getNonJoinedBlocks(left_header, joined_header, max_block_size); }); + output_header, + [left_header, output_header, max_block_size, join]() + { return join->getNonJoinedBlocks(left_header, output_header, max_block_size); }); if (delayed->getInputs().size() != 1 || delayed->getOutputs().size() != 1) throw Exception(ErrorCodes::LOGICAL_ERROR, "DelayedJoinedBlocksWorkerTransform should have one input and one output"); @@ -533,7 +531,7 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesRightLe for (size_t i = 1; i < joined_output_ports.size(); i += 2) delayed_ports_numbers.push_back(i); - auto delayed_processor = std::make_shared(joined_header, 2 * num_streams, delayed_ports_numbers); + auto delayed_processor = std::make_shared(output_header, 2 * num_streams, delayed_ports_numbers); if (collected_processors) collected_processors->emplace_back(delayed_processor); left->pipe.processors->emplace_back(delayed_processor); @@ -545,7 +543,7 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesRightLe left->pipe.output_ports.clear(); for (OutputPort & port : delayed_processor->getOutputs()) left->pipe.output_ports.push_back(&port); - left->pipe.header = joined_header; + left->pipe.header = output_header; left->resize(num_streams); } diff --git a/tests/queries/0_stateless/02861_filter_pushdown_const_bug.reference b/tests/queries/0_stateless/02861_filter_pushdown_const_bug.reference index 428ba88bff0..df8198bc856 100644 --- a/tests/queries/0_stateless/02861_filter_pushdown_const_bug.reference +++ b/tests/queries/0_stateless/02861_filter_pushdown_const_bug.reference @@ -6,3 +6,5 @@ 1 1 1 1 +1 1 +1 1 diff --git a/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql b/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql index a5ddf830d48..a299e50984f 100644 --- a/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql +++ b/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql @@ -15,4 +15,8 @@ SELECT key FROM ( SELECT key FROM t1 ) AS t1 JOIN ( SELECT key FROM t1 ) AS t2 O SELECT key FROM ( SELECT 1 AS key ) AS t1 JOIN ( SELECT 1 AS key ) AS t2 ON t1.key = t2.key WHERE key; SELECT * FROM ( SELECT 1 AS key GROUP BY NULL ) AS t1 INNER JOIN (SELECT 1 AS key) AS t2 ON t1.key = t2.key WHERE t1.key ORDER BY key; +SET join_algorithm = 'grace_hash'; + +SELECT * FROM (SELECT key AS a FROM t1 ) t1 INNER JOIN (SELECT key AS c FROM t1 ) t2 ON c = a WHERE a; + DROP TABLE IF EXISTS t1; From 86ba6ad1e85c5c6651ef4d6d2f83567e50ab6d69 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 8 Nov 2023 10:22:44 +0000 Subject: [PATCH 024/192] Local backup and restore --- .../BackupCoordinationKeeperMapTables.cpp | 23 + .../BackupCoordinationKeeperMapTables.h | 23 + src/Backups/BackupCoordinationLocal.cpp | 12 + src/Backups/BackupCoordinationLocal.h | 14 + src/Backups/BackupCoordinationRemote.cpp | 13 + src/Backups/BackupCoordinationRemote.h | 13 + src/Backups/BackupsWorker.cpp | 11 +- src/Backups/IBackupCoordination.h | 6 + src/Backups/IRestoreCoordination.h | 4 + src/Backups/RestoreCoordinationLocal.cpp | 6 + src/Backups/RestoreCoordinationLocal.h | 5 + src/Backups/RestoreCoordinationRemote.cpp | 5 + src/Backups/RestoreCoordinationRemote.h | 4 + src/Backups/WithRetries.cpp | 20 + src/Backups/WithRetries.h | 5 + src/Core/Settings.h | 1 + src/Storages/StorageKeeperMap.cpp | 420 +++++++++++++++--- src/Storages/StorageKeeperMap.h | 22 +- 18 files changed, 527 insertions(+), 80 deletions(-) create mode 100644 src/Backups/BackupCoordinationKeeperMapTables.cpp create mode 100644 src/Backups/BackupCoordinationKeeperMapTables.h diff --git a/src/Backups/BackupCoordinationKeeperMapTables.cpp b/src/Backups/BackupCoordinationKeeperMapTables.cpp new file mode 100644 index 00000000000..50561560dd5 --- /dev/null +++ b/src/Backups/BackupCoordinationKeeperMapTables.cpp @@ -0,0 +1,23 @@ +#include + +namespace DB +{ + +void BackupCoordinationKeeperMapTables::addTable(const std::string & table_zookeeper_root_path, const std::string & table_id, const std::string & data_path_in_backup) +{ + if (auto it = tables_with_info.find(table_zookeeper_root_path); it != tables_with_info.end()) + { + if (table_id > it->second.table_id) + it->second = KeeperMapTableInfo{table_id, data_path_in_backup}; + return; + } + + tables_with_info.emplace(table_zookeeper_root_path, KeeperMapTableInfo{table_id, data_path_in_backup}); +} + +std::string BackupCoordinationKeeperMapTables::getDataPath(const std::string & table_zookeeper_root_path) const +{ + return tables_with_info.at(table_zookeeper_root_path).data_path_in_backup; +} + +} diff --git a/src/Backups/BackupCoordinationKeeperMapTables.h b/src/Backups/BackupCoordinationKeeperMapTables.h new file mode 100644 index 00000000000..28894bb9c6e --- /dev/null +++ b/src/Backups/BackupCoordinationKeeperMapTables.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include + +namespace DB +{ + +struct BackupCoordinationKeeperMapTables +{ + void addTable(const std::string & table_zookeeper_root_path, const std::string & table_id, const std::string & data_path_in_backup); + std::string getDataPath(const std::string & table_zookeeper_root_path) const; +private: + struct KeeperMapTableInfo + { + std::string table_id; + std::string data_path_in_backup; + }; + + std::unordered_map tables_with_info; +}; + +} diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp index 27e0f173cf3..fb91bae2303 100644 --- a/src/Backups/BackupCoordinationLocal.cpp +++ b/src/Backups/BackupCoordinationLocal.cpp @@ -97,6 +97,18 @@ Strings BackupCoordinationLocal::getReplicatedSQLObjectsDirs(const String & load return replicated_sql_objects.getDirectories(loader_zk_path, object_type, ""); } +void BackupCoordinationLocal::addKeeperMapTable(const String & table_zookeeper_root_path, const String & table_id, const String & data_path_in_backup) +{ + std::lock_guard lock(keeper_map_tables_mutex); + keeper_map_tables.addTable(table_zookeeper_root_path, table_id, data_path_in_backup); +} + +String BackupCoordinationLocal::getKeeperMapDataPath(const String & table_zookeeper_root_path) const +{ + std::lock_guard lock(keeper_map_tables_mutex); + return keeper_map_tables.getDataPath(table_zookeeper_root_path); +} + void BackupCoordinationLocal::addFileInfos(BackupFileInfos && file_infos_) { diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h index 60fcc014720..1fecf30c51c 100644 --- a/src/Backups/BackupCoordinationLocal.h +++ b/src/Backups/BackupCoordinationLocal.h @@ -6,6 +6,8 @@ #include #include #include +#include "Backups/BackupCoordinationKeeperMapTables.h" +#include #include #include @@ -44,6 +46,9 @@ public: void addReplicatedSQLObjectsDir(const String & loader_zk_path, UserDefinedSQLObjectType object_type, const String & dir_path) override; Strings getReplicatedSQLObjectsDirs(const String & loader_zk_path, UserDefinedSQLObjectType object_type) const override; + void addKeeperMapTable(const String & table_zookeeper_root_path, const String & table_id, const String & data_path_in_backup) override; + String getKeeperMapDataPath(const String & table_zookeeper_root_path) const override; + void addFileInfos(BackupFileInfos && file_infos) override; BackupFileInfos getFileInfos() const override; BackupFileInfos getFileInfosForAllHosts() const override; @@ -58,13 +63,22 @@ private: BackupCoordinationReplicatedAccess TSA_GUARDED_BY(replicated_access_mutex) replicated_access; BackupCoordinationReplicatedSQLObjects TSA_GUARDED_BY(replicated_sql_objects_mutex) replicated_sql_objects; BackupCoordinationFileInfos TSA_GUARDED_BY(file_infos_mutex) file_infos; + BackupCoordinationKeeperMapTables keeper_map_tables TSA_GUARDED_BY(keeper_map_tables_mutex); std::unordered_set TSA_GUARDED_BY(writing_files_mutex) writing_files; + + struct KeeperMapTableInfo + { + String table_id; + String data_path_in_backup; + }; + mutable std::mutex replicated_tables_mutex; mutable std::mutex replicated_access_mutex; mutable std::mutex replicated_sql_objects_mutex; mutable std::mutex file_infos_mutex; mutable std::mutex writing_files_mutex; + mutable std::mutex keeper_map_tables_mutex; }; } diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp index e5fcbf26781..72fc2509089 100644 --- a/src/Backups/BackupCoordinationRemote.cpp +++ b/src/Backups/BackupCoordinationRemote.cpp @@ -666,6 +666,19 @@ void BackupCoordinationRemote::prepareReplicatedSQLObjects() const replicated_sql_objects->addDirectory(std::move(directory)); } +void BackupCoordinationRemote::addKeeperMapTable(const String & table_zookeeper_root_path, const String & table_id, const String & data_path_in_backup) +{ + std::lock_guard lock(keeper_map_tables_mutex); + keeper_map_tables.addTable(table_zookeeper_root_path, table_id, data_path_in_backup); +} + +String BackupCoordinationRemote::getKeeperMapDataPath(const String & table_zookeeper_root_path) const +{ + std::lock_guard lock(keeper_map_tables_mutex); + return keeper_map_tables.getDataPath(table_zookeeper_root_path); +} + + void BackupCoordinationRemote::addFileInfos(BackupFileInfos && file_infos_) { { diff --git a/src/Backups/BackupCoordinationRemote.h b/src/Backups/BackupCoordinationRemote.h index c1c7a40fc44..28c24c574a6 100644 --- a/src/Backups/BackupCoordinationRemote.h +++ b/src/Backups/BackupCoordinationRemote.h @@ -7,6 +7,7 @@ #include #include #include +#include "Backups/BackupCoordinationKeeperMapTables.h" namespace DB @@ -63,6 +64,9 @@ public: void addReplicatedSQLObjectsDir(const String & loader_zk_path, UserDefinedSQLObjectType object_type, const String & dir_path) override; Strings getReplicatedSQLObjectsDirs(const String & loader_zk_path, UserDefinedSQLObjectType object_type) const override; + void addKeeperMapTable(const String & table_zookeeper_root_path, const String & table_id, const String & data_path_in_backup) override; + String getKeeperMapDataPath(const String & table_zookeeper_root_path) const override; + void addFileInfos(BackupFileInfos && file_infos) override; BackupFileInfos getFileInfos() const override; BackupFileInfos getFileInfosForAllHosts() const override; @@ -108,12 +112,21 @@ private: mutable std::optional TSA_GUARDED_BY(file_infos_mutex) file_infos; std::unordered_set TSA_GUARDED_BY(writing_files_mutex) writing_files; + struct KeeperMapTableInfo + { + String table_id; + String data_path_in_backup; + }; + + mutable BackupCoordinationKeeperMapTables keeper_map_tables TSA_GUARDED_BY(keeper_map_tables_mutex); + mutable std::mutex zookeeper_mutex; mutable std::mutex replicated_tables_mutex; mutable std::mutex replicated_access_mutex; mutable std::mutex replicated_sql_objects_mutex; mutable std::mutex file_infos_mutex; mutable std::mutex writing_files_mutex; + mutable std::mutex keeper_map_tables_mutex; }; } diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index da814dcbc08..b19135c5cba 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -58,16 +58,7 @@ namespace auto get_zookeeper = [global_context = context->getGlobalContext()] { return global_context->getZooKeeper(); }; - BackupCoordinationRemote::BackupKeeperSettings keeper_settings - { - .keeper_max_retries = context->getSettingsRef().backup_restore_keeper_max_retries, - .keeper_retry_initial_backoff_ms = context->getSettingsRef().backup_restore_keeper_retry_initial_backoff_ms, - .keeper_retry_max_backoff_ms = context->getSettingsRef().backup_restore_keeper_retry_max_backoff_ms, - .batch_size_for_keeper_multiread = context->getSettingsRef().backup_restore_batch_size_for_keeper_multiread, - .keeper_fault_injection_probability = context->getSettingsRef().backup_restore_keeper_fault_injection_probability, - .keeper_fault_injection_seed = context->getSettingsRef().backup_restore_keeper_fault_injection_seed, - .keeper_value_max_size = context->getSettingsRef().backup_restore_keeper_value_max_size, - }; + BackupCoordinationRemote::BackupKeeperSettings keeper_settings = WithRetries::KeeperSettings::fromContext(context); auto all_hosts = BackupSettings::Util::filterHostIDs( backup_settings.cluster_host_ids, backup_settings.shard_num, backup_settings.replica_num); diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h index 75d9202374b..f80b5dee883 100644 --- a/src/Backups/IBackupCoordination.h +++ b/src/Backups/IBackupCoordination.h @@ -56,6 +56,12 @@ public: /// Returns all mutations of a replicated table which are not finished for some data parts added by addReplicatedPartNames(). virtual std::vector getReplicatedMutations(const String & table_shared_id, const String & replica_name) const = 0; + /// Adds information about KeeperMap tables + virtual void addKeeperMapTable(const String & table_zookeeper_root_path, const String & table_id, const String & data_path_in_backup) = 0; + + /// KeeperMap tables use shared storage without local data so only one table should backup the data + virtual String getKeeperMapDataPath(const String & table_zookeeper_root_path) const = 0; + /// Adds a data path in backup for a replicated table. /// Multiple replicas of the replicated table call this function and then all the added paths can be returned by call of the function /// getReplicatedDataPaths(). diff --git a/src/Backups/IRestoreCoordination.h b/src/Backups/IRestoreCoordination.h index fd6f014c326..489292cb88f 100644 --- a/src/Backups/IRestoreCoordination.h +++ b/src/Backups/IRestoreCoordination.h @@ -41,6 +41,10 @@ public: /// The function returns false if user-defined function at a specified zk path are being already restored by another replica. virtual bool acquireReplicatedSQLObjects(const String & loader_zk_path, UserDefinedSQLObjectType object_type) = 0; + /// Sets that this table is going to restore data into Keeper for all KeeperMap tables defined on root_zk_path. + /// The function returns false if data for this specific root path is already being restored by another table. + virtual bool acquireInsertingDataForKeeperMap(const String & root_zk_path) = 0; + /// Generates a new UUID for a table. The same UUID must be used for a replicated table on each replica, /// (because otherwise the macro "{uuid}" in the ZooKeeper path will not work correctly). virtual void generateUUIDForTable(ASTCreateQuery & create_query) = 0; diff --git a/src/Backups/RestoreCoordinationLocal.cpp b/src/Backups/RestoreCoordinationLocal.cpp index 1bd2f8e2ed1..d32625c2c51 100644 --- a/src/Backups/RestoreCoordinationLocal.cpp +++ b/src/Backups/RestoreCoordinationLocal.cpp @@ -52,6 +52,12 @@ bool RestoreCoordinationLocal::acquireReplicatedSQLObjects(const String &, UserD return true; } +bool RestoreCoordinationLocal::acquireInsertingDataForKeeperMap(const String & root_zk_path) +{ + std::lock_guard lock{mutex}; + return acquired_data_in_keeper_map_tables.emplace(root_zk_path).second; +} + void RestoreCoordinationLocal::generateUUIDForTable(ASTCreateQuery & create_query) { String query_str = serializeAST(create_query); diff --git a/src/Backups/RestoreCoordinationLocal.h b/src/Backups/RestoreCoordinationLocal.h index 339b754fca5..93fbdb79d9f 100644 --- a/src/Backups/RestoreCoordinationLocal.h +++ b/src/Backups/RestoreCoordinationLocal.h @@ -40,6 +40,10 @@ public: /// The function returns false if user-defined function at a specified zk path are being already restored by another replica. bool acquireReplicatedSQLObjects(const String & loader_zk_path, UserDefinedSQLObjectType object_type) override; + /// Sets that this table is going to restore data into Keeper for all KeeperMap tables defined on root_zk_path. + /// The function returns false if data for this specific root path is already being restored by another table. + bool acquireInsertingDataForKeeperMap(const String & root_zk_path) override; + /// Generates a new UUID for a table. The same UUID must be used for a replicated table on each replica, /// (because otherwise the macro "{uuid}" in the ZooKeeper path will not work correctly). void generateUUIDForTable(ASTCreateQuery & create_query) override; @@ -52,6 +56,7 @@ private: std::set> acquired_tables_in_replicated_databases; std::unordered_set acquired_data_in_replicated_tables; std::unordered_map create_query_uuids; + std::unordered_set acquired_data_in_keeper_map_tables; mutable std::mutex mutex; }; diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp index c71466ad8f4..7e059b8d9cc 100644 --- a/src/Backups/RestoreCoordinationRemote.cpp +++ b/src/Backups/RestoreCoordinationRemote.cpp @@ -234,6 +234,11 @@ bool RestoreCoordinationRemote::acquireReplicatedSQLObjects(const String & loade return result; } +bool RestoreCoordinationRemote::acquireInsertingDataForKeeperMap(const String & /*root_zk_path*/) +{ + return true; +} + void RestoreCoordinationRemote::generateUUIDForTable(ASTCreateQuery & create_query) { String query_str = serializeAST(create_query); diff --git a/src/Backups/RestoreCoordinationRemote.h b/src/Backups/RestoreCoordinationRemote.h index 22d0c0ed6df..7d3ae4ceec9 100644 --- a/src/Backups/RestoreCoordinationRemote.h +++ b/src/Backups/RestoreCoordinationRemote.h @@ -46,6 +46,10 @@ public: /// The function returns false if user-defined function at a specified zk path are being already restored by another replica. bool acquireReplicatedSQLObjects(const String & loader_zk_path, UserDefinedSQLObjectType object_type) override; + /// Sets that this table is going to restore data into Keeper for all KeeperMap tables defined on root_zk_path. + /// The function returns false if data for this specific root path is already being restored by another table. + bool acquireInsertingDataForKeeperMap(const String & root_zk_path) override; + /// Generates a new UUID for a table. The same UUID must be used for a replicated table on each replica, /// (because otherwise the macro "{uuid}" in the ZooKeeper path will not work correctly). void generateUUIDForTable(ASTCreateQuery & create_query) override; diff --git a/src/Backups/WithRetries.cpp b/src/Backups/WithRetries.cpp index 0893c65d8fd..d1612a7da4f 100644 --- a/src/Backups/WithRetries.cpp +++ b/src/Backups/WithRetries.cpp @@ -5,6 +5,21 @@ namespace DB { +WithRetries::KeeperSettings WithRetries::KeeperSettings::fromContext(ContextPtr context) +{ + return + { + .keeper_max_retries = context->getSettingsRef().backup_restore_keeper_max_retries, + .keeper_retry_initial_backoff_ms = context->getSettingsRef().backup_restore_keeper_retry_initial_backoff_ms, + .keeper_retry_max_backoff_ms = context->getSettingsRef().backup_restore_keeper_retry_max_backoff_ms, + .batch_size_for_keeper_multiread = context->getSettingsRef().backup_restore_batch_size_for_keeper_multiread, + .keeper_fault_injection_probability = context->getSettingsRef().backup_restore_keeper_fault_injection_probability, + .keeper_fault_injection_seed = context->getSettingsRef().backup_restore_keeper_fault_injection_seed, + .keeper_value_max_size = context->getSettingsRef().backup_restore_keeper_value_max_size, + .batch_size_for_keeper_multi = context->getSettingsRef().backup_restore_batch_size_for_keeper_multi, + }; +} + WithRetries::WithRetries(Poco::Logger * log_, zkutil::GetZooKeeper get_zookeeper_, const KeeperSettings & settings_, RenewerCallback callback_) : log(log_) , get_zookeeper(get_zookeeper_) @@ -42,6 +57,11 @@ void WithRetries::renewZooKeeper(FaultyKeeper my_faulty_zookeeper) const } } +const WithRetries::KeeperSettings & WithRetries::getKeeperSettings() const +{ + return settings; +} + WithRetries::FaultyKeeper WithRetries::getFaultyZooKeeper() const { /// We need to create new instance of ZooKeeperWithFaultInjection each time a copy a pointer to ZooKeeper client there diff --git a/src/Backups/WithRetries.h b/src/Backups/WithRetries.h index 3955682be94..8f4a730e6a1 100644 --- a/src/Backups/WithRetries.h +++ b/src/Backups/WithRetries.h @@ -26,6 +26,9 @@ public: Float64 keeper_fault_injection_probability{0}; UInt64 keeper_fault_injection_seed{42}; UInt64 keeper_value_max_size{1048576}; + UInt64 batch_size_for_keeper_multi{1000}; + + static KeeperSettings fromContext(ContextPtr context); }; /// For simplicity a separate ZooKeeperRetriesInfo and a faulty [Zoo]Keeper client @@ -53,6 +56,8 @@ public: /// Used to re-establish new connection inside a retry loop. void renewZooKeeper(FaultyKeeper my_faulty_zookeeper) const; + + const KeeperSettings & getKeeperSettings() const; private: /// This will provide a special wrapper which is useful for testing FaultyKeeper getFaultyZooKeeper() const; diff --git a/src/Core/Settings.h b/src/Core/Settings.h index aa5c8569be6..bab9005a22c 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -465,6 +465,7 @@ class IColumn; M(UInt64, backup_restore_keeper_fault_injection_seed, 0, "0 - random seed, otherwise the setting value", 0) \ M(UInt64, backup_restore_keeper_value_max_size, 1048576, "Maximum size of data of a [Zoo]Keeper's node during backup", 0) \ M(UInt64, backup_restore_batch_size_for_keeper_multiread, 10000, "Maximum size of batch for multiread request to [Zoo]Keeper during backup or restore", 0) \ + M(UInt64, backup_restore_batch_size_for_keeper_multi, 1000, "Maximum size of batch for multi request to [Zoo]Keeper during backup or restore", 0) \ M(UInt64, max_backup_bandwidth, 0, "The maximum read speed in bytes per second for particular backup on server. Zero means unlimited.", 0) \ \ M(Bool, log_profile_events, true, "Log query performance statistics into the query_log, query_thread_log and query_views_log.", 0) \ diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index f98728c012e..33a97af53f1 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -13,6 +14,9 @@ #include #include +#include +#include + #include #include #include @@ -38,6 +42,16 @@ #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include #include @@ -54,6 +68,7 @@ namespace ErrorCodes extern const int KEEPER_EXCEPTION; extern const int LOGICAL_ERROR; extern const int LIMIT_EXCEEDED; + extern const int CANNOT_RESTORE_TABLE; } namespace @@ -296,13 +311,13 @@ StorageKeeperMap::StorageKeeperMap( const StorageInMemoryMetadata & metadata, bool attach, std::string_view primary_key_, - const std::string & root_path_, + const std::string & zk_root_path_, UInt64 keys_limit_) : IStorage(table_id) , WithContext(context_->getGlobalContext()) - , root_path(zkutil::extractZooKeeperPath(root_path_, false)) + , zk_root_path(zkutil::extractZooKeeperPath(zk_root_path_, false)) , primary_key(primary_key_) - , zookeeper_name(zkutil::extractZooKeeperName(root_path_)) + , zookeeper_name(zkutil::extractZooKeeperName(zk_root_path_)) , keys_limit(keys_limit_) , log(&Poco::Logger::get(fmt::format("StorageKeeperMap ({})", table_id.getNameForLogs()))) { @@ -320,10 +335,10 @@ StorageKeeperMap::StorageKeeperMap( << "primary key: " << formattedAST(metadata.getPrimaryKey().expression_list_ast) << "\n"; metadata_string = out.str(); - if (root_path.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "root_path should not be empty"); - if (!root_path.starts_with('/')) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "root_path should start with '/'"); + if (zk_root_path.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "zk_root_path should not be empty"); + if (!zk_root_path.starts_with('/')) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "zk_root_path should start with '/'"); auto config_keys_limit = context_->getConfigRef().getUInt64("keeper_map_keys_limit", 0); if (config_keys_limit != 0 && (keys_limit == 0 || keys_limit > config_keys_limit)) @@ -341,20 +356,20 @@ StorageKeeperMap::StorageKeeperMap( LOG_INFO(log, "Keys limit will be set to {}", keys_limit); } - auto root_path_fs = fs::path(path_prefix) / std::string_view{root_path}.substr(1); - root_path = root_path_fs.generic_string(); + auto zk_root_path_fs = fs::path(path_prefix) / std::string_view{zk_root_path}.substr(1); + zk_root_path = zk_root_path_fs.generic_string(); - data_path = root_path_fs / "data"; + zk_data_path = zk_root_path_fs / "data"; - auto metadata_path_fs = root_path_fs / "metadata"; - metadata_path = metadata_path_fs; - tables_path = metadata_path_fs / "tables"; + auto metadata_path_fs = zk_root_path_fs / "metadata"; + zk_metadata_path = metadata_path_fs; + zk_tables_path = metadata_path_fs / "tables"; auto table_unique_id = toString(table_id.uuid) + toString(ServerUUID::get()); - table_path = fs::path(tables_path) / table_unique_id; + zk_table_path = fs::path(zk_tables_path) / table_unique_id; - dropped_path = metadata_path_fs / "dropped"; - dropped_lock_path = fs::path(dropped_path) / "lock"; + zk_dropped_path = metadata_path_fs / "dropped"; + zk_dropped_lock_path = fs::path(zk_dropped_path) / "lock"; if (attach) { @@ -364,17 +379,17 @@ StorageKeeperMap::StorageKeeperMap( auto client = getClient(); - if (root_path != "/" && !client->exists(root_path)) + if (zk_root_path != "/" && !client->exists(zk_root_path)) { - LOG_TRACE(log, "Creating root path {}", root_path); - client->createAncestors(root_path); - client->createIfNotExists(root_path, ""); + LOG_TRACE(log, "Creating root path {}", zk_root_path); + client->createAncestors(zk_root_path); + client->createIfNotExists(zk_root_path, ""); } for (size_t i = 0; i < 1000; ++i) { std::string stored_metadata_string; - auto exists = client->tryGet(metadata_path, stored_metadata_string); + auto exists = client->tryGet(zk_metadata_path, stored_metadata_string); if (exists) { @@ -384,10 +399,10 @@ StorageKeeperMap::StorageKeeperMap( throw Exception( ErrorCodes::BAD_ARGUMENTS, "Path {} is already used but the stored table definition doesn't match. Stored metadata: {}", - root_path, + zk_root_path, stored_metadata_string); - auto code = client->tryCreate(table_path, "", zkutil::CreateMode::Persistent); + auto code = client->tryCreate(zk_table_path, "", zkutil::CreateMode::Persistent); // tables_path was removed with drop if (code == Coordination::Error::ZNONODE) @@ -397,16 +412,16 @@ StorageKeeperMap::StorageKeeperMap( } else if (code != Coordination::Error::ZOK) { - throw zkutil::KeeperException(code, "Failed to create table on path {} because a table with same UUID already exists", root_path); + throw zkutil::KeeperException(code, "Failed to create table on path {} because a table with same UUID already exists", zk_root_path); } return; } - if (client->exists(dropped_path)) + if (client->exists(zk_dropped_path)) { LOG_INFO(log, "Removing leftover nodes"); - auto code = client->tryCreate(dropped_lock_path, "", zkutil::CreateMode::Ephemeral); + auto code = client->tryCreate(zk_dropped_lock_path, "", zkutil::CreateMode::Ephemeral); if (code == Coordination::Error::ZNONODE) { @@ -419,11 +434,11 @@ StorageKeeperMap::StorageKeeperMap( } else if (code != Coordination::Error::ZOK) { - throw Coordination::Exception::fromPath(code, dropped_lock_path); + throw Coordination::Exception::fromPath(code, zk_dropped_lock_path); } else { - auto metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(dropped_lock_path, *client); + auto metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(zk_dropped_lock_path, *client); if (!dropTable(client, metadata_drop_lock)) continue; } @@ -431,17 +446,17 @@ StorageKeeperMap::StorageKeeperMap( Coordination::Requests create_requests { - zkutil::makeCreateRequest(metadata_path, metadata_string, zkutil::CreateMode::Persistent), - zkutil::makeCreateRequest(data_path, metadata_string, zkutil::CreateMode::Persistent), - zkutil::makeCreateRequest(tables_path, "", zkutil::CreateMode::Persistent), - zkutil::makeCreateRequest(table_path, "", zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest(zk_metadata_path, metadata_string, zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest(zk_data_path, metadata_string, zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest(zk_tables_path, "", zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest(zk_table_path, "", zkutil::CreateMode::Persistent), }; Coordination::Responses create_responses; auto code = client->tryMulti(create_requests, create_responses); if (code == Coordination::Error::ZNODEEXISTS) { - LOG_INFO(log, "It looks like a table on path {} was created by another server at the same moment, will retry", root_path); + LOG_INFO(log, "It looks like a table on path {} was created by another server at the same moment, will retry", zk_root_path); continue; } else if (code != Coordination::Error::ZOK) @@ -456,7 +471,7 @@ StorageKeeperMap::StorageKeeperMap( throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot create metadata for table, because it is removed concurrently or because " - "of wrong root_path ({})", root_path); + "of wrong zk_root_path ({})", zk_root_path); } @@ -519,7 +534,7 @@ Pipe StorageKeeperMap::read( auto client = getClient(); if (all_scan) - return process_keys(std::make_shared>(client->getChildren(data_path))); + return process_keys(std::make_shared>(client->getChildren(zk_data_path))); return process_keys(std::move(filtered_keys)); } @@ -534,19 +549,19 @@ void StorageKeeperMap::truncate(const ASTPtr &, const StorageMetadataPtr &, Cont { checkTable(); auto client = getClient(); - client->tryRemoveChildrenRecursive(data_path, true); + client->tryRemoveChildrenRecursive(zk_data_path, true); } bool StorageKeeperMap::dropTable(zkutil::ZooKeeperPtr zookeeper, const zkutil::EphemeralNodeHolder::Ptr & metadata_drop_lock) { - zookeeper->removeChildrenRecursive(data_path); + zookeeper->removeChildrenRecursive(zk_data_path); bool completely_removed = false; Coordination::Requests ops; ops.emplace_back(zkutil::makeRemoveRequest(metadata_drop_lock->getPath(), -1)); - ops.emplace_back(zkutil::makeRemoveRequest(dropped_path, -1)); - ops.emplace_back(zkutil::makeRemoveRequest(data_path, -1)); - ops.emplace_back(zkutil::makeRemoveRequest(metadata_path, -1)); + ops.emplace_back(zkutil::makeRemoveRequest(zk_dropped_path, -1)); + ops.emplace_back(zkutil::makeRemoveRequest(zk_data_path, -1)); + ops.emplace_back(zkutil::makeRemoveRequest(zk_metadata_path, -1)); Coordination::Responses responses; auto code = zookeeper->tryMulti(ops, responses); @@ -557,7 +572,7 @@ bool StorageKeeperMap::dropTable(zkutil::ZooKeeperPtr zookeeper, const zkutil::E { metadata_drop_lock->setAlreadyRemoved(); completely_removed = true; - LOG_INFO(log, "Metadata ({}) and data ({}) was successfully removed from ZooKeeper", metadata_path, data_path); + LOG_INFO(log, "Metadata ({}) and data ({}) was successfully removed from ZooKeeper", zk_metadata_path, zk_data_path); break; } case ZNONODE: @@ -578,25 +593,25 @@ void StorageKeeperMap::drop() auto client = getClient(); // we allow ZNONODE in case we got hardware error on previous drop - if (auto code = client->tryRemove(table_path); code == Coordination::Error::ZNOTEMPTY) + if (auto code = client->tryRemove(zk_table_path); code == Coordination::Error::ZNOTEMPTY) { throw zkutil::KeeperException( - code, "{} contains children which shouldn't happen. Please DETACH the table if you want to delete it", table_path); + code, "{} contains children which shouldn't happen. Please DETACH the table if you want to delete it", zk_table_path); } std::vector children; // if the tables_path is not found, some other table removed it // if there are children, some other tables are still using this path as storage - if (auto code = client->tryGetChildren(tables_path, children); + if (auto code = client->tryGetChildren(zk_tables_path, children); code != Coordination::Error::ZOK || !children.empty()) return; Coordination::Requests ops; Coordination::Responses responses; - ops.emplace_back(zkutil::makeRemoveRequest(tables_path, -1)); - ops.emplace_back(zkutil::makeCreateRequest(dropped_path, "", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest(dropped_lock_path, "", zkutil::CreateMode::Ephemeral)); + ops.emplace_back(zkutil::makeRemoveRequest(zk_tables_path, -1)); + ops.emplace_back(zkutil::makeCreateRequest(zk_dropped_path, "", zkutil::CreateMode::Persistent)); + ops.emplace_back(zkutil::makeCreateRequest(zk_dropped_lock_path, "", zkutil::CreateMode::Ephemeral)); auto code = client->tryMulti(ops, responses); @@ -613,7 +628,7 @@ void StorageKeeperMap::drop() else if (code != Coordination::Error::ZOK) zkutil::KeeperMultiException::check(code, ops, responses); - auto metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(dropped_lock_path, *client); + auto metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(zk_dropped_lock_path, *client); dropTable(client, metadata_drop_lock); } @@ -623,6 +638,285 @@ NamesAndTypesList StorageKeeperMap::getVirtuals() const {std::string{version_column_name}, std::make_shared()}}; } +namespace +{ + +constexpr std::string_view backup_data_filename = "data.bin"; +constexpr std::string_view backup_data_location_filename = "data_location.bin"; + +class KeeperMapBackup : public IBackupEntriesLazyBatch, boost::noncopyable +{ +public: + KeeperMapBackup( + const std::string & data_zookeeper_path_, + const std::string & data_path_in_backup, + const DiskPtr & temp_disk_, + UInt64 max_compress_block_size_, + std::shared_ptr with_retries_) + : data_zookeeper_path(data_zookeeper_path_) + , temp_disk(temp_disk_) + , max_compress_block_size(max_compress_block_size_) + , with_retries(std::move(with_retries_)) + { + file_path = fs::path(data_path_in_backup) / backup_data_filename; + } + +private: + size_t getSize() const override + { + return 1; + } + + const String & getName(size_t i) const override + { + chassert(i == 0); + return file_path; + } + + BackupEntries generate() override + { + temp_dir_owner.emplace(temp_disk); + fs::path temp_dir = temp_dir_owner->getRelativePath(); + temp_disk->createDirectories(temp_dir); + + auto data_file_path = temp_dir / fs::path{file_path}.filename(); + auto data_out_compressed = temp_disk->writeFile(data_file_path); + auto data_out = std::make_unique(*data_out_compressed, CompressionCodecFactory::instance().getDefaultCodec(), max_compress_block_size); + std::vector data_children; + { + auto holder = with_retries->createRetriesControlHolder("getKeeperMapDataKeys"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper]() + { + with_retries->renewZooKeeper(zk); + data_children = zk->getChildren(data_zookeeper_path); + }); + } + LOG_INFO(&Poco::Logger::get("BACKUPER"), "Got {} children", data_children.size()); + + const auto write_rows = [&](std::span keys) + { + std::vector keys_full_path; + keys_full_path.reserve(data_children.size()); + + for (const auto & key : data_children) + keys_full_path.push_back(data_zookeeper_path / key); + + zkutil::ZooKeeper::MultiGetResponse data; + auto holder = with_retries->createRetriesControlHolder("getKeeperMapDataKeys"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper] + { + with_retries->renewZooKeeper(zk); + data = zk->get(keys_full_path); + }); + + for (size_t i = 0; i < keys.size(); ++i) + { + auto & child_data = data[i]; + if (child_data.error != Coordination::Error::ZOK) + continue; + + writeStringBinary(keys[i], *data_out); + writeStringBinary(child_data.data, *data_out); + } + }; + + auto max_multiread_size = with_retries->getKeeperSettings().batch_size_for_keeper_multiread; + + auto keys_it = data_children.begin(); + while (keys_it != data_children.end()) + { + auto step = std::min(static_cast(std::distance(keys_it, data_children.end())), max_multiread_size); + write_rows(std::span{keys_it, keys_it + step}); + keys_it = keys_it + step; + } + + data_out->finalize(); + data_out.reset(); + data_out_compressed->finalize(); + data_out_compressed.reset(); + + return {{file_path, std::make_shared(temp_disk, data_file_path)}}; + } + + fs::path data_zookeeper_path; + DiskPtr temp_disk; + std::optional temp_dir_owner; + UInt64 max_compress_block_size; + String file_path; + std::shared_ptr with_retries; +}; +} + +void StorageKeeperMap::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /*partitions*/) +{ + auto table_id = toString(getStorageID().uuid); + + std::cout << "Backing up for path " << zk_root_path << " table id " << table_id << std::endl; + auto coordination = backup_entries_collector.getBackupCoordination(); + coordination->addKeeperMapTable(zk_root_path, table_id, data_path_in_backup); + + /// This task will be executed after all tables have registered their root zk path and the coordination is ready to + /// assign each path to a single table only. + auto post_collecting_task = [my_table_id = std::move(table_id), coordination, &backup_entries_collector, my_data_path_in_backup = data_path_in_backup, this] + { + auto path_with_data = coordination->getKeeperMapDataPath(zk_root_path); + if (path_with_data == my_data_path_in_backup) + { + std::cout << "Will be backing up data for path " << zk_root_path << " table id " << my_table_id << std::endl; + + auto temp_disk = backup_entries_collector.getContext()->getGlobalTemporaryVolume()->getDisk(0); + auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef().max_compress_block_size; + + auto with_retries = std::make_shared + ( + &Poco::Logger::get(fmt::format("StorageKeeperMapBackup ({})", getStorageID().getNameForLogs())), + [&] { return getClient(); }, + WithRetries::KeeperSettings::fromContext(backup_entries_collector.getContext()), + [](WithRetries::FaultyKeeper &) {} + ); + + backup_entries_collector.addBackupEntries( + std::make_shared(this->zk_data_path, path_with_data, temp_disk, max_compress_block_size, std::move(with_retries)) + ->getBackupEntries()); + return; + } + + std::cout << "Not backing up data for path " << zk_root_path << " table id " << my_table_id << " writing only path with data " << path_with_data << std::endl; + auto file_path = fs::path(my_data_path_in_backup) / backup_data_location_filename; + backup_entries_collector.addBackupEntries({{file_path, std::make_shared(path_with_data)}}); + }; + + backup_entries_collector.addPostTask(post_collecting_task); +} + +void StorageKeeperMap::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /*partitions*/) +{ + auto backup = restorer.getBackup(); + if (!backup->hasFiles(data_path_in_backup)) + return; + + if (!restorer.getRestoreCoordination()->acquireInsertingDataForKeeperMap(zk_root_path)) + { + /// Other table is already restoring the data for this Keeper path. + /// Tables defined on the same path share data + return; + } + + auto with_retries = std::make_shared + ( + &Poco::Logger::get(fmt::format("StorageKeeperMapRestore ({})", getStorageID().getNameForLogs())), + [&] { return getClient(); }, + WithRetries::KeeperSettings::fromContext(restorer.getContext()), + [](WithRetries::FaultyKeeper &) {} + ); + + bool allow_non_empty_tables = restorer.isNonEmptyTableAllowed(); + if (!allow_non_empty_tables) + { + Coordination::Stat data_stats; + + auto holder = with_retries->createRetriesControlHolder("checkKeeperMapData"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper]() + { + with_retries->renewZooKeeper(zk); + zk->get(zk_data_path, &data_stats); + }); + + if (data_stats.numChildren != 0) + RestorerFromBackup::throwTableIsNotEmpty(getStorageID()); + } + + /// TODO: Should we backup and verify the table structure? + + //auto temp_disk = restorer.getContext()->getGlobalTemporaryVolume()->getDisk(0); + /// only 1 table should restore data for a single path + restorer.addDataRestoreTask( + [storage = std::static_pointer_cast(shared_from_this()), backup, data_path_in_backup, with_retries, allow_non_empty_tables] + { storage->restoreDataImpl(backup, data_path_in_backup, with_retries, allow_non_empty_tables); }); +} + +void StorageKeeperMap::restoreDataImpl(const BackupPtr & backup, const String & data_path_in_backup, std::shared_ptr with_retries, bool allow_non_empty_tables) +{ + auto table_id = toString(getStorageID().uuid); + + std::cout << "Restoring into " << zk_root_path << " table id " << table_id << std::endl; + + fs::path data_path_in_backup_fs = data_path_in_backup; + + String data_file = data_path_in_backup_fs / backup_data_filename; + + if (!backup->fileExists(data_file)) + { + String data_location_file = data_path_in_backup_fs / "data_location.bin"; + if (!backup->fileExists(data_location_file)) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Files {} or {} in backup are required to restore table", data_file, data_location_file); + + auto in = backup->readFile(data_location_file); + readStringUntilEOF(data_file, *in); + + data_file = fs::path(data_file) / backup_data_filename; + + if (!backup->fileExists(data_file)) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File {} in backup is required to restore table", data_file); + } + + /// should we store locally in temp file? + auto in = backup->readFile(data_file); + CompressedReadBuffer compressed_in{*in}; + fs::path data_path_fs(zk_data_path); + + auto max_multi_size = with_retries->getKeeperSettings().batch_size_for_keeper_multi; + + Coordination::Requests create_requests; + const auto flush_create_requests = [&] + { + auto holder = with_retries->createRetriesControlHolder("addKeeperMapData"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper]() + { + with_retries->renewZooKeeper(zk); + zk->multi(create_requests); + }); + }; + + while (!in->eof()) + { + std::string key; + std::string value; + readStringBinary(key, compressed_in); + readStringBinary(value, compressed_in); + + /// if a table can be non empty we can have conflicting keys so we need to do single create for each row + if (allow_non_empty_tables) + { + auto holder = with_retries->createRetriesControlHolder("addKeeperMapData"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper]() + { + with_retries->renewZooKeeper(zk); + zk->tryCreate(data_path_fs / key, value, zkutil::CreateMode::Persistent); + }); + } + /// otherwise we can do multi requests + else + { + create_requests.push_back(zkutil::makeCreateRequest(data_path_fs / key, value, zkutil::CreateMode::Persistent)); + + if (create_requests.size() == max_multi_size) + { + flush_create_requests(); + create_requests.clear(); + } + } + } + + if (!create_requests.empty()) + flush_create_requests(); +} + zkutil::ZooKeeperPtr StorageKeeperMap::getClient() const { std::lock_guard lock{zookeeper_mutex}; @@ -634,7 +928,7 @@ zkutil::ZooKeeperPtr StorageKeeperMap::getClient() const else zookeeper_client = getContext()->getAuxiliaryZooKeeper(zookeeper_name); - zookeeper_client->sync(root_path); + zookeeper_client->sync(zk_root_path); } return zookeeper_client; @@ -642,12 +936,12 @@ zkutil::ZooKeeperPtr StorageKeeperMap::getClient() const const std::string & StorageKeeperMap::dataPath() const { - return data_path; + return zk_data_path; } std::string StorageKeeperMap::fullPathForKey(const std::string_view key) const { - return fs::path(data_path) / key; + return fs::path(zk_data_path) / key; } UInt64 StorageKeeperMap::keysLimit() const @@ -668,7 +962,7 @@ std::optional StorageKeeperMap::isTableValid() const auto client = getClient(); Coordination::Stat metadata_stat; - auto stored_metadata_string = client->get(metadata_path, &metadata_stat); + auto stored_metadata_string = client->get(zk_metadata_path, &metadata_stat); if (metadata_stat.numChildren == 0) { @@ -681,7 +975,7 @@ std::optional StorageKeeperMap::isTableValid() const LOG_ERROR( log, "Table definition does not match to the one stored in the path {}. Stored definition: {}", - root_path, + zk_root_path, stored_metadata_string); table_is_valid = false; return; @@ -689,9 +983,9 @@ std::optional StorageKeeperMap::isTableValid() const // validate all metadata and data nodes are present Coordination::Requests requests; - requests.push_back(zkutil::makeCheckRequest(table_path, -1)); - requests.push_back(zkutil::makeCheckRequest(data_path, -1)); - requests.push_back(zkutil::makeCheckRequest(dropped_path, -1)); + requests.push_back(zkutil::makeCheckRequest(zk_table_path, -1)); + requests.push_back(zkutil::makeCheckRequest(zk_data_path, -1)); + requests.push_back(zkutil::makeCheckRequest(zk_dropped_path, -1)); Coordination::Responses responses; client->tryMulti(requests, responses); @@ -699,19 +993,19 @@ std::optional StorageKeeperMap::isTableValid() const table_is_valid = false; if (responses[0]->error != Coordination::Error::ZOK) { - LOG_ERROR(log, "Table node ({}) is missing", table_path); + LOG_ERROR(log, "Table node ({}) is missing", zk_table_path); return; } if (responses[1]->error != Coordination::Error::ZOK) { - LOG_ERROR(log, "Data node ({}) is missing", data_path); + LOG_ERROR(log, "Data node ({}) is missing", zk_data_path); return; } if (responses[2]->error == Coordination::Error::ZOK) { - LOG_ERROR(log, "Tables with root node {} are being dropped", root_path); + LOG_ERROR(log, "Tables with root node {} are being dropped", zk_root_path); return; } @@ -962,11 +1256,11 @@ StoragePtr create(const StorageFactory::Arguments & args) throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Storage KeeperMap requires 1-3 arguments:\n" - "root_path: path in the Keeper where the values will be stored (required)\n" + "zk_root_path: path in the Keeper where the values will be stored (required)\n" "keys_limit: number of keys allowed to be stored, 0 is no limit (default: 0)"); - const auto root_path_node = evaluateConstantExpressionAsLiteral(engine_args[0], args.getLocalContext()); - auto root_path = checkAndGetLiteralArgument(root_path_node, "root_path"); + const auto zk_root_path_node = evaluateConstantExpressionAsLiteral(engine_args[0], args.getLocalContext()); + auto zk_root_path = checkAndGetLiteralArgument(zk_root_path_node, "zk_root_path"); UInt64 keys_limit = 0; if (engine_args.size() > 1) @@ -985,7 +1279,7 @@ StoragePtr create(const StorageFactory::Arguments & args) throw Exception(ErrorCodes::BAD_ARGUMENTS, "StorageKeeperMap requires one column in primary key"); return std::make_shared( - args.getContext(), args.table_id, metadata, args.query.attach, primary_key_names[0], root_path, keys_limit); + args.getContext(), args.table_id, metadata, args.query.attach, primary_key_names[0], zk_root_path, keys_limit); } } diff --git a/src/Storages/StorageKeeperMap.h b/src/Storages/StorageKeeperMap.h index ad7b719e972..94b02ca0242 100644 --- a/src/Storages/StorageKeeperMap.h +++ b/src/Storages/StorageKeeperMap.h @@ -10,6 +10,9 @@ #include #include +#include +#include + #include namespace DB @@ -72,6 +75,9 @@ public: } bool supportsDelete() const override { return true; } + void backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) override; + void restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) override; + zkutil::ZooKeeperPtr getClient() const; const std::string & dataPath() const; std::string fullPathForKey(std::string_view key) const; @@ -114,18 +120,20 @@ private: std::optional isTableValid() const; - std::string root_path; + void restoreDataImpl(const BackupPtr & backup, const String & data_path_in_backup, std::shared_ptr with_retries, bool allow_non_empty_tables); + + std::string zk_root_path; std::string primary_key; - std::string data_path; + std::string zk_data_path; - std::string metadata_path; + std::string zk_metadata_path; - std::string tables_path; - std::string table_path; + std::string zk_tables_path; + std::string zk_table_path; - std::string dropped_path; - std::string dropped_lock_path; + std::string zk_dropped_path; + std::string zk_dropped_lock_path; std::string zookeeper_name; From 18a5eeec38663bca5a894bb01f0cf84ad39e0b64 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 8 Nov 2023 13:14:09 +0000 Subject: [PATCH 025/192] Make on cluster backup/restore work --- .../BackupCoordinationKeeperMapTables.h | 4 +- src/Backups/BackupCoordinationLocal.h | 4 +- src/Backups/BackupCoordinationRemote.cpp | 65 ++++++++++++++++++- src/Backups/BackupCoordinationRemote.h | 5 +- src/Backups/RestoreCoordinationRemote.cpp | 39 ++++++++++- src/Storages/StorageKeeperMap.cpp | 8 +-- 6 files changed, 107 insertions(+), 18 deletions(-) diff --git a/src/Backups/BackupCoordinationKeeperMapTables.h b/src/Backups/BackupCoordinationKeeperMapTables.h index 28894bb9c6e..a642903cfae 100644 --- a/src/Backups/BackupCoordinationKeeperMapTables.h +++ b/src/Backups/BackupCoordinationKeeperMapTables.h @@ -10,13 +10,13 @@ struct BackupCoordinationKeeperMapTables { void addTable(const std::string & table_zookeeper_root_path, const std::string & table_id, const std::string & data_path_in_backup); std::string getDataPath(const std::string & table_zookeeper_root_path) const; -private: + struct KeeperMapTableInfo { std::string table_id; std::string data_path_in_backup; }; - +private: std::unordered_map tables_with_info; }; diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h index 1fecf30c51c..6f8e750697c 100644 --- a/src/Backups/BackupCoordinationLocal.h +++ b/src/Backups/BackupCoordinationLocal.h @@ -5,8 +5,8 @@ #include #include #include +#include #include -#include "Backups/BackupCoordinationKeeperMapTables.h" #include #include #include @@ -65,7 +65,7 @@ private: BackupCoordinationFileInfos TSA_GUARDED_BY(file_infos_mutex) file_infos; BackupCoordinationKeeperMapTables keeper_map_tables TSA_GUARDED_BY(keeper_map_tables_mutex); std::unordered_set TSA_GUARDED_BY(writing_files_mutex) writing_files; - + struct KeeperMapTableInfo { String table_id; diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp index 72fc2509089..309cbc8be6a 100644 --- a/src/Backups/BackupCoordinationRemote.cpp +++ b/src/Backups/BackupCoordinationRemote.cpp @@ -230,6 +230,7 @@ void BackupCoordinationRemote::createRootNodes() ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/repl_data_paths", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/repl_access", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/repl_sql_objects", "", zkutil::CreateMode::Persistent)); + ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/keeper_map_tables", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/file_infos", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/writing_files", "", zkutil::CreateMode::Persistent)); zk->tryMulti(ops, responses); @@ -668,14 +669,72 @@ void BackupCoordinationRemote::prepareReplicatedSQLObjects() const void BackupCoordinationRemote::addKeeperMapTable(const String & table_zookeeper_root_path, const String & table_id, const String & data_path_in_backup) { - std::lock_guard lock(keeper_map_tables_mutex); - keeper_map_tables.addTable(table_zookeeper_root_path, table_id, data_path_in_backup); + { + std::lock_guard lock{keeper_map_tables_mutex}; + if (keeper_map_tables) + throw Exception(ErrorCodes::LOGICAL_ERROR, "addKeeperMapTable() must not be called after preparing"); + } + + auto holder = with_retries.createRetriesControlHolder("addKeeperMapTable"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper]() + { + with_retries.renewZooKeeper(zk); + String path = zookeeper_path + "/keeper_map_tables/" + escapeForFileName(table_id); + zk->create(path, fmt::format("{}\n{}", table_zookeeper_root_path, data_path_in_backup), zkutil::CreateMode::Persistent); + }); +} + +void BackupCoordinationRemote::prepareKeeperMapTables() const +{ + if (keeper_map_tables) + return; + + std::vector> keeper_map_table_infos; + auto holder = with_retries.createRetriesControlHolder("prepareKeeperMapTables"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper]() + { + keeper_map_table_infos.clear(); + + with_retries.renewZooKeeper(zk); + + fs::path tables_path = fs::path(zookeeper_path) / "keeper_map_tables"; + + auto tables = zk->getChildren(tables_path); + keeper_map_table_infos.reserve(tables.size()); + + for (auto & table : tables) + table = tables_path / table; + + auto tables_info = zk->get(tables); + for (size_t i = 0; i < tables_info.size(); ++i) + { + const auto & table_info = tables_info[i]; + + if (table_info.error != Coordination::Error::ZOK) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Path in Keeper {} is unexpectedly missing", tables[i]); + + std::vector data; + boost::split(data, table_info.data, [](char c) { return c == '\n'; }); + keeper_map_table_infos.emplace_back( + std::move(data[0]), + BackupCoordinationKeeperMapTables::KeeperMapTableInfo{ + .table_id = fs::path(tables[i]).filename(), .data_path_in_backup = std::move(data[1])}); + } + }); + + keeper_map_tables.emplace(); + for (const auto & [zk_root_path, table_info] : keeper_map_table_infos) + keeper_map_tables->addTable(zk_root_path, table_info.table_id, table_info.data_path_in_backup); + } String BackupCoordinationRemote::getKeeperMapDataPath(const String & table_zookeeper_root_path) const { std::lock_guard lock(keeper_map_tables_mutex); - return keeper_map_tables.getDataPath(table_zookeeper_root_path); + prepareKeeperMapTables(); + return keeper_map_tables->getDataPath(table_zookeeper_root_path); } diff --git a/src/Backups/BackupCoordinationRemote.h b/src/Backups/BackupCoordinationRemote.h index 28c24c574a6..a0a9224bf71 100644 --- a/src/Backups/BackupCoordinationRemote.h +++ b/src/Backups/BackupCoordinationRemote.h @@ -5,9 +5,9 @@ #include #include #include +#include #include #include -#include "Backups/BackupCoordinationKeeperMapTables.h" namespace DB @@ -89,6 +89,7 @@ private: void prepareReplicatedTables() const TSA_REQUIRES(replicated_tables_mutex); void prepareReplicatedAccess() const TSA_REQUIRES(replicated_access_mutex); void prepareReplicatedSQLObjects() const TSA_REQUIRES(replicated_sql_objects_mutex); + void prepareKeeperMapTables() const TSA_REQUIRES(keeper_map_tables_mutex); void prepareFileInfos() const TSA_REQUIRES(file_infos_mutex); const String root_zookeeper_path; @@ -110,6 +111,7 @@ private: mutable std::optional TSA_GUARDED_BY(replicated_access_mutex) replicated_access; mutable std::optional TSA_GUARDED_BY(replicated_sql_objects_mutex) replicated_sql_objects; mutable std::optional TSA_GUARDED_BY(file_infos_mutex) file_infos; + mutable std::optional keeper_map_tables TSA_GUARDED_BY(keeper_map_tables_mutex); std::unordered_set TSA_GUARDED_BY(writing_files_mutex) writing_files; struct KeeperMapTableInfo @@ -118,7 +120,6 @@ private: String data_path_in_backup; }; - mutable BackupCoordinationKeeperMapTables keeper_map_tables TSA_GUARDED_BY(keeper_map_tables_mutex); mutable std::mutex zookeeper_mutex; mutable std::mutex replicated_tables_mutex; diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp index 7e059b8d9cc..12a67d2a55d 100644 --- a/src/Backups/RestoreCoordinationRemote.cpp +++ b/src/Backups/RestoreCoordinationRemote.cpp @@ -89,6 +89,7 @@ void RestoreCoordinationRemote::createRootNodes() ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/repl_tables_data_acquired", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/repl_access_storages_acquired", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/repl_sql_objects_acquired", "", zkutil::CreateMode::Persistent)); + ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/keeper_map_tables", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/table_uuids", "", zkutil::CreateMode::Persistent)); zk->tryMulti(ops, responses); }); @@ -234,9 +235,43 @@ bool RestoreCoordinationRemote::acquireReplicatedSQLObjects(const String & loade return result; } -bool RestoreCoordinationRemote::acquireInsertingDataForKeeperMap(const String & /*root_zk_path*/) +bool RestoreCoordinationRemote::acquireInsertingDataForKeeperMap(const String & root_zk_path) { - return true; + bool result = false; + auto holder = with_retries.createRetriesControlHolder("acquireInsertingDataForKeeperMap"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper]() + { + with_retries.renewZooKeeper(zk); + + fs::path base_path = fs::path(zookeeper_path) / "keeper_map_tables" / root_zk_path; + zk->createAncestors(base_path); + std::string restore_lock_path = base_path / "restore_lock"; + result = zk->tryCreate(restore_lock_path, "restorelock", zkutil::CreateMode::Persistent) == Coordination::Error::ZOK; + + if (result) + return; + + /// there can be an edge case where a path contains `/restore_lock/ in the middle of it + /// to differentiate that case from lock we also set the data + for (size_t i = 0; i < 1000; ++i) + { + Coordination::Stat lock_stat; + auto data = zk->get(restore_lock_path, &lock_stat); + if (data == "restorelock") + return; + + if (auto set_result = zk->trySet(restore_lock_path, "restorelock", lock_stat.version); + set_result == Coordination::Error::ZOK) + { + result = true; + return; + } + else if (set_result == Coordination::Error::ZNONODE) + throw zkutil::KeeperException::fromPath(set_result, restore_lock_path); + } + }); + return result; } void RestoreCoordinationRemote::generateUUIDForTable(ASTCreateQuery & create_query) diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index 33a97af53f1..c583a693035 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -753,7 +753,6 @@ void StorageKeeperMap::backupData(BackupEntriesCollector & backup_entries_collec { auto table_id = toString(getStorageID().uuid); - std::cout << "Backing up for path " << zk_root_path << " table id " << table_id << std::endl; auto coordination = backup_entries_collector.getBackupCoordination(); coordination->addKeeperMapTable(zk_root_path, table_id, data_path_in_backup); @@ -764,8 +763,6 @@ void StorageKeeperMap::backupData(BackupEntriesCollector & backup_entries_collec auto path_with_data = coordination->getKeeperMapDataPath(zk_root_path); if (path_with_data == my_data_path_in_backup) { - std::cout << "Will be backing up data for path " << zk_root_path << " table id " << my_table_id << std::endl; - auto temp_disk = backup_entries_collector.getContext()->getGlobalTemporaryVolume()->getDisk(0); auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef().max_compress_block_size; @@ -783,7 +780,6 @@ void StorageKeeperMap::backupData(BackupEntriesCollector & backup_entries_collec return; } - std::cout << "Not backing up data for path " << zk_root_path << " table id " << my_table_id << " writing only path with data " << path_with_data << std::endl; auto file_path = fs::path(my_data_path_in_backup) / backup_data_location_filename; backup_entries_collector.addBackupEntries({{file_path, std::make_shared(path_with_data)}}); }; @@ -842,8 +838,6 @@ void StorageKeeperMap::restoreDataImpl(const BackupPtr & backup, const String & { auto table_id = toString(getStorageID().uuid); - std::cout << "Restoring into " << zk_root_path << " table id " << table_id << std::endl; - fs::path data_path_in_backup_fs = data_path_in_backup; String data_file = data_path_in_backup_fs / backup_data_filename; @@ -882,7 +876,7 @@ void StorageKeeperMap::restoreDataImpl(const BackupPtr & backup, const String & }); }; - while (!in->eof()) + while (!compressed_in.eof()) { std::string key; std::string value; From 4438c2f70aeadd6ec6ab221f2980210351b7d1c5 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 8 Nov 2023 14:36:39 +0000 Subject: [PATCH 026/192] Remove unnecassary log --- src/Storages/StorageKeeperMap.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index c583a693035..74c1905cd61 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -692,14 +692,13 @@ private: data_children = zk->getChildren(data_zookeeper_path); }); } - LOG_INFO(&Poco::Logger::get("BACKUPER"), "Got {} children", data_children.size()); const auto write_rows = [&](std::span keys) { std::vector keys_full_path; - keys_full_path.reserve(data_children.size()); + keys_full_path.reserve(keys.size()); - for (const auto & key : data_children) + for (const auto & key : keys) keys_full_path.push_back(data_zookeeper_path / key); zkutil::ZooKeeper::MultiGetResponse data; From f9895ab37b2133a36296b67b8904d251ffdaf3e4 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 9 Nov 2023 15:56:57 +0000 Subject: [PATCH 027/192] Small fixes and add test --- src/Backups/BackupCoordinationRemote.cpp | 5 +- src/Backups/RestoreCoordinationRemote.cpp | 7 +- src/Common/ZooKeeper/ZooKeeper.cpp | 1 + src/Storages/StorageKeeperMap.cpp | 43 +++++-- src/Storages/StorageKeeperMap.h | 7 +- .../__init__.py | 0 .../configs/backups_disk.xml | 13 ++ .../configs/keeper_map_path_prefix.xml | 3 + .../configs/remote_servers.xml | 22 ++++ .../configs/zookeeper_retries.xml | 11 ++ .../test_backup_restore_keeper_map/test.py | 111 ++++++++++++++++++ .../02911_backup_restore_keeper_map.reference | 13 ++ .../02911_backup_restore_keeper_map.sh | 47 ++++++++ 13 files changed, 269 insertions(+), 14 deletions(-) create mode 100644 tests/integration/test_backup_restore_keeper_map/__init__.py create mode 100644 tests/integration/test_backup_restore_keeper_map/configs/backups_disk.xml create mode 100644 tests/integration/test_backup_restore_keeper_map/configs/keeper_map_path_prefix.xml create mode 100644 tests/integration/test_backup_restore_keeper_map/configs/remote_servers.xml create mode 100644 tests/integration/test_backup_restore_keeper_map/configs/zookeeper_retries.xml create mode 100644 tests/integration/test_backup_restore_keeper_map/test.py create mode 100644 tests/queries/0_stateless/02911_backup_restore_keeper_map.reference create mode 100755 tests/queries/0_stateless/02911_backup_restore_keeper_map.sh diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp index 309cbc8be6a..064e0599f6e 100644 --- a/src/Backups/BackupCoordinationRemote.cpp +++ b/src/Backups/BackupCoordinationRemote.cpp @@ -681,7 +681,10 @@ void BackupCoordinationRemote::addKeeperMapTable(const String & table_zookeeper_ { with_retries.renewZooKeeper(zk); String path = zookeeper_path + "/keeper_map_tables/" + escapeForFileName(table_id); - zk->create(path, fmt::format("{}\n{}", table_zookeeper_root_path, data_path_in_backup), zkutil::CreateMode::Persistent); + if (auto res + = zk->tryCreate(path, fmt::format("{}\n{}", table_zookeeper_root_path, data_path_in_backup), zkutil::CreateMode::Persistent); + res != Coordination::Error::ZOK && res != Coordination::Error::ZNODEEXISTS) + throw zkutil::KeeperException(res); }); } diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp index 12a67d2a55d..1b814c2889e 100644 --- a/src/Backups/RestoreCoordinationRemote.cpp +++ b/src/Backups/RestoreCoordinationRemote.cpp @@ -244,9 +244,10 @@ bool RestoreCoordinationRemote::acquireInsertingDataForKeeperMap(const String & { with_retries.renewZooKeeper(zk); - fs::path base_path = fs::path(zookeeper_path) / "keeper_map_tables" / root_zk_path; - zk->createAncestors(base_path); - std::string restore_lock_path = base_path / "restore_lock"; + /// we need to remove leading '/' from root_zk_path + auto normalized_root_zk_path = std::string_view{root_zk_path}.substr(1); + std::string restore_lock_path = fs::path(zookeeper_path) / "keeper_map_tables" / normalized_root_zk_path / "restore_lock"; + zk->createAncestors(restore_lock_path); result = zk->tryCreate(restore_lock_path, "restorelock", zkutil::CreateMode::Persistent) == Coordination::Error::ZOK; if (result) diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 436a4e14f14..8a97362aa96 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -385,6 +385,7 @@ void ZooKeeper::createAncestors(const std::string & path) size_t last_pos = path.rfind('/'); if (last_pos == std::string::npos || last_pos == 0) return; + std::string current_node = path.substr(0, last_pos); while (true) diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index 74c1905cd61..15ebc4d92d1 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -15,7 +15,7 @@ #include #include -#include +#include #include #include @@ -52,6 +52,8 @@ #include #include +#include + #include #include @@ -824,16 +826,24 @@ void StorageKeeperMap::restoreDataFromBackup(RestorerFromBackup & restorer, cons RestorerFromBackup::throwTableIsNotEmpty(getStorageID()); } - /// TODO: Should we backup and verify the table structure? + auto temp_disk = restorer.getContext()->getGlobalTemporaryVolume()->getDisk(0); - //auto temp_disk = restorer.getContext()->getGlobalTemporaryVolume()->getDisk(0); /// only 1 table should restore data for a single path restorer.addDataRestoreTask( - [storage = std::static_pointer_cast(shared_from_this()), backup, data_path_in_backup, with_retries, allow_non_empty_tables] - { storage->restoreDataImpl(backup, data_path_in_backup, with_retries, allow_non_empty_tables); }); + [storage = std::static_pointer_cast(shared_from_this()), + backup, + data_path_in_backup, + with_retries, + allow_non_empty_tables, + temp_disk] { storage->restoreDataImpl(backup, data_path_in_backup, with_retries, allow_non_empty_tables, temp_disk); }); } -void StorageKeeperMap::restoreDataImpl(const BackupPtr & backup, const String & data_path_in_backup, std::shared_ptr with_retries, bool allow_non_empty_tables) +void StorageKeeperMap::restoreDataImpl( + const BackupPtr & backup, + const String & data_path_in_backup, + std::shared_ptr with_retries, + bool allow_non_empty_tables, + const DiskPtr & temporary_disk) { auto table_id = toString(getStorageID().uuid); @@ -858,7 +868,17 @@ void StorageKeeperMap::restoreDataImpl(const BackupPtr & backup, const String & /// should we store locally in temp file? auto in = backup->readFile(data_file); - CompressedReadBuffer compressed_in{*in}; + std::optional temp_data_file; + if (!dynamic_cast(in.get())) + { + temp_data_file.emplace(temporary_disk); + auto out = std::make_unique(temp_data_file->getAbsolutePath()); + copyData(*in, *out); + out.reset(); + in = createReadBufferFromFileBase(temp_data_file->getAbsolutePath(), {}); + } + std::unique_ptr in_from_file{static_cast(in.release())}; + CompressedReadBufferFromFile compressed_in{std::move(in_from_file)}; fs::path data_path_fs(zk_data_path); auto max_multi_size = with_retries->getKeeperSettings().batch_size_for_keeper_multi; @@ -871,7 +891,10 @@ void StorageKeeperMap::restoreDataImpl(const BackupPtr & backup, const String & [&, &zk = holder.faulty_zookeeper]() { with_retries->renewZooKeeper(zk); - zk->multi(create_requests); + Coordination::Responses create_responses; + if (auto res = zk->tryMulti(create_requests, create_responses); + res != Coordination::Error::ZOK && res != Coordination::Error::ZNODEEXISTS) + throw zkutil::KeeperMultiException(res, create_requests, create_responses); }); }; @@ -890,7 +913,9 @@ void StorageKeeperMap::restoreDataImpl(const BackupPtr & backup, const String & [&, &zk = holder.faulty_zookeeper]() { with_retries->renewZooKeeper(zk); - zk->tryCreate(data_path_fs / key, value, zkutil::CreateMode::Persistent); + if (auto res = zk->tryCreate(data_path_fs / key, value, zkutil::CreateMode::Persistent); + res != Coordination::Error::ZOK && res != Coordination::Error::ZNODEEXISTS) + throw zkutil::KeeperException::fromPath(res, data_path_fs / key); }); } /// otherwise we can do multi requests diff --git a/src/Storages/StorageKeeperMap.h b/src/Storages/StorageKeeperMap.h index 94b02ca0242..10eebdd0129 100644 --- a/src/Storages/StorageKeeperMap.h +++ b/src/Storages/StorageKeeperMap.h @@ -120,7 +120,12 @@ private: std::optional isTableValid() const; - void restoreDataImpl(const BackupPtr & backup, const String & data_path_in_backup, std::shared_ptr with_retries, bool allow_non_empty_tables); + void restoreDataImpl( + const BackupPtr & backup, + const String & data_path_in_backup, + std::shared_ptr with_retries, + bool allow_non_empty_tables, + const DiskPtr & temporary_disk); std::string zk_root_path; std::string primary_key; diff --git a/tests/integration/test_backup_restore_keeper_map/__init__.py b/tests/integration/test_backup_restore_keeper_map/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_backup_restore_keeper_map/configs/backups_disk.xml b/tests/integration/test_backup_restore_keeper_map/configs/backups_disk.xml new file mode 100644 index 00000000000..b99a51cd56d --- /dev/null +++ b/tests/integration/test_backup_restore_keeper_map/configs/backups_disk.xml @@ -0,0 +1,13 @@ + + + + + local + /backups/ + + + + + backups + + diff --git a/tests/integration/test_backup_restore_keeper_map/configs/keeper_map_path_prefix.xml b/tests/integration/test_backup_restore_keeper_map/configs/keeper_map_path_prefix.xml new file mode 100644 index 00000000000..91d7b9d3f8f --- /dev/null +++ b/tests/integration/test_backup_restore_keeper_map/configs/keeper_map_path_prefix.xml @@ -0,0 +1,3 @@ + + /keeper_map_tables + diff --git a/tests/integration/test_backup_restore_keeper_map/configs/remote_servers.xml b/tests/integration/test_backup_restore_keeper_map/configs/remote_servers.xml new file mode 100644 index 00000000000..5cf07c69fd6 --- /dev/null +++ b/tests/integration/test_backup_restore_keeper_map/configs/remote_servers.xml @@ -0,0 +1,22 @@ + + + + + + node1 + 9000 + + + node2 + 9000 + + + + + node3 + 9000 + + + + + diff --git a/tests/integration/test_backup_restore_keeper_map/configs/zookeeper_retries.xml b/tests/integration/test_backup_restore_keeper_map/configs/zookeeper_retries.xml new file mode 100644 index 00000000000..1283f28a8cb --- /dev/null +++ b/tests/integration/test_backup_restore_keeper_map/configs/zookeeper_retries.xml @@ -0,0 +1,11 @@ + + + + 1000 + 1 + 1 + 42 + 0.002 + + + diff --git a/tests/integration/test_backup_restore_keeper_map/test.py b/tests/integration/test_backup_restore_keeper_map/test.py new file mode 100644 index 00000000000..95e8a8b3027 --- /dev/null +++ b/tests/integration/test_backup_restore_keeper_map/test.py @@ -0,0 +1,111 @@ +from time import sleep +import pytest +from helpers.cluster import ClickHouseCluster + + +cluster = ClickHouseCluster(__file__) + +main_configs = [ + "configs/remote_servers.xml", + "configs/backups_disk.xml", + "configs/keeper_map_path_prefix.xml", +] + +user_configs = [ + "configs/zookeeper_retries.xml", +] + +node1 = cluster.add_instance( + "node1", + main_configs=main_configs, + user_configs=user_configs, + external_dirs=["/backups/"], + macros={"replica": "node1", "shard": "shard1"}, + with_zookeeper=True, + stay_alive=True, +) + +node2 = cluster.add_instance( + "node2", + main_configs=main_configs, + user_configs=user_configs, + external_dirs=["/backups/"], + macros={"replica": "node2", "shard": "shard1"}, + with_zookeeper=True, + stay_alive=True, +) + + +node3 = cluster.add_instance( + "node3", + main_configs=main_configs, + user_configs=user_configs, + external_dirs=["/backups/"], + macros={"replica": "node3", "shard": "shard2"}, + with_zookeeper=True, + stay_alive=True, +) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +backup_id_counter = 0 + +def new_backup_name(base_name): + global backup_id_counter + backup_id_counter += 1 + return f"Disk('backups', '{base_name}{backup_id_counter}')" + +def test_on_cluster(): + node1.query_with_retry("CREATE DATABASE keeper_backup ON CLUSTER cluster") + node1.query_with_retry("CREATE TABLE keeper_backup.keeper1 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key") + node1.query_with_retry("CREATE TABLE keeper_backup.keeper2 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key") + node1.query_with_retry("CREATE TABLE keeper_backup.keeper3 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster2') PRIMARY KEY key") + node1.query_with_retry("INSERT INTO keeper_backup.keeper2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5") + node1.query_with_retry("INSERT INTO keeper_backup.keeper3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5") + + expected_result = ''.join(f'{i}\ttest{i}\n' for i in range(5)) + + def verify_data(): + for node in [node1, node2, node3]: + for i in range(1, 4): + result = node.query_with_retry(f'SELECT key, value FROM keeper_backup.keeper{i} ORDER BY key FORMAT TSV') + assert result == expected_result + + verify_data() + + backup_name = new_backup_name('test_on_cluster') + node1.query(f"BACKUP DATABASE keeper_backup ON CLUSTER cluster TO {backup_name} SETTINGS async = false;") + + node1.query("DROP DATABASE keeper_backup ON CLUSTER cluster SYNC;") + + def apply_for_all_nodes(f): + for node in [node1, node2, node3]: + f(node) + + def change_keeper_map_prefix(node): + node.replace_config( + "/etc/clickhouse-server/config.d/keeper_map_path_prefix.xml", """ + + /different_path/keeper_map + +""") + + apply_for_all_nodes(lambda node: node.stop_clickhouse()) + apply_for_all_nodes(change_keeper_map_prefix) + apply_for_all_nodes(lambda node: node.start_clickhouse()) + + node1.query(f"RESTORE DATABASE keeper_backup ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;") + + verify_data() + + node1.query("DROP TABLE keeper_backup.keeper3 ON CLUSTER cluster SYNC;") + node1.query(f"RESTORE TABLE keeper_backup.keeper3 ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;") + + verify_data() \ No newline at end of file diff --git a/tests/queries/0_stateless/02911_backup_restore_keeper_map.reference b/tests/queries/0_stateless/02911_backup_restore_keeper_map.reference new file mode 100644 index 00000000000..e58335de67c --- /dev/null +++ b/tests/queries/0_stateless/02911_backup_restore_keeper_map.reference @@ -0,0 +1,13 @@ +5000 +5000 +3000 +OK +OK +OK +5000 +5000 +3000 +OK +5000 +5000 +3000 diff --git a/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh b/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh new file mode 100755 index 00000000000..6c463beb221 --- /dev/null +++ b/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -nm -q " + DROP DATABASE IF EXISTS 02911_keeper_map; + CREATE DATABASE 02911_keeper_map; + CREATE TABLE 02911_keeper_map.02911_backup_restore_keeper_map1 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911') PRIMARY KEY key; + CREATE TABLE 02911_keeper_map.02911_backup_restore_keeper_map2 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911') PRIMARY KEY key; + CREATE TABLE 02911_keeper_map.02911_backup_restore_keeper_map3 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911_different') PRIMARY KEY key; + + INSERT INTO 02911_keeper_map.02911_backup_restore_keeper_map2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5000; + INSERT INTO 02911_keeper_map.02911_backup_restore_keeper_map3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 3000; +" + +backup_path="$CLICKHOUSE_DATABASE/02911_keeper_map" +for i in $(seq 1 3); do + $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" +done + +$CLICKHOUSE_CLIENT -q "BACKUP DATABASE 02911_keeper_map TO Disk('backups', '$backup_path');" > /dev/null + +$CLICKHOUSE_CLIENT -q "DROP DATABASE 02911_keeper_map SYNC;" + +for i in $(seq 1 3); do + $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" 2>&1 | grep -Fq "UNKNOWN_DATABASE" && echo 'OK' || echo 'ERROR' +done + +$CLICKHOUSE_CLIENT -q "RESTORE DATABASE 02911_keeper_map FROM Disk('backups', '$backup_path');" > /dev/null + +for i in $(seq 1 3); do + $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" +done + +$CLICKHOUSE_CLIENT -q "DROP TABLE 02911_keeper_map.02911_backup_restore_keeper_map3 SYNC;" + +$CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map3;" 2>&1 | grep -Fq "UNKNOWN_TABLE" && echo 'OK' || echo 'ERROR' + +$CLICKHOUSE_CLIENT -q "RESTORE TABLE 02911_keeper_map.02911_backup_restore_keeper_map3 FROM Disk('backups', '$backup_path');" > /dev/null + +for i in $(seq 1 3); do + $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" +done + +$CLICKHOUSE_CLIENT -q "DROP DATABASE 02911_keeper_map SYNC;" \ No newline at end of file From 188a88fa3391cd2044fce96d00da12c22b654319 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 9 Nov 2023 16:15:14 +0000 Subject: [PATCH 028/192] Automatic style fix --- .../test_backup_restore_keeper_map/test.py | 59 +++++++++++++------ 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/tests/integration/test_backup_restore_keeper_map/test.py b/tests/integration/test_backup_restore_keeper_map/test.py index 95e8a8b3027..8343ad3177f 100644 --- a/tests/integration/test_backup_restore_keeper_map/test.py +++ b/tests/integration/test_backup_restore_keeper_map/test.py @@ -55,33 +55,50 @@ def start_cluster(): finally: cluster.shutdown() + backup_id_counter = 0 + def new_backup_name(base_name): global backup_id_counter backup_id_counter += 1 return f"Disk('backups', '{base_name}{backup_id_counter}')" + def test_on_cluster(): node1.query_with_retry("CREATE DATABASE keeper_backup ON CLUSTER cluster") - node1.query_with_retry("CREATE TABLE keeper_backup.keeper1 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key") - node1.query_with_retry("CREATE TABLE keeper_backup.keeper2 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key") - node1.query_with_retry("CREATE TABLE keeper_backup.keeper3 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster2') PRIMARY KEY key") - node1.query_with_retry("INSERT INTO keeper_backup.keeper2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5") - node1.query_with_retry("INSERT INTO keeper_backup.keeper3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5") + node1.query_with_retry( + "CREATE TABLE keeper_backup.keeper1 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key" + ) + node1.query_with_retry( + "CREATE TABLE keeper_backup.keeper2 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key" + ) + node1.query_with_retry( + "CREATE TABLE keeper_backup.keeper3 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster2') PRIMARY KEY key" + ) + node1.query_with_retry( + "INSERT INTO keeper_backup.keeper2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5" + ) + node1.query_with_retry( + "INSERT INTO keeper_backup.keeper3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5" + ) - expected_result = ''.join(f'{i}\ttest{i}\n' for i in range(5)) + expected_result = "".join(f"{i}\ttest{i}\n" for i in range(5)) def verify_data(): for node in [node1, node2, node3]: for i in range(1, 4): - result = node.query_with_retry(f'SELECT key, value FROM keeper_backup.keeper{i} ORDER BY key FORMAT TSV') + result = node.query_with_retry( + f"SELECT key, value FROM keeper_backup.keeper{i} ORDER BY key FORMAT TSV" + ) assert result == expected_result verify_data() - backup_name = new_backup_name('test_on_cluster') - node1.query(f"BACKUP DATABASE keeper_backup ON CLUSTER cluster TO {backup_name} SETTINGS async = false;") + backup_name = new_backup_name("test_on_cluster") + node1.query( + f"BACKUP DATABASE keeper_backup ON CLUSTER cluster TO {backup_name} SETTINGS async = false;" + ) node1.query("DROP DATABASE keeper_backup ON CLUSTER cluster SYNC;") @@ -91,21 +108,27 @@ def test_on_cluster(): def change_keeper_map_prefix(node): node.replace_config( - "/etc/clickhouse-server/config.d/keeper_map_path_prefix.xml", """ + "/etc/clickhouse-server/config.d/keeper_map_path_prefix.xml", + """ /different_path/keeper_map -""") +""", + ) apply_for_all_nodes(lambda node: node.stop_clickhouse()) apply_for_all_nodes(change_keeper_map_prefix) apply_for_all_nodes(lambda node: node.start_clickhouse()) - node1.query(f"RESTORE DATABASE keeper_backup ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;") - - verify_data() - - node1.query("DROP TABLE keeper_backup.keeper3 ON CLUSTER cluster SYNC;") - node1.query(f"RESTORE TABLE keeper_backup.keeper3 ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;") + node1.query( + f"RESTORE DATABASE keeper_backup ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;" + ) - verify_data() \ No newline at end of file + verify_data() + + node1.query("DROP TABLE keeper_backup.keeper3 ON CLUSTER cluster SYNC;") + node1.query( + f"RESTORE TABLE keeper_backup.keeper3 ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;" + ) + + verify_data() From 124af73f1d51dd52b26ac6cb697c34a548dff29e Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Thu, 9 Nov 2023 17:54:45 +0100 Subject: [PATCH 029/192] Add support of arbitrary types to concat --- src/Functions/concat.cpp | 35 ++++++++++++++------------- src/Functions/concatWithSeparator.cpp | 2 +- src/Functions/formatString.cpp | 2 +- src/Functions/formatString.h | 2 +- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 9eb222d8c09..350cbee58a3 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -56,18 +57,6 @@ public: getName(), arguments.size()); - for (const auto arg_idx : collections::range(0, arguments.size())) - { - const auto * arg = arguments[arg_idx].get(); - if (!isStringOrFixedString(arg)) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of argument {} of function {}", - arg->getName(), - arg_idx + 1, - getName()); - } - return std::make_shared(); } @@ -76,7 +65,7 @@ public: /// Format function is not proven to be faster for two arguments. /// Actually there is overhead of 2 to 5 extra instructions for each string for checking empty strings in FormatImpl. /// Though, benchmarks are really close, for most examples we saw executeBinary is slightly faster (0-3%). - /// For 3 and more arguments FormatImpl is much faster (up to 50-60%). + /// For 3 and more arguments FormatStringImpl is much faster (up to 50-60%). if (arguments.size() == 2) return executeBinary(arguments, input_rows_count); else @@ -107,6 +96,7 @@ private: else { /// Fallback: use generic implementation for not very important cases. + /// Concat of arbitrary types also goes here. return executeFormatImpl(arguments, input_rows_count); } @@ -145,8 +135,18 @@ private: constant_strings[i] = const_col->getValue(); } else - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", - column->getName(), getName()); + { + // An arbitrary type argument: converting it to a StringColumn as if `toString` was called + ColumnsWithTypeAndName args; + args.emplace_back(column, arguments[i].type, "tmp"); + const ColumnPtr converted_col_ptr = ConvertImplGenericToString::execute( + args, std::make_shared(), column->size()); + const ColumnString * converted_col_str = assert_cast(converted_col_ptr.get()); + // Same as the normal `ColumnString` branch + has_column_string = true; + data[i] = &converted_col_str->getChars(); + offsets[i] = &converted_col_str->getOffsets(); + } } String pattern; @@ -155,7 +155,7 @@ private: for (size_t i = 0; i < num_arguments; ++i) pattern += "{}"; - FormatImpl::formatExecute( + FormatStringImpl::formatExecute( has_column_string, has_column_fixed_string, std::move(pattern), @@ -185,7 +185,8 @@ using FunctionConcat = ConcatImpl; using FunctionConcatAssumeInjective = ConcatImpl; -/// Also works with arrays. +/// Works with arrays via `arrayConcat`, maps via `mapConcat`, and tuples via `tupleConcat`. +/// Additionally, allows concatenation of arbitrary types that can be cast to string using the corresponding default serialization. class ConcatOverloadResolver : public IFunctionOverloadResolver { public: diff --git a/src/Functions/concatWithSeparator.cpp b/src/Functions/concatWithSeparator.cpp index bfd1bc392db..f0a7afbbaa7 100644 --- a/src/Functions/concatWithSeparator.cpp +++ b/src/Functions/concatWithSeparator.cpp @@ -122,7 +122,7 @@ public: for (size_t i = 0; i < num_args; ++i) pattern += "{}"; - FormatImpl::formatExecute( + FormatStringImpl::formatExecute( has_column_string, has_column_fixed_string, std::move(pattern), diff --git a/src/Functions/formatString.cpp b/src/Functions/formatString.cpp index ee6e26b775a..8e0b3a238cb 100644 --- a/src/Functions/formatString.cpp +++ b/src/Functions/formatString.cpp @@ -110,7 +110,7 @@ public: column->getName(), getName()); } - FormatImpl::formatExecute( + FormatStringImpl::formatExecute( has_column_string, has_column_fixed_string, std::move(pattern), diff --git a/src/Functions/formatString.h b/src/Functions/formatString.h index 44fbdac9378..30149e9a5b0 100644 --- a/src/Functions/formatString.h +++ b/src/Functions/formatString.h @@ -18,7 +18,7 @@ namespace DB { -struct FormatImpl +struct FormatStringImpl { static constexpr size_t right_padding = 15; From 5fea9f9dc6fd33baf332affc458d2bbbd82f4d0f Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 10 Nov 2023 09:25:57 +0000 Subject: [PATCH 030/192] Small fixes --- src/Common/ZooKeeper/ZooKeeper.h | 19 +++++++++ .../ZooKeeper/ZooKeeperWithFaultInjection.h | 5 +++ src/Storages/StorageKeeperMap.cpp | 5 ++- .../02911_backup_restore_keeper_map.sh | 39 ++++++++++--------- 4 files changed, 47 insertions(+), 21 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index c41d1d8dbab..785842b94bd 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -135,6 +135,16 @@ struct MultiReadResponses responses); } + /// If Keeper/ZooKeeper doesn't support MultiRead feature we will dispatch + /// asynchronously all the read requests separately + /// Sometimes it's important to process all requests instantly + /// e.g. we want to trigger exceptions while we are in the ZK client retry loop + void waitForResponses() + { + if (auto * responses_with_futures = std::get_if(&responses)) + responses_with_futures->waitForResponses(); + } + private: using RegularResponses = std::vector; using FutureResponses = std::vector>; @@ -158,6 +168,15 @@ private: return *cached_responses[index]; } + void waitForResponses() + { + for (size_t i = 0; i < size(); ++i) + { + if (!cached_responses[i].has_value()) + cached_responses[i] = future_responses[i].get(); + } + } + size_t size() const { return future_responses.size(); } }; diff --git a/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h b/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h index 4887e896e9b..be4642c2988 100644 --- a/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h +++ b/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h @@ -242,6 +242,11 @@ public: return access("get", !paths.empty() ? paths.front() : "", [&]() { return keeper->get(paths); }); } + zkutil::ZooKeeper::MultiTryGetResponse tryGet(const std::vector & paths) + { + return access("tryGet", !paths.empty() ? paths.front() : "", [&]() { return keeper->tryGet(paths); }); + } + bool exists(const std::string & path, Coordination::Stat * stat = nullptr, const zkutil::EventPtr & watch = nullptr) { return access("exists", path, [&]() { return keeper->exists(path, stat, watch); }); diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index 15ebc4d92d1..3032973c411 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -703,13 +703,14 @@ private: for (const auto & key : keys) keys_full_path.push_back(data_zookeeper_path / key); - zkutil::ZooKeeper::MultiGetResponse data; + zkutil::ZooKeeper::MultiTryGetResponse data; auto holder = with_retries->createRetriesControlHolder("getKeeperMapDataKeys"); holder.retries_ctl.retryLoop( [&, &zk = holder.faulty_zookeeper] { with_retries->renewZooKeeper(zk); - data = zk->get(keys_full_path); + data = zk->tryGet(keys_full_path); + data.waitForResponses(); }); for (size_t i = 0; i < keys.size(); ++i) diff --git a/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh b/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh index 6c463beb221..ae7c22f6820 100755 --- a/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh +++ b/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh @@ -4,44 +4,45 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh +database_name="$CLICKHOUSE_DATABASE"_02911_keeper_map $CLICKHOUSE_CLIENT -nm -q " - DROP DATABASE IF EXISTS 02911_keeper_map; - CREATE DATABASE 02911_keeper_map; - CREATE TABLE 02911_keeper_map.02911_backup_restore_keeper_map1 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911') PRIMARY KEY key; - CREATE TABLE 02911_keeper_map.02911_backup_restore_keeper_map2 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911') PRIMARY KEY key; - CREATE TABLE 02911_keeper_map.02911_backup_restore_keeper_map3 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911_different') PRIMARY KEY key; + DROP DATABASE IF EXISTS $database_name; + CREATE DATABASE $database_name; + CREATE TABLE $database_name.02911_backup_restore_keeper_map1 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911') PRIMARY KEY key; + CREATE TABLE $database_name.02911_backup_restore_keeper_map2 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911') PRIMARY KEY key; + CREATE TABLE $database_name.02911_backup_restore_keeper_map3 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911_different') PRIMARY KEY key; - INSERT INTO 02911_keeper_map.02911_backup_restore_keeper_map2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5000; - INSERT INTO 02911_keeper_map.02911_backup_restore_keeper_map3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 3000; + INSERT INTO $database_name.02911_backup_restore_keeper_map2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5000; + INSERT INTO $database_name.02911_backup_restore_keeper_map3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 3000; " -backup_path="$CLICKHOUSE_DATABASE/02911_keeper_map" +backup_path="$database_name" for i in $(seq 1 3); do - $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" + $CLICKHOUSE_CLIENT -q "SELECT count() FROM $database_name.02911_backup_restore_keeper_map$i;" done -$CLICKHOUSE_CLIENT -q "BACKUP DATABASE 02911_keeper_map TO Disk('backups', '$backup_path');" > /dev/null +$CLICKHOUSE_CLIENT -q "BACKUP DATABASE $database_name TO Disk('backups', '$backup_path');" > /dev/null -$CLICKHOUSE_CLIENT -q "DROP DATABASE 02911_keeper_map SYNC;" +$CLICKHOUSE_CLIENT -q "DROP DATABASE $database_name SYNC;" for i in $(seq 1 3); do - $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" 2>&1 | grep -Fq "UNKNOWN_DATABASE" && echo 'OK' || echo 'ERROR' + $CLICKHOUSE_CLIENT -q "SELECT count() FROM $database_name.02911_backup_restore_keeper_map$i;" 2>&1 | grep -Fq "UNKNOWN_DATABASE" && echo 'OK' || echo 'ERROR' done -$CLICKHOUSE_CLIENT -q "RESTORE DATABASE 02911_keeper_map FROM Disk('backups', '$backup_path');" > /dev/null +$CLICKHOUSE_CLIENT -q "RESTORE DATABASE $database_name FROM Disk('backups', '$backup_path');" > /dev/null for i in $(seq 1 3); do - $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" + $CLICKHOUSE_CLIENT -q "SELECT count() FROM $database_name.02911_backup_restore_keeper_map$i;" done -$CLICKHOUSE_CLIENT -q "DROP TABLE 02911_keeper_map.02911_backup_restore_keeper_map3 SYNC;" +$CLICKHOUSE_CLIENT -q "DROP TABLE $database_name.02911_backup_restore_keeper_map3 SYNC;" -$CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map3;" 2>&1 | grep -Fq "UNKNOWN_TABLE" && echo 'OK' || echo 'ERROR' +$CLICKHOUSE_CLIENT -q "SELECT count() FROM $database_name.02911_backup_restore_keeper_map3;" 2>&1 | grep -Fq "UNKNOWN_TABLE" && echo 'OK' || echo 'ERROR' -$CLICKHOUSE_CLIENT -q "RESTORE TABLE 02911_keeper_map.02911_backup_restore_keeper_map3 FROM Disk('backups', '$backup_path');" > /dev/null +$CLICKHOUSE_CLIENT -q "RESTORE TABLE $database_name.02911_backup_restore_keeper_map3 FROM Disk('backups', '$backup_path');" > /dev/null for i in $(seq 1 3); do - $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" + $CLICKHOUSE_CLIENT -q "SELECT count() FROM $database_name.02911_backup_restore_keeper_map$i;" done -$CLICKHOUSE_CLIENT -q "DROP DATABASE 02911_keeper_map SYNC;" \ No newline at end of file +$CLICKHOUSE_CLIENT -q "DROP DATABASE $database_name SYNC;" \ No newline at end of file From 356ae52e9b122545ea9a26a18ccce0e5311d4df5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 11 Nov 2023 08:46:10 +0100 Subject: [PATCH 031/192] Enable access and named collections control by default --- .../config/users.d/perf-comparison-tweaks-users.xml | 5 ----- programs/server/embedded.xml | 2 ++ programs/server/users.xml | 5 ++++- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml b/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml index cb591f1a184..e780a99ecde 100644 --- a/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml +++ b/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml @@ -34,9 +34,4 @@ 0 - - - 1 - - diff --git a/programs/server/embedded.xml b/programs/server/embedded.xml index c2336e0d582..9311749a173 100644 --- a/programs/server/embedded.xml +++ b/programs/server/embedded.xml @@ -23,7 +23,9 @@ default default + 1 + 1 diff --git a/programs/server/users.xml b/programs/server/users.xml index fbb5a2c228f..57bc6309a54 100644 --- a/programs/server/users.xml +++ b/programs/server/users.xml @@ -85,7 +85,10 @@ default - + 1 + + + 1 0 + 0 20000 @@ -33,6 +34,7 @@ true 1 + 0 1 20000 diff --git a/tests/integration/test_storage_s3/configs/defaultS3.xml b/tests/integration/test_storage_s3/configs/defaultS3.xml index 37454ef6781..7dac6d9fbb5 100644 --- a/tests/integration/test_storage_s3/configs/defaultS3.xml +++ b/tests/integration/test_storage_s3/configs/defaultS3.xml @@ -1,9 +1,4 @@ - - - 5 - - http://resolver:8080 diff --git a/tests/integration/test_storage_s3/configs/s3_retry.xml b/tests/integration/test_storage_s3/configs/s3_retry.xml index 727e23273cf..581fc44c8d4 100644 --- a/tests/integration/test_storage_s3/configs/s3_retry.xml +++ b/tests/integration/test_storage_s3/configs/s3_retry.xml @@ -1,7 +1,7 @@ - 5 + 10 From 45de9beab4231a806ea247adef0a1fc5180748ba Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 8 Nov 2023 13:19:28 +0100 Subject: [PATCH 044/192] set new timeout for session from connection pool --- base/poco/Net/src/HTTPSession.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index d2663baaf9f..d30f5590280 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -94,8 +94,22 @@ void HTTPSession::setTimeout(const Poco::Timespan& timeout) void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco::Timespan& sendTimeout, const Poco::Timespan& receiveTimeout) { _connectionTimeout = connectionTimeout; - _sendTimeout = sendTimeout; - _receiveTimeout = receiveTimeout; + + if (_sendTimeout != sendTimeout) + { + _sendTimeout = sendTimeout; + + if (connected()) + _socket.setSendTimeout(_sendTimeout); + } + + if (_receiveTimeout != receiveTimeout) + { + _receiveTimeout = receiveTimeout; + + if (connected()) + _socket.setReceiveTimeout(_receiveTimeout); + } } From be01a5cd3e07eeba990a8dcc3e69e62f3492d05e Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 8 Nov 2023 17:32:06 +0100 Subject: [PATCH 045/192] turn off agressive timeouts for heavy requests --- src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp | 6 +++++- src/Disks/ObjectStorages/S3/S3ObjectStorage.h | 1 + src/IO/S3/Client.cpp | 9 ++++++--- src/IO/S3/Client.h | 6 ++---- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index b36185249af..aa4bcd7fbad 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -537,7 +537,11 @@ std::unique_ptr S3ObjectStorage::cloneObjectStorage( } S3ObjectStorage::Clients::Clients(std::shared_ptr client_, const S3ObjectStorageSettings & settings) - : client(std::move(client_)), client_with_long_timeout(client->clone(std::nullopt, settings.request_settings.long_request_timeout_ms)) {} + : client(std::move(client_)) + , client_with_long_timeout(client->clone( + /*override_aggressive_timeouts*/ false, + settings.request_settings.long_request_timeout_ms)) +{} ObjectStorageKey S3ObjectStorage::generateObjectKeyForPath(const std::string &) const { diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index b1b3fb22366..37e491e21dc 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -184,6 +184,7 @@ private: std::string bucket; String object_key_prefix; + MultiVersion clients; MultiVersion s3_settings; S3Capabilities s3_capabilities; diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 4250342c49f..12a0cb8f93c 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -119,14 +119,17 @@ std::unique_ptr Client::create( } std::unique_ptr Client::clone( - std::optional> override_retry_strategy, + std::optional override_aggressive_timeouts, std::optional override_request_timeout_ms) const { PocoHTTPClientConfiguration new_configuration = client_configuration; - if (override_retry_strategy.has_value()) - new_configuration.retryStrategy = *override_retry_strategy; + if (override_request_timeout_ms.has_value()) new_configuration.requestTimeoutMs = *override_request_timeout_ms; + + if (override_aggressive_timeouts.has_value()) + new_configuration.s3_aggressive_timeouts = *override_aggressive_timeouts; + return std::unique_ptr(new Client(*this, new_configuration)); } diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index 48310bc21af..81ab3854d3d 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -119,13 +119,11 @@ public: bool use_virtual_addressing); /// Create a client with adjusted settings: - /// * override_retry_strategy can be used to disable retries to avoid nested retries when we have - /// a retry loop outside of S3 client. Specifically, for read and write buffers. Currently not - /// actually used. /// * override_request_timeout_ms is used to increase timeout for CompleteMultipartUploadRequest /// because it often sits idle for 10 seconds: https://github.com/ClickHouse/ClickHouse/pull/42321 + /// * s3_aggressive_timeouts is used to turn off s3_aggressive_timeouts feature for CompleteMultipartUploadRequest std::unique_ptr clone( - std::optional> override_retry_strategy = std::nullopt, + std::optional override_aggressive_timeouts = std::nullopt, std::optional override_request_timeout_ms = std::nullopt) const; Client & operator=(const Client &) = delete; From 27fb25d056c420bca141ce2ecd83868d15fd07ef Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 9 Nov 2023 13:10:52 +0100 Subject: [PATCH 046/192] alter the naming, fix client_with_long_timeout in s3 storage --- src/Core/Settings.h | 2 +- src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp | 2 +- src/Disks/ObjectStorages/S3/diskSettings.cpp | 4 ++-- src/IO/S3/Client.cpp | 8 ++++---- src/IO/S3/Client.h | 4 ++-- src/IO/S3/PocoHTTPClient.cpp | 8 ++++---- src/IO/S3/PocoHTTPClient.h | 6 +++--- src/Storages/StorageS3.cpp | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b1459b6f328..3f80c83ff5f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -94,7 +94,7 @@ class IColumn; M(UInt64, s3_max_put_rps, 0, "Limit on S3 PUT request per second rate before throttling. Zero means unlimited.", 0) \ M(UInt64, s3_max_put_burst, 0, "Max number of requests that can be issued simultaneously before hitting request per second limit. By default (0) equals to `s3_max_put_rps`", 0) \ M(UInt64, s3_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ - M(Bool, s3_aggressive_timeouts, true, "When aggressive timeouts are enabled first two attempts are made with low receive and send timeout", 0) \ + M(Bool, s3_use_adaptive_timeouts, true, "When aggressive timeouts are enabled first two attempts are made with low receive and send timeout", 0) \ M(UInt64, azure_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ M(Bool, s3_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables.", 0) \ M(Bool, azure_truncate_on_insert, false, "Enables or disables truncate before insert in azure engine tables.", 0) \ diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index aa4bcd7fbad..8a46bfd59d1 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -539,7 +539,7 @@ std::unique_ptr S3ObjectStorage::cloneObjectStorage( S3ObjectStorage::Clients::Clients(std::shared_ptr client_, const S3ObjectStorageSettings & settings) : client(std::move(client_)) , client_with_long_timeout(client->clone( - /*override_aggressive_timeouts*/ false, + /*override_use_adaptive_timeouts*/ false, settings.request_settings.long_request_timeout_ms)) {} diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 43618c64776..573fa744ce6 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -67,8 +67,8 @@ std::unique_ptr getClient( config_prefix + ".http_keep_alive_timeout_ms", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT * 1000); client_configuration.http_connection_pool_size = config.getUInt(config_prefix + ".http_connection_pool_size", 1000); client_configuration.wait_on_pool_size_limit = false; - client_configuration.s3_aggressive_timeouts = config.getUInt( - config_prefix + ".aggressive_timeouts", client_configuration.s3_aggressive_timeouts); + client_configuration.s3_use_adaptive_timeouts = config.getUInt( + config_prefix + ".use_adaptive_timeouts", client_configuration.s3_use_adaptive_timeouts); /* * Override proxy configuration for backwards compatibility with old configuration format. diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 12a0cb8f93c..90806852c1e 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -119,7 +119,7 @@ std::unique_ptr Client::create( } std::unique_ptr Client::clone( - std::optional override_aggressive_timeouts, + std::optional override_use_adaptive_timeouts, std::optional override_request_timeout_ms) const { PocoHTTPClientConfiguration new_configuration = client_configuration; @@ -127,8 +127,8 @@ std::unique_ptr Client::clone( if (override_request_timeout_ms.has_value()) new_configuration.requestTimeoutMs = *override_request_timeout_ms; - if (override_aggressive_timeouts.has_value()) - new_configuration.s3_aggressive_timeouts = *override_aggressive_timeouts; + if (override_use_adaptive_timeouts.has_value()) + new_configuration.s3_use_adaptive_timeouts = *override_use_adaptive_timeouts; return std::unique_ptr(new Client(*this, new_configuration)); } @@ -908,7 +908,7 @@ PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT s3_retry_attempts, enable_s3_requests_logging, for_disk_s3, - context->getGlobalContext()->getSettingsRef().s3_aggressive_timeouts, + context->getGlobalContext()->getSettingsRef().s3_use_adaptive_timeouts, get_request_throttler, put_request_throttler, error_report); diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index 81ab3854d3d..be7235eb9f1 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -121,9 +121,9 @@ public: /// Create a client with adjusted settings: /// * override_request_timeout_ms is used to increase timeout for CompleteMultipartUploadRequest /// because it often sits idle for 10 seconds: https://github.com/ClickHouse/ClickHouse/pull/42321 - /// * s3_aggressive_timeouts is used to turn off s3_aggressive_timeouts feature for CompleteMultipartUploadRequest + /// * s3_use_adaptive_timeouts is used to turn off s3_use_adaptive_timeouts feature for CompleteMultipartUploadRequest std::unique_ptr clone( - std::optional override_aggressive_timeouts = std::nullopt, + std::optional override_use_adaptive_timeouts = std::nullopt, std::optional override_request_timeout_ms = std::nullopt) const; Client & operator=(const Client &) = delete; diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 08ba04ee875..f783a886877 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -100,7 +100,7 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( unsigned int s3_retry_attempts_, bool enable_s3_requests_logging_, bool for_disk_s3_, - bool s3_aggressive_timeouts_, + bool s3_use_adaptive_timeouts_, const ThrottlerPtr & get_request_throttler_, const ThrottlerPtr & put_request_throttler_, std::function error_report_) @@ -113,7 +113,7 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( , for_disk_s3(for_disk_s3_) , get_request_throttler(get_request_throttler_) , put_request_throttler(put_request_throttler_) - , s3_aggressive_timeouts(s3_aggressive_timeouts_) + , s3_use_adaptive_timeouts(s3_use_adaptive_timeouts_) , error_report(error_report_) { } @@ -160,7 +160,7 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & client_config Poco::Timespan(client_configuration.http_keep_alive_timeout_ms * 1000))) /// flag indicating whether keep-alive is enabled is set to each session upon creation , remote_host_filter(client_configuration.remote_host_filter) , s3_max_redirects(client_configuration.s3_max_redirects) - , s3_aggressive_timeouts(client_configuration.s3_aggressive_timeouts) + , s3_use_adaptive_timeouts(client_configuration.s3_use_adaptive_timeouts) , enable_s3_requests_logging(client_configuration.enable_s3_requests_logging) , for_disk_s3(client_configuration.for_disk_s3) , get_request_throttler(client_configuration.get_request_throttler) @@ -295,7 +295,7 @@ UInt32 extractAttempt(const Aws::String & request_info) ConnectionTimeouts PocoHTTPClient::getTimeouts(Aws::Http::HttpRequest & request) const { - if (!s3_aggressive_timeouts) + if (!s3_use_adaptive_timeouts) return timeouts; const auto & request_info = request.GetHeaderValue(Aws::Http::SDK_REQUEST_HEADER); diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 6eeff431569..9ba5f4ffe64 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -55,7 +55,7 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration size_t http_connection_pool_size = 0; /// See PoolBase::BehaviourOnLimit bool wait_on_pool_size_limit = true; - bool s3_aggressive_timeouts = false; + bool s3_use_adaptive_timeouts = false; std::function error_report; @@ -70,7 +70,7 @@ private: unsigned int s3_retry_attempts, bool enable_s3_requests_logging_, bool for_disk_s3_, - bool s3_aggressive_timeouts_, + bool s3_use_adaptive_timeouts_, const ThrottlerPtr & get_request_throttler_, const ThrottlerPtr & put_request_throttler_, std::function error_report_ @@ -182,7 +182,7 @@ protected: ConnectionTimeouts timeouts; const RemoteHostFilter & remote_host_filter; unsigned int s3_max_redirects; - bool s3_aggressive_timeouts = false; + bool s3_use_adaptive_timeouts = false; bool enable_s3_requests_logging; bool for_disk_s3; diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 63ed84680c9..231efb87e87 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -1330,7 +1330,7 @@ void StorageS3::Configuration::connect(ContextPtr context) auth_settings.no_sign_request.value_or(context->getConfigRef().getBool("s3.no_sign_request", false)), }); - client_with_long_timeout = client->clone(std::nullopt, request_settings.long_request_timeout_ms); + client_with_long_timeout = client->clone(/*override_use_adaptive_timeouts*/ false, request_settings.long_request_timeout_ms); } void StorageS3::processNamedCollectionResult(StorageS3::Configuration & configuration, const NamedCollection & collection) From bb0b6afe14319799028fbd8483b3bc4042e6f951 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 9 Nov 2023 13:12:38 +0100 Subject: [PATCH 047/192] reduce cuncurrent request number to the minio in test_storage_s3 --- tests/integration/test_storage_s3/configs/s3_retry.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_storage_s3/configs/s3_retry.xml b/tests/integration/test_storage_s3/configs/s3_retry.xml index 581fc44c8d4..b7a7bbc8a9b 100644 --- a/tests/integration/test_storage_s3/configs/s3_retry.xml +++ b/tests/integration/test_storage_s3/configs/s3_retry.xml @@ -2,6 +2,7 @@ 10 + 5 From 76d11687a76ede0a0fd080fd76ff04da501e7af6 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 9 Nov 2023 13:12:56 +0100 Subject: [PATCH 048/192] adjuct docs --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 306529c4b96..34ed85c773a 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -4821,7 +4821,7 @@ When set to `false` the metadata files are written with the previous format vers Default value: `false`. -## s3_aggressive_timeouts {#s3_aggressive_timeouts} +## s3_use_adaptive_timeouts {#s3_use_adaptive_timeouts} When set to `true` than for all s3 requests first two attempts are made with low send and receive timeouts. When set to `false` than all attempts are made with identical timeouts. From 8d36fd6e54cc66ab826d80fb6c4ec867fad4b731 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 9 Nov 2023 23:54:31 +0100 Subject: [PATCH 049/192] get rid off of client_with_long_timeout_ptr --- src/Backups/BackupIO_S3.cpp | 5 +- src/Coordination/KeeperSnapshotManagerS3.cpp | 1 - src/Core/Settings.h | 2 +- .../ObjectStorages/S3/S3ObjectStorage.cpp | 62 ++++++------------- src/Disks/ObjectStorages/S3/S3ObjectStorage.h | 14 +---- src/Disks/ObjectStorages/S3/diskSettings.cpp | 2 +- src/IO/ConnectionTimeouts.cpp | 53 ++++++++++++---- src/IO/ConnectionTimeouts.h | 3 +- src/IO/S3/Client.cpp | 14 +---- src/IO/S3/Client.h | 8 +-- src/IO/S3/PocoHTTPClient.cpp | 2 +- src/IO/S3/copyS3File.cpp | 26 +++----- src/IO/S3/copyS3File.h | 7 --- src/IO/S3/tests/gtest_aws_s3_client.cpp | 1 - src/IO/WriteBufferFromS3.cpp | 4 +- src/IO/WriteBufferFromS3.h | 3 - src/IO/tests/gtest_writebuffer_s3.cpp | 1 - src/Storages/StorageS3.cpp | 3 - src/Storages/StorageS3.h | 1 - src/Storages/StorageS3Settings.h | 3 +- 20 files changed, 83 insertions(+), 132 deletions(-) diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 0b700665988..4f83158d07d 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -169,7 +169,6 @@ void BackupReaderS3::copyFileToDisk(const String & path_in_backup, size_t file_s blob_path.size(), mode); copyS3File( - client, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, @@ -231,7 +230,6 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src { LOG_TRACE(log, "Copying file {} from disk {} to S3", src_path, src_disk->getName()); copyS3File( - client, client, /* src_bucket */ blob_path[1], /* src_key= */ blob_path[0], @@ -253,7 +251,7 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src void BackupWriterS3::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) { - copyDataToS3File(create_read_buffer, start_pos, length, client, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {}, + copyDataToS3File(create_read_buffer, start_pos, length, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {}, threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterS3")); } @@ -283,7 +281,6 @@ std::unique_ptr BackupWriterS3::writeFile(const String & file_name) { return std::make_unique( client, - client, // already has long timeout s3_uri.bucket, fs::path(s3_uri.key) / file_name, DBMS_DEFAULT_BUFFER_SIZE, diff --git a/src/Coordination/KeeperSnapshotManagerS3.cpp b/src/Coordination/KeeperSnapshotManagerS3.cpp index 302e05c8418..bedde0d7b39 100644 --- a/src/Coordination/KeeperSnapshotManagerS3.cpp +++ b/src/Coordination/KeeperSnapshotManagerS3.cpp @@ -148,7 +148,6 @@ void KeeperSnapshotManagerS3::uploadSnapshotImpl(const SnapshotFileInfo & snapsh const auto create_writer = [&](const auto & key) { return WriteBufferFromS3( - s3_client->client, s3_client->client, s3_client->uri.bucket, key, diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 3f80c83ff5f..34547aded9c 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -105,7 +105,7 @@ class IColumn; M(Bool, s3_allow_parallel_part_upload, true, "Use multiple threads for s3 multipart upload. It may lead to slightly higher memory usage", 0) \ M(Bool, s3_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \ M(UInt64, s3_retry_attempts, 100, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries", 0) \ - M(UInt64, s3_request_timeout_ms, 3000, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \ + M(UInt64, s3_request_timeout_ms, 30000, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \ M(UInt64, s3_http_connection_pool_size, 1000, "How many reusable open connections to keep per S3 endpoint. Only applies to the S3 table engine and table function, not to S3 disks (for disks, use disk config instead). Global setting, can only be set in config, overriding it per session or per query has no effect.", 0) \ M(Bool, enable_s3_requests_logging, false, "Enable very explicit logging of S3 requests. Makes sense for debug only.", 0) \ M(String, s3queue_default_zookeeper_path, "/clickhouse/s3queue/", "Default zookeeper path prefix for S3Queue engine", 0) \ diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 8a46bfd59d1..75dd405f6aa 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -153,7 +153,7 @@ private: bool S3ObjectStorage::exists(const StoredObject & object) const { auto settings_ptr = s3_settings.get(); - return S3::objectExists(*clients.get()->client, bucket, object.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + return S3::objectExists(*client.get(), bucket, object.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); } std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT @@ -172,7 +172,7 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT (const std::string & path, size_t read_until_position) -> std::unique_ptr { return std::make_unique( - clients.get()->client, + client.get(), bucket, path, version_id, @@ -222,7 +222,7 @@ std::unique_ptr S3ObjectStorage::readObject( /// NOLINT { auto settings_ptr = s3_settings.get(); return std::make_unique( - clients.get()->client, + client.get(), bucket, object.remote_path, version_id, @@ -247,10 +247,8 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN if (write_settings.s3_allow_parallel_part_upload) scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "VFSWrite"); - auto clients_ = clients.get(); return std::make_unique( - clients_->client, - clients_->client_with_long_timeout, + client.get(), bucket, object.remote_path, buf_size, @@ -264,15 +262,12 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN ObjectStorageIteratorPtr S3ObjectStorage::iterate(const std::string & path_prefix) const { auto settings_ptr = s3_settings.get(); - auto client_ptr = clients.get()->client; - - return std::make_shared(bucket, path_prefix, client_ptr, settings_ptr->list_object_keys_size); + return std::make_shared(bucket, path_prefix, client.get(), settings_ptr->list_object_keys_size); } void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const { auto settings_ptr = s3_settings.get(); - auto client_ptr = clients.get()->client; S3::ListObjectsV2Request request; request.SetBucket(bucket); @@ -287,7 +282,7 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet { ProfileEvents::increment(ProfileEvents::S3ListObjects); ProfileEvents::increment(ProfileEvents::DiskS3ListObjects); - outcome = client_ptr->ListObjectsV2(request); + outcome = client.get()->ListObjectsV2(request); throwIfError(outcome); auto result = outcome.GetResult(); @@ -318,14 +313,12 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exists) { - auto client_ptr = clients.get()->client; - ProfileEvents::increment(ProfileEvents::S3DeleteObjects); ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); S3::DeleteObjectRequest request; request.SetBucket(bucket); request.SetKey(object.remote_path); - auto outcome = client_ptr->DeleteObject(request); + auto outcome = client.get()->DeleteObject(request); throwIfUnexpectedError(outcome, if_exists); @@ -344,7 +337,6 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e } else { - auto client_ptr = clients.get()->client; auto settings_ptr = s3_settings.get(); size_t chunk_size_limit = settings_ptr->objects_chunk_size_to_delete; @@ -373,7 +365,7 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e S3::DeleteObjectsRequest request; request.SetBucket(bucket); request.SetDelete(delkeys); - auto outcome = client_ptr->DeleteObjects(request); + auto outcome = client.get()->DeleteObjects(request); throwIfUnexpectedError(outcome, if_exists); @@ -405,7 +397,7 @@ void S3ObjectStorage::removeObjectsIfExist(const StoredObjects & objects) std::optional S3ObjectStorage::tryGetObjectMetadata(const std::string & path) const { auto settings_ptr = s3_settings.get(); - auto object_info = S3::getObjectInfo(*clients.get()->client, bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true, /* throw_on_error= */ false); + auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true, /* throw_on_error= */ false); if (object_info.size == 0 && object_info.last_modification_time == 0 && object_info.metadata.empty()) return {}; @@ -421,7 +413,7 @@ std::optional S3ObjectStorage::tryGetObjectMetadata(const std::s ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path) const { auto settings_ptr = s3_settings.get(); - auto object_info = S3::getObjectInfo(*clients.get()->client, bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true); + auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true); ObjectMetadata result; result.size_bytes = object_info.size; @@ -442,12 +434,12 @@ void S3ObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT /// Shortcut for S3 if (auto * dest_s3 = dynamic_cast(&object_storage_to); dest_s3 != nullptr) { - auto clients_ = clients.get(); + auto client_ = client.get(); auto settings_ptr = s3_settings.get(); - auto size = S3::getObjectSize(*clients_->client, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + auto size = S3::getObjectSize(*client_, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); auto scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "S3ObjStor_copy"); - copyS3File(clients_->client, - clients_->client_with_long_timeout, + copyS3File( + client.get(), bucket, object_from.remote_path, 0, @@ -471,12 +463,11 @@ void S3ObjectStorage::copyObject( // NOLINT const WriteSettings &, std::optional object_to_attributes) { - auto clients_ = clients.get(); + auto client_ = client.get(); auto settings_ptr = s3_settings.get(); - auto size = S3::getObjectSize(*clients_->client, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + auto size = S3::getObjectSize(*client_, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); auto scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "S3ObjStor_copy"); - copyS3File(clients_->client, - clients_->client_with_long_timeout, + copyS3File(client_, bucket, object_from.remote_path, 0, @@ -497,31 +488,25 @@ void S3ObjectStorage::setNewSettings(std::unique_ptr && void S3ObjectStorage::shutdown() { - auto clients_ptr = clients.get(); /// This call stops any next retry attempts for ongoing S3 requests. /// If S3 request is failed and the method below is executed S3 client immediately returns the last failed S3 request outcome. /// If S3 is healthy nothing wrong will be happened and S3 requests will be processed in a regular way without errors. /// This should significantly speed up shutdown process if S3 is unhealthy. - const_cast(*clients_ptr->client).DisableRequestProcessing(); - const_cast(*clients_ptr->client_with_long_timeout).DisableRequestProcessing(); + const_cast(*client.get()).DisableRequestProcessing(); } void S3ObjectStorage::startup() { - auto clients_ptr = clients.get(); - /// Need to be enabled if it was disabled during shutdown() call. - const_cast(*clients_ptr->client).EnableRequestProcessing(); - const_cast(*clients_ptr->client_with_long_timeout).EnableRequestProcessing(); + const_cast(*client.get()).EnableRequestProcessing(); } void S3ObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) { auto new_s3_settings = getSettings(config, config_prefix, context); auto new_client = getClient(config, config_prefix, context, *new_s3_settings); - auto new_clients = std::make_unique(std::move(new_client), *new_s3_settings); s3_settings.set(std::move(new_s3_settings)); - clients.set(std::move(new_clients)); + client.set(std::move(new_client)); } std::unique_ptr S3ObjectStorage::cloneObjectStorage( @@ -536,13 +521,6 @@ std::unique_ptr S3ObjectStorage::cloneObjectStorage( endpoint, object_key_prefix); } -S3ObjectStorage::Clients::Clients(std::shared_ptr client_, const S3ObjectStorageSettings & settings) - : client(std::move(client_)) - , client_with_long_timeout(client->clone( - /*override_use_adaptive_timeouts*/ false, - settings.request_settings.long_request_timeout_ms)) -{} - ObjectStorageKey S3ObjectStorage::generateObjectKeyForPath(const std::string &) const { /// Path to store the new S3 object. diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index 37e491e21dc..7d14482311f 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -39,16 +39,6 @@ struct S3ObjectStorageSettings class S3ObjectStorage : public IObjectStorage { -public: - struct Clients - { - std::shared_ptr client; - std::shared_ptr client_with_long_timeout; - - Clients() = default; - Clients(std::shared_ptr client, const S3ObjectStorageSettings & settings); - }; - private: friend class S3PlainObjectStorage; @@ -63,7 +53,7 @@ private: String object_key_prefix_) : bucket(std::move(bucket_)) , object_key_prefix(std::move(object_key_prefix_)) - , clients(std::make_unique(std::move(client_), *s3_settings_)) + , client(std::move(client_)) , s3_settings(std::move(s3_settings_)) , s3_capabilities(s3_capabilities_) , version_id(std::move(version_id_)) @@ -185,7 +175,7 @@ private: String object_key_prefix; - MultiVersion clients; + MultiVersion client; MultiVersion s3_settings; S3Capabilities s3_capabilities; diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 573fa744ce6..b0384daab2d 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -60,7 +60,7 @@ std::unique_ptr getClient( uri.uri.getScheme()); client_configuration.connectTimeoutMs = config.getUInt(config_prefix + ".connect_timeout_ms", 1000); - client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 3000); + client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 30000); client_configuration.maxConnections = config.getUInt(config_prefix + ".max_connections", 100); client_configuration.endpointOverride = uri.endpoint; client_configuration.http_keep_alive_timeout_ms = config.getUInt( diff --git a/src/IO/ConnectionTimeouts.cpp b/src/IO/ConnectionTimeouts.cpp index a9eebb1a755..90406dcf409 100644 --- a/src/IO/ConnectionTimeouts.cpp +++ b/src/IO/ConnectionTimeouts.cpp @@ -133,22 +133,51 @@ ConnectionTimeouts ConnectionTimeouts::getHTTPTimeouts(const Settings & settings settings.http_receive_timeout); } -ConnectionTimeouts ConnectionTimeouts::aggressiveTimeouts(UInt32 attempt) const +ConnectionTimeouts ConnectionTimeouts::getAdaptiveTimeouts(Aws::Http::HttpMethod method, UInt32 attempt) const { + constexpr size_t first_method_index = size_t(Aws::Http::HttpMethod::HTTP_GET); + constexpr size_t last_method_index = size_t(Aws::Http::HttpMethod::HTTP_PATCH); + constexpr size_t methods_count = last_method_index - first_method_index + 1; + + /// HTTP_POST is used for CompleteMultipartUpload requests. + /// These requests need longer timeout, especially when minio is used + /// The same assumption are made for HTTP_DELETE, HTTP_PATCH + /// That requests are more heavy that HTTP_GET, HTTP_HEAD, HTTP_PUT + + static const UInt32 first_attempt_send_receive_timeouts_ms[methods_count][2] = { + /*HTTP_GET*/ {200, 200}, + /*HTTP_POST*/ {200, 30000}, + /*HTTP_DELETE*/ {200, 1000}, + /*HTTP_PUT*/ {200, 200}, + /*HTTP_HEAD*/ {200, 200}, + /*HTTP_PATCH*/ {200, 1000}, + }; + + static const UInt32 second_attempt_send_receive_timeouts_ms[methods_count][2] = { + /*HTTP_GET*/ {1000, 1000}, + /*HTTP_POST*/ {1000, 30000}, + /*HTTP_DELETE*/ {1000, 10000}, + /*HTTP_PUT*/ {1000, 1000}, + /*HTTP_HEAD*/ {1000, 1000}, + /*HTTP_PATCH*/ {1000, 10000}, + }; + + static_assert(methods_count == 6); + static_assert(sizeof(first_attempt_send_receive_timeouts_ms) == sizeof(second_attempt_send_receive_timeouts_ms)); + static_assert(sizeof(first_attempt_send_receive_timeouts_ms) == methods_count * sizeof(UInt32) * 2); + auto aggressive = *this; + if (attempt > 2) + return aggressive; + + auto timeout_map = first_attempt_send_receive_timeouts_ms; if (attempt == 2) - { - auto one_second = Poco::Timespan(1, 0); - aggressive.send_timeout = saturate(one_second, send_timeout); - aggressive.receive_timeout = saturate(one_second, receive_timeout); - } - else if (attempt == 1) - { - auto two_hundred_ms = Poco::Timespan(0, 200 * 1000); - aggressive.send_timeout = saturate(two_hundred_ms, send_timeout); - aggressive.receive_timeout = saturate(two_hundred_ms, receive_timeout); - } + timeout_map = second_attempt_send_receive_timeouts_ms; + + const size_t method_index = size_t(method) - first_method_index; + aggressive.send_timeout = saturate(Poco::Timespan(timeout_map[method_index][0]), send_timeout); + aggressive.receive_timeout = saturate(Poco::Timespan(timeout_map[method_index][1]), receive_timeout); return aggressive; } diff --git a/src/IO/ConnectionTimeouts.h b/src/IO/ConnectionTimeouts.h index 17ee1907d89..0ef133c8378 100644 --- a/src/IO/ConnectionTimeouts.h +++ b/src/IO/ConnectionTimeouts.h @@ -4,6 +4,7 @@ #include #include +#include namespace DB { @@ -68,7 +69,7 @@ struct ConnectionTimeouts static ConnectionTimeouts getTCPTimeoutsWithFailover(const Settings & settings); static ConnectionTimeouts getHTTPTimeouts(const Settings & settings, Poco::Timespan http_keep_alive_timeout); - ConnectionTimeouts aggressiveTimeouts(UInt32 attempt) const; + ConnectionTimeouts getAdaptiveTimeouts(Aws::Http::HttpMethod method, UInt32 attempt) const; }; } diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 90806852c1e..4630e68fbb6 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -118,19 +118,9 @@ std::unique_ptr Client::create( new Client(max_redirects_, std::move(sse_kms_config_), credentials_provider, client_configuration, sign_payloads, use_virtual_addressing)); } -std::unique_ptr Client::clone( - std::optional override_use_adaptive_timeouts, - std::optional override_request_timeout_ms) const +std::unique_ptr Client::clone() const { - PocoHTTPClientConfiguration new_configuration = client_configuration; - - if (override_request_timeout_ms.has_value()) - new_configuration.requestTimeoutMs = *override_request_timeout_ms; - - if (override_use_adaptive_timeouts.has_value()) - new_configuration.s3_use_adaptive_timeouts = *override_use_adaptive_timeouts; - - return std::unique_ptr(new Client(*this, new_configuration)); + return std::unique_ptr(new Client(*this, client_configuration)); } namespace diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index be7235eb9f1..5ad57a9d827 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -118,13 +118,7 @@ public: Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy sign_payloads, bool use_virtual_addressing); - /// Create a client with adjusted settings: - /// * override_request_timeout_ms is used to increase timeout for CompleteMultipartUploadRequest - /// because it often sits idle for 10 seconds: https://github.com/ClickHouse/ClickHouse/pull/42321 - /// * s3_use_adaptive_timeouts is used to turn off s3_use_adaptive_timeouts feature for CompleteMultipartUploadRequest - std::unique_ptr clone( - std::optional override_use_adaptive_timeouts = std::nullopt, - std::optional override_request_timeout_ms = std::nullopt) const; + std::unique_ptr clone() const; Client & operator=(const Client &) = delete; diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index f783a886877..b26c36f8029 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -300,7 +300,7 @@ ConnectionTimeouts PocoHTTPClient::getTimeouts(Aws::Http::HttpRequest & request) const auto & request_info = request.GetHeaderValue(Aws::Http::SDK_REQUEST_HEADER); auto attempt = extractAttempt(request_info); - return timeouts.aggressiveTimeouts(attempt); + return timeouts.getAdaptiveTimeouts(request.GetMethod(), attempt); } void PocoHTTPClient::makeRequestInternal( diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index a16a1a41505..30da1c580c1 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -53,7 +53,6 @@ namespace public: UploadHelper( const std::shared_ptr & client_ptr_, - const std::shared_ptr & client_with_long_timeout_ptr_, const String & dest_bucket_, const String & dest_key_, const S3Settings::RequestSettings & request_settings_, @@ -62,7 +61,6 @@ namespace bool for_disk_s3_, const Poco::Logger * log_) : client_ptr(client_ptr_) - , client_with_long_timeout_ptr(client_with_long_timeout_ptr_) , dest_bucket(dest_bucket_) , dest_key(dest_key_) , request_settings(request_settings_) @@ -78,7 +76,6 @@ namespace protected: std::shared_ptr client_ptr; - std::shared_ptr client_with_long_timeout_ptr; const String & dest_bucket; const String & dest_key; const S3Settings::RequestSettings & request_settings; @@ -179,7 +176,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); - auto outcome = client_with_long_timeout_ptr->CompleteMultipartUpload(request); + auto outcome = client_ptr->CompleteMultipartUpload(request); if (outcome.IsSuccess()) { @@ -433,14 +430,13 @@ namespace size_t offset_, size_t size_, const std::shared_ptr & client_ptr_, - const std::shared_ptr & client_with_long_timeout_ptr_, const String & dest_bucket_, const String & dest_key_, const S3Settings::RequestSettings & request_settings_, const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, bool for_disk_s3_) - : UploadHelper(client_ptr_, client_with_long_timeout_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyDataToS3File")) + : UploadHelper(client_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyDataToS3File")) , create_read_buffer(create_read_buffer_) , offset(offset_) , size(size_) @@ -602,7 +598,6 @@ namespace public: CopyFileHelper( const std::shared_ptr & client_ptr_, - const std::shared_ptr & client_with_long_timeout_ptr_, const String & src_bucket_, const String & src_key_, size_t src_offset_, @@ -614,7 +609,7 @@ namespace const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, bool for_disk_s3_) - : UploadHelper(client_ptr_, client_with_long_timeout_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyS3File")) + : UploadHelper(client_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyS3File")) , src_bucket(src_bucket_) , src_key(src_key_) , offset(src_offset_) @@ -677,7 +672,7 @@ namespace /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 request.SetContentType("binary/octet-stream"); - client_with_long_timeout_ptr->setKMSHeaders(request); + client_ptr->setKMSHeaders(request); } void processCopyRequest(const S3::CopyObjectRequest & request) @@ -689,7 +684,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3CopyObject); - auto outcome = client_with_long_timeout_ptr->CopyObject(request); + auto outcome = client_ptr->CopyObject(request); if (outcome.IsSuccess()) { LOG_TRACE( @@ -714,7 +709,6 @@ namespace offset, size, client_ptr, - client_with_long_timeout_ptr, dest_bucket, dest_key, request_settings, @@ -788,7 +782,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3UploadPartCopy); - auto outcome = client_with_long_timeout_ptr->UploadPartCopy(req); + auto outcome = client_ptr->UploadPartCopy(req); if (!outcome.IsSuccess()) { abortMultipartUpload(); @@ -806,7 +800,6 @@ void copyDataToS3File( size_t offset, size_t size, const std::shared_ptr & dest_s3_client, - const std::shared_ptr & dest_s3_client_with_long_timeout, const String & dest_bucket, const String & dest_key, const S3Settings::RequestSettings & settings, @@ -814,14 +807,13 @@ void copyDataToS3File( ThreadPoolCallbackRunner schedule, bool for_disk_s3) { - CopyDataToFileHelper helper{create_read_buffer, offset, size, dest_s3_client, dest_s3_client_with_long_timeout, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3}; + CopyDataToFileHelper helper{create_read_buffer, offset, size, dest_s3_client, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3}; helper.performCopy(); } void copyS3File( const std::shared_ptr & s3_client, - const std::shared_ptr & s3_client_with_long_timeout, const String & src_bucket, const String & src_key, size_t src_offset, @@ -836,7 +828,7 @@ void copyS3File( { if (settings.allow_native_copy) { - CopyFileHelper helper{s3_client, s3_client_with_long_timeout, src_bucket, src_key, src_offset, src_size, dest_bucket, dest_key, settings, read_settings, object_metadata, schedule, for_disk_s3}; + CopyFileHelper helper{s3_client, src_bucket, src_key, src_offset, src_size, dest_bucket, dest_key, settings, read_settings, object_metadata, schedule, for_disk_s3}; helper.performCopy(); } else @@ -845,7 +837,7 @@ void copyS3File( { return std::make_unique(s3_client, src_bucket, src_key, "", settings, read_settings); }; - copyDataToS3File(create_read_buffer, src_offset, src_size, s3_client, s3_client_with_long_timeout, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3); + copyDataToS3File(create_read_buffer, src_offset, src_size, s3_client, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3); } } diff --git a/src/IO/S3/copyS3File.h b/src/IO/S3/copyS3File.h index 1bcbfd7735e..33e22fdfba2 100644 --- a/src/IO/S3/copyS3File.h +++ b/src/IO/S3/copyS3File.h @@ -27,15 +27,9 @@ using CreateReadBuffer = std::function()>; /// because it is a known issue, it is fallbacks to read-write copy /// (copyDataToS3File()). /// -/// s3_client_with_long_timeout (may be equal to s3_client) is used for native copy and -/// CompleteMultipartUpload requests. These requests need longer timeout because S3 servers often -/// block on them for multiple seconds without sending or receiving data from us (maybe the servers -/// are copying data internally, or maybe throttling, idk). -/// /// read_settings - is used for throttling in case of native copy is not possible void copyS3File( const std::shared_ptr & s3_client, - const std::shared_ptr & s3_client_with_long_timeout, const String & src_bucket, const String & src_key, size_t src_offset, @@ -58,7 +52,6 @@ void copyDataToS3File( size_t offset, size_t size, const std::shared_ptr & dest_s3_client, - const std::shared_ptr & dest_s3_client_with_long_timeout, const String & dest_bucket, const String & dest_key, const S3Settings::RequestSettings & settings, diff --git a/src/IO/S3/tests/gtest_aws_s3_client.cpp b/src/IO/S3/tests/gtest_aws_s3_client.cpp index c42f14e9a53..d4b9a017398 100644 --- a/src/IO/S3/tests/gtest_aws_s3_client.cpp +++ b/src/IO/S3/tests/gtest_aws_s3_client.cpp @@ -91,7 +91,6 @@ void doWriteRequest(std::shared_ptr client, const DB::S3:: DB::S3Settings::RequestSettings request_settings; request_settings.max_unexpected_write_error_retries = max_unexpected_write_error_retries; DB::WriteBufferFromS3 write_buffer( - client, client, uri.bucket, uri.key, diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index e1b9c17efe9..62d0c80f1f2 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -77,7 +77,6 @@ struct WriteBufferFromS3::PartData WriteBufferFromS3::WriteBufferFromS3( std::shared_ptr client_ptr_, - std::shared_ptr client_with_long_timeout_ptr_, const String & bucket_, const String & key_, size_t buf_size_, @@ -92,7 +91,6 @@ WriteBufferFromS3::WriteBufferFromS3( , upload_settings(request_settings.getUploadSettings()) , write_settings(write_settings_) , client_ptr(std::move(client_ptr_)) - , client_with_long_timeout_ptr(std::move(client_with_long_timeout_ptr_)) , object_metadata(std::move(object_metadata_)) , buffer_allocation_policy(ChooseBufferPolicy(upload_settings)) , task_tracker( @@ -566,7 +564,7 @@ void WriteBufferFromS3::completeMultipartUpload() ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Stopwatch watch; - auto outcome = client_with_long_timeout_ptr->CompleteMultipartUpload(req); + auto outcome = client_ptr->CompleteMultipartUpload(req); watch.stop(); ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Microseconds, watch.elapsedMicroseconds()); diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 95148c49779..590342cc997 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -30,8 +30,6 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase public: WriteBufferFromS3( std::shared_ptr client_ptr_, - /// for CompleteMultipartUploadRequest, because it blocks on recv() for a few seconds on big uploads - std::shared_ptr client_with_long_timeout_ptr_, const String & bucket_, const String & key_, size_t buf_size_, @@ -90,7 +88,6 @@ private: const S3Settings::RequestSettings::PartUploadSettings & upload_settings; const WriteSettings write_settings; const std::shared_ptr client_ptr; - const std::shared_ptr client_with_long_timeout_ptr; const std::optional> object_metadata; Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3"); LogSeriesLimiterPtr limitedLog = std::make_shared(log, 1, 5); diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp index 21bdd9a6f26..c82f97f8b20 100644 --- a/src/IO/tests/gtest_writebuffer_s3.cpp +++ b/src/IO/tests/gtest_writebuffer_s3.cpp @@ -549,7 +549,6 @@ public: getAsyncPolicy().setAutoExecute(false); return std::make_unique( - client, client, bucket, file_name, diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 231efb87e87..b0cd40a2e05 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -824,7 +824,6 @@ public: write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique( configuration_.client, - configuration_.client_with_long_timeout, bucket, key, DBMS_DEFAULT_BUFFER_SIZE, @@ -1329,8 +1328,6 @@ void StorageS3::Configuration::connect(ContextPtr context) context->getConfigRef().getUInt64("s3.expiration_window_seconds", S3::DEFAULT_EXPIRATION_WINDOW_SECONDS)), auth_settings.no_sign_request.value_or(context->getConfigRef().getBool("s3.no_sign_request", false)), }); - - client_with_long_timeout = client->clone(/*override_use_adaptive_timeouts*/ false, request_settings.long_request_timeout_ms); } void StorageS3::processNamedCollectionResult(StorageS3::Configuration & configuration, const NamedCollection & collection) diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 3330ac6c210..3f35c578e19 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -311,7 +311,6 @@ public: HTTPHeaderEntries headers_from_ast; std::shared_ptr client; - std::shared_ptr client_with_long_timeout; std::vector keys; }; diff --git a/src/Storages/StorageS3Settings.h b/src/Storages/StorageS3Settings.h index e3d577ca0b3..728972c948c 100644 --- a/src/Storages/StorageS3Settings.h +++ b/src/Storages/StorageS3Settings.h @@ -69,8 +69,7 @@ struct S3Settings ThrottlerPtr get_request_throttler; ThrottlerPtr put_request_throttler; size_t retry_attempts = 10; - size_t request_timeout_ms = 3000; - size_t long_request_timeout_ms = 30000; // TODO: Take this from config like request_timeout_ms + size_t request_timeout_ms = 30000; bool allow_native_copy = true; bool throw_on_zero_files_match = false; From 3075bd97450d42f00320b18c1b177fd700a19bec Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Fri, 10 Nov 2023 15:15:24 +0100 Subject: [PATCH 050/192] track clickhouse high level retries --- base/poco/Net/src/HTTPSession.cpp | 4 +- src/IO/ConnectionTimeouts.cpp | 101 ++++++++++++------ src/IO/ConnectionTimeouts.h | 3 +- src/IO/HTTPCommon.cpp | 12 +-- src/IO/HTTPCommon.h | 2 + src/IO/ReadBufferFromS3.cpp | 24 +++-- src/IO/ReadBufferFromS3.h | 4 +- src/IO/S3/PocoHTTPClient.cpp | 89 ++++++++------- src/IO/S3/PocoHTTPClient.h | 2 +- src/IO/S3/tests/gtest_aws_s3_client.cpp | 2 + .../configs/inf_s3_retries.xml | 2 +- .../configs/s3_retries.xml | 2 +- .../configs/storage_conf.xml | 1 + .../test_checking_s3_blobs_paranoid/test.py | 16 +-- .../test_storage_s3/configs/s3_retry.xml | 1 + .../s3_mocks/unstable_server.py | 17 ++- tests/integration/test_storage_s3/test.py | 9 ++ 17 files changed, 181 insertions(+), 110 deletions(-) diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index d30f5590280..9ebbd7d04cd 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -95,7 +95,7 @@ void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco { _connectionTimeout = connectionTimeout; - if (_sendTimeout != sendTimeout) + if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) { _sendTimeout = sendTimeout; @@ -103,7 +103,7 @@ void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco _socket.setSendTimeout(_sendTimeout); } - if (_receiveTimeout != receiveTimeout) + if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) { _receiveTimeout = receiveTimeout; diff --git a/src/IO/ConnectionTimeouts.cpp b/src/IO/ConnectionTimeouts.cpp index 90406dcf409..970afc75ec3 100644 --- a/src/IO/ConnectionTimeouts.cpp +++ b/src/IO/ConnectionTimeouts.cpp @@ -133,51 +133,84 @@ ConnectionTimeouts ConnectionTimeouts::getHTTPTimeouts(const Settings & settings settings.http_receive_timeout); } -ConnectionTimeouts ConnectionTimeouts::getAdaptiveTimeouts(Aws::Http::HttpMethod method, UInt32 attempt) const +class SendReceiveTimeoutsForFirstAttempt { - constexpr size_t first_method_index = size_t(Aws::Http::HttpMethod::HTTP_GET); - constexpr size_t last_method_index = size_t(Aws::Http::HttpMethod::HTTP_PATCH); - constexpr size_t methods_count = last_method_index - first_method_index + 1; +private: + static constexpr size_t known_methods_count = 6; + using KnownMethodsArray = std::array; + static const KnownMethodsArray known_methods; - /// HTTP_POST is used for CompleteMultipartUpload requests. - /// These requests need longer timeout, especially when minio is used + /// HTTP_POST is used for CompleteMultipartUpload requests. Its latency could be high. + /// These requests need longer timeout, especially when minio is used. /// The same assumption are made for HTTP_DELETE, HTTP_PATCH /// That requests are more heavy that HTTP_GET, HTTP_HEAD, HTTP_PUT - static const UInt32 first_attempt_send_receive_timeouts_ms[methods_count][2] = { - /*HTTP_GET*/ {200, 200}, - /*HTTP_POST*/ {200, 30000}, - /*HTTP_DELETE*/ {200, 1000}, - /*HTTP_PUT*/ {200, 200}, - /*HTTP_HEAD*/ {200, 200}, - /*HTTP_PATCH*/ {200, 1000}, + static constexpr Poco::Timestamp::TimeDiff first_byte_ms[known_methods_count][2] = + { + /* GET */ {200, 200}, + /* POST */ {200, 200}, + /* DELETE */ {200, 200}, + /* PUT */ {200, 200}, + /* HEAD */ {200, 200}, + /* PATCH */ {200, 200}, }; - static const UInt32 second_attempt_send_receive_timeouts_ms[methods_count][2] = { - /*HTTP_GET*/ {1000, 1000}, - /*HTTP_POST*/ {1000, 30000}, - /*HTTP_DELETE*/ {1000, 10000}, - /*HTTP_PUT*/ {1000, 1000}, - /*HTTP_HEAD*/ {1000, 1000}, - /*HTTP_PATCH*/ {1000, 10000}, + static constexpr Poco::Timestamp::TimeDiff rest_bytes_ms[known_methods_count][2] = + { + /* GET */ {500, 500}, + /* POST */ {1000, 30000}, + /* DELETE */ {1000, 10000}, + /* PUT */ {1000, 3000}, + /* HEAD */ {500, 500}, + /* PATCH */ {1000, 10000}, }; - static_assert(methods_count == 6); - static_assert(sizeof(first_attempt_send_receive_timeouts_ms) == sizeof(second_attempt_send_receive_timeouts_ms)); - static_assert(sizeof(first_attempt_send_receive_timeouts_ms) == methods_count * sizeof(UInt32) * 2); + static_assert(sizeof(first_byte_ms) == sizeof(rest_bytes_ms)); + static_assert(sizeof(first_byte_ms) == known_methods_count * sizeof(Poco::Timestamp::TimeDiff) * 2); + + static size_t getMethodIndex(const String & method) + { + KnownMethodsArray::const_iterator it = std::find(known_methods.begin(), known_methods.end(), method); + chassert(it != known_methods.end()); + if (it == known_methods.end()) + return 0; + return std::distance(known_methods.begin(), it); + } + +public: + static std::pair getSendReceiveTimeout(const String & method, bool first_byte) + { + auto idx = getMethodIndex(method); + + if (first_byte) + return std::make_pair( + Poco::Timespan(first_byte_ms[idx][0] * 1000), + Poco::Timespan(first_byte_ms[idx][1] * 1000) + ); + + return std::make_pair( + Poco::Timespan(rest_bytes_ms[idx][0] * 1000), + Poco::Timespan(rest_bytes_ms[idx][1] * 1000) + ); + } +}; + +const SendReceiveTimeoutsForFirstAttempt::KnownMethodsArray SendReceiveTimeoutsForFirstAttempt::known_methods = +{ + "GET", "POST", "DELETE", "PUT", "HEAD", "PATCH" +}; + + +ConnectionTimeouts ConnectionTimeouts::getAdaptiveTimeouts(const String & method, bool first_attempt, bool first_byte) const +{ + if (!first_attempt) + return *this; + + auto [send, recv] = SendReceiveTimeoutsForFirstAttempt::getSendReceiveTimeout(method, first_byte); auto aggressive = *this; - - if (attempt > 2) - return aggressive; - - auto timeout_map = first_attempt_send_receive_timeouts_ms; - if (attempt == 2) - timeout_map = second_attempt_send_receive_timeouts_ms; - - const size_t method_index = size_t(method) - first_method_index; - aggressive.send_timeout = saturate(Poco::Timespan(timeout_map[method_index][0]), send_timeout); - aggressive.receive_timeout = saturate(Poco::Timespan(timeout_map[method_index][1]), receive_timeout); + aggressive.send_timeout = saturate(send, send_timeout); + aggressive.receive_timeout = saturate(recv, receive_timeout); return aggressive; } diff --git a/src/IO/ConnectionTimeouts.h b/src/IO/ConnectionTimeouts.h index 0ef133c8378..aabebdb836d 100644 --- a/src/IO/ConnectionTimeouts.h +++ b/src/IO/ConnectionTimeouts.h @@ -4,7 +4,6 @@ #include #include -#include namespace DB { @@ -69,7 +68,7 @@ struct ConnectionTimeouts static ConnectionTimeouts getTCPTimeoutsWithFailover(const Settings & settings); static ConnectionTimeouts getHTTPTimeouts(const Settings & settings, Poco::Timespan http_keep_alive_timeout); - ConnectionTimeouts getAdaptiveTimeouts(Aws::Http::HttpMethod method, UInt32 attempt) const; + ConnectionTimeouts getAdaptiveTimeouts(const String & method, bool first_attempt, bool first_byte) const; }; } diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 65ffa51a466..cce394c67c9 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -50,12 +50,6 @@ namespace ErrorCodes namespace { - void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts) - { - session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout); - session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); - } - Poco::Net::HTTPClientSession::ProxyConfig proxyConfigurationToPocoProxyConfig(const ProxyConfiguration & proxy_configuration) { Poco::Net::HTTPClientSession::ProxyConfig poco_proxy_config; @@ -359,6 +353,12 @@ namespace }; } +void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts) +{ + session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout); + session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); +} + void setResponseDefaultHeaders(HTTPServerResponse & response, size_t keep_alive_timeout) { if (!response.getKeepAlive()) diff --git a/src/IO/HTTPCommon.h b/src/IO/HTTPCommon.h index de62b5d5c16..c9968fc6915 100644 --- a/src/IO/HTTPCommon.h +++ b/src/IO/HTTPCommon.h @@ -113,4 +113,6 @@ std::istream * receiveResponse( void assertResponseIsOk( const Poco::Net::HTTPRequest & request, Poco::Net::HTTPResponse & response, std::istream & istr, bool allow_redirects = false); + +void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts); } diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index f19978ccb47..c9c9319c44c 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -167,9 +167,9 @@ bool ReadBufferFromS3::nextImpl() } size_t sleep_time_with_backoff_milliseconds = 100; - for (size_t attempt = 0; !next_result; ++attempt) + for (size_t attempt = 1; !next_result; ++attempt) { - bool last_attempt = attempt + 1 >= request_settings.max_single_read_retries; + bool last_attempt = attempt >= request_settings.max_single_read_retries; ProfileEventTimeIncrement watch(ProfileEvents::ReadBufferFromS3Microseconds); @@ -177,7 +177,7 @@ bool ReadBufferFromS3::nextImpl() { if (!impl) { - impl = initialize(); + impl = initialize(attempt); if (use_external_buffer) { @@ -232,9 +232,9 @@ size_t ReadBufferFromS3::readBigAt(char * to, size_t n, size_t range_begin, cons { size_t initial_n = n; size_t sleep_time_with_backoff_milliseconds = 100; - for (size_t attempt = 0; n > 0; ++attempt) + for (size_t attempt = 1; n > 0; ++attempt) { - bool last_attempt = attempt + 1 >= request_settings.max_single_read_retries; + bool last_attempt = attempt >= request_settings.max_single_read_retries; size_t bytes_copied = 0; ProfileEventTimeIncrement watch(ProfileEvents::ReadBufferFromS3Microseconds); @@ -266,7 +266,7 @@ size_t ReadBufferFromS3::readBigAt(char * to, size_t n, size_t range_begin, cons try { - result = sendRequest(range_begin, range_begin + n - 1); + result = sendRequest(attempt, range_begin, range_begin + n - 1); std::istream & istr = result->GetBody(); copyFromIStreamWithProgressCallback(istr, to, n, progress_callback, &bytes_copied); @@ -304,8 +304,8 @@ bool ReadBufferFromS3::processException(Poco::Exception & e, size_t read_offset, LOG_DEBUG( log, "Caught exception while reading S3 object. Bucket: {}, Key: {}, Version: {}, Offset: {}, " - "Attempt: {}, Message: {}", - bucket, key, version_id.empty() ? "Latest" : version_id, read_offset, attempt, e.message()); + "Attempt: {}/{}, Message: {}", + bucket, key, version_id.empty() ? "Latest" : version_id, read_offset, attempt, request_settings.max_single_read_retries, e.message()); if (auto * s3_exception = dynamic_cast(&e)) @@ -463,7 +463,7 @@ ReadBufferFromS3::~ReadBufferFromS3() } } -std::unique_ptr ReadBufferFromS3::initialize() +std::unique_ptr ReadBufferFromS3::initialize(size_t attempt) { resetSessionIfNeeded(readAllRangeSuccessfully(), read_result); read_all_range_successfully = false; @@ -475,13 +475,13 @@ std::unique_ptr ReadBufferFromS3::initialize() if (read_until_position && offset >= read_until_position) throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read beyond right offset ({} > {})", offset, read_until_position - 1); - read_result = sendRequest(offset, read_until_position ? std::make_optional(read_until_position - 1) : std::nullopt); + read_result = sendRequest(attempt, offset, read_until_position ? std::make_optional(read_until_position - 1) : std::nullopt); size_t buffer_size = use_external_buffer ? 0 : read_settings.remote_fs_buffer_size; return std::make_unique(read_result->GetBody(), buffer_size); } -Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t range_begin, std::optional range_end_incl) const +Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t attempt, size_t range_begin, std::optional range_end_incl) const { S3::GetObjectRequest req; req.SetBucket(bucket); @@ -489,6 +489,8 @@ Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t range_begin if (!version_id.empty()) req.SetVersionId(version_id); + req.SetAdditionalCustomHeaderValue("clickhouse-request", fmt::format("attempt={}", attempt)); + if (range_end_incl) { req.SetRange(fmt::format("bytes={}-{}", range_begin, *range_end_incl)); diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h index 0835e52a5b2..101e25f8b43 100644 --- a/src/IO/ReadBufferFromS3.h +++ b/src/IO/ReadBufferFromS3.h @@ -79,7 +79,7 @@ public: bool supportsReadAt() override { return true; } private: - std::unique_ptr initialize(); + std::unique_ptr initialize(size_t attempt); /// If true, if we destroy impl now, no work was wasted. Just for metrics. bool atEndOfRequestedRangeGuess(); @@ -88,7 +88,7 @@ private: /// Returns true if the error looks retriable. bool processException(Poco::Exception & e, size_t read_offset, size_t attempt) const; - Aws::S3::Model::GetObjectResult sendRequest(size_t range_begin, std::optional range_end_incl) const; + Aws::S3::Model::GetObjectResult sendRequest(size_t attempt, size_t range_begin, std::optional range_end_incl) const; bool readAllRangeSuccessfully() const; diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index b26c36f8029..904e2324145 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -272,35 +272,36 @@ void PocoHTTPClient::addMetric(const Aws::Http::HttpRequest & request, S3MetricT ProfileEvents::increment(disk_s3_events_map[static_cast(type)][static_cast(kind)], amount); } -UInt32 extractAttempt(const Aws::String & request_info) +String extractAttemptFromInfo(const Aws::String & request_info) { static auto key = Aws::String("attempt="); auto key_begin = request_info.find(key, 0); if (key_begin == Aws::String::npos) - return 1; + return "1"; auto val_begin = key_begin + key.size(); auto val_end = request_info.find(';', val_begin); if (val_end == Aws::String::npos) val_end = request_info.size(); - Aws::String value = request_info.substr(val_begin, val_end-val_begin); - - UInt32 attempt = 1; - ReadBufferFromString buf(value); - readIntText(attempt, buf); - return attempt; + return request_info.substr(val_begin, val_end-val_begin); } -ConnectionTimeouts PocoHTTPClient::getTimeouts(Aws::Http::HttpRequest & request) const +String getOrEmpty(const Aws::Http::HeaderValueCollection & map, const String & key) +{ + auto it = map.find(key); + if (it == map.end()) + return {}; + return it->second; +} + +ConnectionTimeouts PocoHTTPClient::getTimeouts(const String & method, bool first_attempt, bool first_byte) const { if (!s3_use_adaptive_timeouts) return timeouts; - const auto & request_info = request.GetHeaderValue(Aws::Http::SDK_REQUEST_HEADER); - auto attempt = extractAttempt(request_info); - return timeouts.getAdaptiveTimeouts(request.GetMethod(), attempt); + return timeouts.getAdaptiveTimeouts(method, first_attempt, first_byte); } void PocoHTTPClient::makeRequestInternal( @@ -317,6 +318,25 @@ void PocoHTTPClient::makeRequestInternal( makeRequestInternalImpl(request, request_configuration, response, readLimiter, writeLimiter); } +String getMethod(const Aws::Http::HttpRequest & request) +{ + switch (request.GetMethod()) + { + case Aws::Http::HttpMethod::HTTP_GET: + return Poco::Net::HTTPRequest::HTTP_GET; + case Aws::Http::HttpMethod::HTTP_POST: + return Poco::Net::HTTPRequest::HTTP_POST; + case Aws::Http::HttpMethod::HTTP_DELETE: + return Poco::Net::HTTPRequest::HTTP_DELETE; + case Aws::Http::HttpMethod::HTTP_PUT: + return Poco::Net::HTTPRequest::HTTP_PUT; + case Aws::Http::HttpMethod::HTTP_HEAD: + return Poco::Net::HTTPRequest::HTTP_HEAD; + case Aws::Http::HttpMethod::HTTP_PATCH: + return Poco::Net::HTTPRequest::HTTP_PATCH; + } +} + template void PocoHTTPClient::makeRequestInternalImpl( Aws::Http::HttpRequest & request, @@ -330,9 +350,14 @@ void PocoHTTPClient::makeRequestInternalImpl( Poco::Logger * log = &Poco::Logger::get("AWSClient"); auto uri = request.GetUri().GetURIString(); + auto method = getMethod(request); + + auto sdk_attempt = extractAttemptFromInfo(getOrEmpty(request.GetHeaders(), Aws::Http::SDK_REQUEST_HEADER)); + auto ch_attempt = extractAttemptFromInfo(getOrEmpty(request.GetHeaders(), "clickhouse-request")); + bool first_attempt = ch_attempt == "1" && sdk_attempt == "1"; if (enable_s3_requests_logging) - LOG_TEST(log, "Make request to: {}", uri); + LOG_TEST(log, "Make request to: {}, aws sdk attempt: {}, clickhouse attempt: {}", uri, sdk_attempt, ch_attempt); switch (request.GetMethod()) { @@ -383,17 +408,17 @@ void PocoHTTPClient::makeRequestInternalImpl( /// This can lead to request signature difference on S3 side. if constexpr (pooled) session = makePooledHTTPSession( - target_uri, getTimeouts(request), http_connection_pool_size, wait_on_pool_size_limit, proxy_configuration); + target_uri, getTimeouts(method, first_attempt), http_connection_pool_size, wait_on_pool_size_limit, proxy_configuration); else - session = makeHTTPSession(target_uri, getTimeouts(request), proxy_configuration); + session = makeHTTPSession(target_uri, getTimeouts(method, first_attempt), proxy_configuration); } else { if constexpr (pooled) session = makePooledHTTPSession( - target_uri, getTimeouts(request), http_connection_pool_size, wait_on_pool_size_limit); + target_uri, getTimeouts(method, first_attempt), http_connection_pool_size, wait_on_pool_size_limit); else - session = makeHTTPSession(target_uri, getTimeouts(request)); + session = makeHTTPSession(target_uri, getTimeouts(method, first_attempt)); } /// In case of error this address will be written to logs @@ -427,28 +452,7 @@ void PocoHTTPClient::makeRequestInternalImpl( path_and_query = "/"; poco_request.setURI(path_and_query); - - switch (request.GetMethod()) - { - case Aws::Http::HttpMethod::HTTP_GET: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_GET); - break; - case Aws::Http::HttpMethod::HTTP_POST: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_POST); - break; - case Aws::Http::HttpMethod::HTTP_DELETE: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_DELETE); - break; - case Aws::Http::HttpMethod::HTTP_PUT: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_PUT); - break; - case Aws::Http::HttpMethod::HTTP_HEAD: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_HEAD); - break; - case Aws::Http::HttpMethod::HTTP_PATCH: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_PATCH); - break; - } + poco_request.setMethod(method); /// Headers coming from SDK are lower-cased. for (const auto & [header_name, header_value] : request.GetHeaders()) @@ -473,6 +477,7 @@ void PocoHTTPClient::makeRequestInternalImpl( request.GetContentBody()->clear(); request.GetContentBody()->seekg(0); + setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); auto size = Poco::StreamCopier::copyStream(*request.GetContentBody(), request_body_stream); if (enable_s3_requests_logging) LOG_TEST(log, "Written {} bytes to request body", size); @@ -482,6 +487,8 @@ void PocoHTTPClient::makeRequestInternalImpl( LOG_TEST(log, "Receiving response..."); auto & response_body_stream = session->receiveResponse(poco_response); + setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); + watch.stop(); addMetric(request, S3MetricType::Microseconds, watch.elapsedMicroseconds()); @@ -533,6 +540,7 @@ void PocoHTTPClient::makeRequestInternalImpl( /// Request is successful but for some special requests we can have actual error message in body if (status_code >= SUCCESS_RESPONSE_MIN && status_code <= SUCCESS_RESPONSE_MAX && checkRequestCanReturn2xxAndErrorInBody(request)) { + /// reading the full response std::string response_string((std::istreambuf_iterator(response_body_stream)), std::istreambuf_iterator()); @@ -547,7 +555,6 @@ void PocoHTTPClient::makeRequestInternalImpl( addMetric(request, S3MetricType::Errors); if (error_report) error_report(proxy_configuration); - } /// Set response from string @@ -566,6 +573,8 @@ void PocoHTTPClient::makeRequestInternalImpl( if (status_code >= 500 && error_report) error_report(proxy_configuration); } + + /// expose stream, after that client reads data from that stream without built-in retries response->SetResponseBody(response_body_stream, session); } diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 9ba5f4ffe64..14c4fec5dd7 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -171,7 +171,7 @@ private: Aws::Utils::RateLimits::RateLimiterInterface * readLimiter, Aws::Utils::RateLimits::RateLimiterInterface * writeLimiter) const; - ConnectionTimeouts getTimeouts(Aws::Http::HttpRequest & request) const; + ConnectionTimeouts getTimeouts(const String & method, bool first_attempt, bool first_byte = true) const; protected: static S3MetricKind getMetricKind(const Aws::Http::HttpRequest & request); diff --git a/src/IO/S3/tests/gtest_aws_s3_client.cpp b/src/IO/S3/tests/gtest_aws_s3_client.cpp index d4b9a017398..bff9ca6fa7b 100644 --- a/src/IO/S3/tests/gtest_aws_s3_client.cpp +++ b/src/IO/S3/tests/gtest_aws_s3_client.cpp @@ -170,6 +170,7 @@ TEST(IOTestAwsS3Client, AppendExtraSSECHeadersRead) "authorization: ... SignedHeaders=" "amz-sdk-invocation-id;" "amz-sdk-request;" + "clickhouse-request;" "content-type;" "host;" "x-amz-api-version;" @@ -215,6 +216,7 @@ TEST(IOTestAwsS3Client, AppendExtraSSEKMSHeadersRead) "authorization: ... SignedHeaders=" "amz-sdk-invocation-id;" "amz-sdk-request;" + "clickhouse-request;" "content-type;" "host;" "x-amz-api-version;" diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml index 5f0860ac120..4210c13b727 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml @@ -4,7 +4,7 @@ 1000000 - 1 + 1 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml index f215a89f613..95a313ea4f2 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml @@ -4,7 +4,7 @@ 5 - 0 + 0 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml index 264c411b59b..7b1f503ed55 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml @@ -18,6 +18,7 @@ http://resolver:8083/root/data/ minio minio123 + 1 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py index 441a5a541e8..b000ccabcf4 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/test.py +++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py @@ -556,7 +556,7 @@ def test_query_is_canceled_with_inf_retries(cluster, broken_s3): @pytest.mark.parametrize("node_name", ["node", "node_with_inf_s3_retries"]) -def test_aggressive_timeouts(cluster, broken_s3, node_name): +def test_adaptive_timeouts(cluster, broken_s3, node_name): node = cluster.instances[node_name] broken_s3.setup_fake_puts(part_length=1) @@ -565,12 +565,12 @@ def test_aggressive_timeouts(cluster, broken_s3, node_name): count=1000000, ) - insert_query_id = f"TEST_AGGRESSIVE_TIMEOUTS_{node_name}" + insert_query_id = f"TEST_ADAPTIVE_TIMEOUTS_{node_name}" node.query( f""" INSERT INTO TABLE FUNCTION s3( - 'http://resolver:8083/root/data/aggressive_timeouts', + 'http://resolver:8083/root/data/adaptive_timeouts', 'minio', 'minio123', 'CSV', auto, 'none' ) @@ -593,20 +593,20 @@ def test_aggressive_timeouts(cluster, broken_s3, node_name): assert put_objects == 1 - s3_aggressive_timeouts_state = node.query( + s3_use_adaptive_timeouts = node.query( f""" SELECT value FROM system.settings WHERE - name='s3_aggressive_timeouts' + name='s3_use_adaptive_timeouts' """ ).strip() if node_name == "node_with_inf_s3_retries": # first 2 attempts failed - assert s3_aggressive_timeouts_state == "1" - assert s3_errors == 2 + assert s3_use_adaptive_timeouts == "1" + assert s3_errors == 1 else: - assert s3_aggressive_timeouts_state == "0" + assert s3_use_adaptive_timeouts == "0" assert s3_errors == 0 diff --git a/tests/integration/test_storage_s3/configs/s3_retry.xml b/tests/integration/test_storage_s3/configs/s3_retry.xml index b7a7bbc8a9b..3171da051d0 100644 --- a/tests/integration/test_storage_s3/configs/s3_retry.xml +++ b/tests/integration/test_storage_s3/configs/s3_retry.xml @@ -1,6 +1,7 @@ + 1 10 5 diff --git a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py index 103dd30340c..5ef781bdc9e 100644 --- a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py +++ b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py @@ -4,6 +4,7 @@ import re import socket import struct import sys +import time def gen_n_digit_number(n): @@ -39,14 +40,14 @@ random.seed("Unstable server/1.0") # Generating some "random" data and append a line which contains sum of numbers in column 4. lines = ( - b"".join((gen_line() for _ in range(500000))) + b"".join([gen_line() for _ in range(500000)]) + f"0,0,0,{-sum_in_4_column}\n".encode() ) class RequestHandler(http.server.BaseHTTPRequestHandler): def do_HEAD(self): - if self.path == "/root/test.csv": + if self.path == "/root/test.csv" or self.path == "/root/slow_send_test.csv": self.from_bytes = 0 self.end_bytes = len(lines) self.size = self.end_bytes @@ -101,6 +102,18 @@ class RequestHandler(http.server.BaseHTTPRequestHandler): print("Dropping connection") break + if self.path == "/root/slow_send_test.csv": + self.send_block_size = 81920 + + for c, i in enumerate( + range(self.from_bytes, self.end_bytes, self.send_block_size) + ): + self.wfile.write( + lines[i : min(i + self.send_block_size, self.end_bytes)] + ) + self.wfile.flush() + time.sleep(1) + elif self.path == "/": self.wfile.write(b"OK") diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 01ade1acc4d..8c79ad02445 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -818,6 +818,15 @@ def test_storage_s3_get_unstable(started_cluster): assert result.splitlines() == ["500001,500000,0"] +def test_storage_s3_get_slow(started_cluster): + bucket = started_cluster.minio_bucket + instance = started_cluster.instances["dummy"] + table_format = "column1 Int64, column2 Int64, column3 Int64, column4 Int64" + get_query = f"SELECT count(), sum(column3), sum(column4) FROM s3('http://resolver:8081/{started_cluster.minio_bucket}/slow_send_test.csv', 'CSV', '{table_format}') FORMAT CSV" + result = run_query(instance, get_query) + assert result.splitlines() == ["500001,500000,0"] + + def test_storage_s3_put_uncompressed(started_cluster): bucket = started_cluster.minio_bucket instance = started_cluster.instances["dummy"] From d5907e10de95bc9a3e50f877338fae29dfc3d3ea Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 14 Nov 2023 12:51:25 +0000 Subject: [PATCH 051/192] Fixing tests. --- .../PredicateExpressionsOptimizer.cpp | 4 ++- src/Storages/StorageInput.cpp | 25 +++++++++++++------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/PredicateExpressionsOptimizer.cpp b/src/Interpreters/PredicateExpressionsOptimizer.cpp index 885c99aeb90..8dc8c1c92cc 100644 --- a/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -84,8 +84,10 @@ std::vector PredicateExpressionsOptimizer::extractTablesPredicates(const A return {}; /// Not optimized when predicate contains stateful function or indeterministic function or window functions } + /// Skip predicate like `... IN (SELECT ... FROM input())` because + /// it can be duplicated but we can't execute `input()` twice. if (hasInputTableFunction(predicate_expression)) - return {}; /// Not optimized when predicate contains input table function + return {}; if (!expression_info.is_array_join) { diff --git a/src/Storages/StorageInput.cpp b/src/Storages/StorageInput.cpp index 2314d3fb581..7f1eeaedfb1 100644 --- a/src/Storages/StorageInput.cpp +++ b/src/Storages/StorageInput.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -60,16 +61,16 @@ public: ReadFromInput( Block sample_block, - //StorageSnapshotPtr storage_snapshot_, + Pipe pipe_, StorageInput & storage_) : ISourceStep(DataStream{.header = std::move(sample_block)}) - //, storage_snapshot(std::move(storage_snapshot_)) + , pipe(std::move(pipe_)) , storage(storage_) { } private: - //StorageSnapshotPtr storage_snapshot; + Pipe pipe; StorageInput & storage; }; @@ -85,21 +86,20 @@ void StorageInput::read( { storage_snapshot->check(column_names); Block sample_block = storage_snapshot->metadata->getSampleBlock(); + Pipe input_source_pipe; auto query_context = context->getQueryContext(); /// It is TCP request if we have callbacks for input(). - if (!was_pipe_initialized && query_context->getInputBlocksReaderCallback()) + if (query_context->getInputBlocksReaderCallback()) { /// Send structure to the client. query_context->initializeInput(shared_from_this()); + input_source_pipe = Pipe(std::make_shared(query_context, sample_block)); } - if (!was_pipe_initialized) - throw Exception(ErrorCodes::INVALID_USAGE_OF_INPUT, "Input stream is not initialized, input() must be used only in INSERT SELECT query"); - auto reading = std::make_unique( std::move(sample_block), - //storage_snapshot, + std::move(input_source_pipe), *this); query_plan.addStep(std::move(reading)); @@ -107,6 +107,15 @@ void StorageInput::read( void ReadFromInput::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { + if (!pipe.empty()) + { + pipeline.init(std::move(pipe)); + return; + } + + if (!storage.was_pipe_initialized) + throw Exception(ErrorCodes::INVALID_USAGE_OF_INPUT, "Input stream is not initialized, input() must be used only in INSERT SELECT query"); + if (storage.was_pipe_used) throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to read from input() twice."); From 2f9ac9b49cc8b2f8fbd7e824138855f73403ce38 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 14 Nov 2023 14:33:34 +0000 Subject: [PATCH 052/192] Address comments --- src/Backups/BackupCoordinationFileInfos.cpp | 81 +++++++++++++++++-- src/Backups/BackupFileInfo.h | 4 + src/Backups/BackupIO.h | 2 + src/Backups/BackupIO_Default.cpp | 2 + src/Backups/BackupIO_Disk.cpp | 9 +++ src/Backups/BackupIO_Disk.h | 2 + src/Backups/BackupIO_File.cpp | 10 +++ src/Backups/BackupIO_File.h | 2 + src/Backups/BackupIO_S3.cpp | 18 +++++ src/Backups/BackupIO_S3.h | 2 + src/Backups/BackupImpl.cpp | 27 ++----- .../test_backup_restore_keeper_map/test.py | 28 ++++--- 12 files changed, 149 insertions(+), 38 deletions(-) diff --git a/src/Backups/BackupCoordinationFileInfos.cpp b/src/Backups/BackupCoordinationFileInfos.cpp index ac2cbc337aa..b17b755b966 100644 --- a/src/Backups/BackupCoordinationFileInfos.cpp +++ b/src/Backups/BackupCoordinationFileInfos.cpp @@ -80,24 +80,76 @@ void BackupCoordinationFileInfos::prepare() const if (plain_backup) { + std::vector unresolved_references; + std::unordered_map file_name_to_info; + + const auto try_resolve_reference = [&](BackupFileInfo & reference) + { + auto it = file_name_to_info.find(reference.reference_target); + + if (it == file_name_to_info.end()) + return false; + + auto & target_info = it->second; + target_info->reference_sources.push_back(reference.file_name); + reference.size = target_info->size; + total_size_of_files += reference.size; + reference.checksum = target_info->checksum; + return true; + }; + /// For plain backup all file infos are stored as is, without checking for duplicates or skipping empty files. for (size_t i = 0; i != file_infos_for_all_hosts.size(); ++i) { auto & info = *(file_infos_for_all_hosts[i]); - - if (!info.reference_target.empty()) - continue; - info.data_file_name = info.file_name; info.data_file_index = i; info.base_size = 0; /// Base backup must not be used while creating a plain backup. info.base_checksum = 0; - total_size_of_files += info.size; + + if (info.reference_target.empty()) + { + file_name_to_info.emplace(info.file_name, &info); + total_size_of_files += info.size; + } + else if (!try_resolve_reference(info)) + { + unresolved_references.push_back(&info); + } } + + for (auto * reference : unresolved_references) + { + if (!try_resolve_reference(*reference)) + throw DB::Exception( + ErrorCodes::LOGICAL_ERROR, + "Couldn't resolve reference {} with target {}", + reference->file_name, + reference->reference_target); + } + num_files = file_infos_for_all_hosts.size(); } else { + std::vector unresolved_references; + std::unordered_map file_name_to_info; + + const auto try_resolve_reference = [&](BackupFileInfo & reference) + { + auto it = file_name_to_info.find(reference.reference_target); + + if (it == file_name_to_info.end()) + return false; + + auto & target_info = it->second; + reference.size = target_info->size; + reference.checksum = target_info->checksum; + reference.data_file_name = target_info->data_file_name; + reference.data_file_index = target_info->data_file_index; + return true; + }; + /// For non-plain backups files with the same size and checksum are stored only once, /// in order to find those files we'll use this map. std::map data_file_index_by_checksum; @@ -107,7 +159,12 @@ void BackupCoordinationFileInfos::prepare() const auto & info = *(file_infos_for_all_hosts[i]); if (!info.reference_target.empty()) + { + if (!try_resolve_reference(info)) + unresolved_references.push_back(&info); + continue; + } if (info.size == info.base_size) { @@ -134,7 +191,21 @@ void BackupCoordinationFileInfos::prepare() const info.data_file_name = file_infos_for_all_hosts[it->second]->data_file_name; } } + + file_name_to_info.emplace(info.file_name, &info); } + + for (auto * reference : unresolved_references) + { + if (!try_resolve_reference(*reference)) + throw DB::Exception( + ErrorCodes::LOGICAL_ERROR, + "Couldn't resolve reference {} with target {}", + reference->file_name, + reference->reference_target); + } + + num_files = file_infos_for_all_hosts.size(); } prepared = true; diff --git a/src/Backups/BackupFileInfo.h b/src/Backups/BackupFileInfo.h index 1d5607fd418..42bda3aa6ed 100644 --- a/src/Backups/BackupFileInfo.h +++ b/src/Backups/BackupFileInfo.h @@ -42,6 +42,10 @@ struct BackupFileInfo /// Set if this file is just a reference to another file String reference_target; + /// List of files that are referencing this file + /// Used for plain backup which needs to resolve all references + Strings reference_sources; + struct LessByFileName { bool operator()(const BackupFileInfo & lhs, const BackupFileInfo & rhs) const { return (lhs.file_name < rhs.file_name); } diff --git a/src/Backups/BackupIO.h b/src/Backups/BackupIO.h index e4a82a604e8..91d57e5ab0a 100644 --- a/src/Backups/BackupIO.h +++ b/src/Backups/BackupIO.h @@ -61,6 +61,8 @@ public: virtual void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, bool copy_encrypted, UInt64 start_pos, UInt64 length) = 0; + virtual void copyFile(const String & destination, const String & source, size_t size) = 0; + virtual void removeFile(const String & file_name) = 0; virtual void removeFiles(const Strings & file_names) = 0; diff --git a/src/Backups/BackupIO_Default.cpp b/src/Backups/BackupIO_Default.cpp index 5ac522695ce..95f2c66b6b9 100644 --- a/src/Backups/BackupIO_Default.cpp +++ b/src/Backups/BackupIO_Default.cpp @@ -91,4 +91,6 @@ void BackupWriterDefault::copyFileFromDisk(const String & path_in_backup, DiskPt copyDataToFile(path_in_backup, create_read_buffer, start_pos, length); } + + } diff --git a/src/Backups/BackupIO_Disk.cpp b/src/Backups/BackupIO_Disk.cpp index 1e260ad22d9..91e8b97bc20 100644 --- a/src/Backups/BackupIO_Disk.cpp +++ b/src/Backups/BackupIO_Disk.cpp @@ -128,4 +128,13 @@ void BackupWriterDisk::copyFileFromDisk(const String & path_in_backup, DiskPtr s BackupWriterDefault::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length); } +void BackupWriterDisk::copyFile(const String & destination, const String & source, size_t /*size*/) +{ + LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination); + auto dest_file_path = root_path / destination; + auto src_file_path = root_path / source; + disk->createDirectories(dest_file_path.parent_path()); + disk->copyFile(src_file_path, *disk, dest_file_path, read_settings, write_settings); +} + } diff --git a/src/Backups/BackupIO_Disk.h b/src/Backups/BackupIO_Disk.h index 70d31eacc1a..575ec3f5707 100644 --- a/src/Backups/BackupIO_Disk.h +++ b/src/Backups/BackupIO_Disk.h @@ -44,6 +44,8 @@ public: void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, bool copy_encrypted, UInt64 start_pos, UInt64 length) override; + void copyFile(const String & destination, const String & source, size_t size) override; + void removeFile(const String & file_name) override; void removeFiles(const Strings & file_names) override; diff --git a/src/Backups/BackupIO_File.cpp b/src/Backups/BackupIO_File.cpp index 2bedb5470fb..5384637a969 100644 --- a/src/Backups/BackupIO_File.cpp +++ b/src/Backups/BackupIO_File.cpp @@ -152,4 +152,14 @@ void BackupWriterFile::copyFileFromDisk(const String & path_in_backup, DiskPtr s BackupWriterDefault::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length); } +void BackupWriterFile::copyFile(const String & destination, const String & source, size_t /*size*/) +{ + LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination); + + auto abs_source_path = root_path / source; + auto abs_dest_path = root_path / destination; + fs::create_directories(abs_dest_path.parent_path()); + fs::copy(abs_source_path, abs_dest_path, fs::copy_options::overwrite_existing); +} + } diff --git a/src/Backups/BackupIO_File.h b/src/Backups/BackupIO_File.h index 6bb4b11e134..ebe9a0f02cb 100644 --- a/src/Backups/BackupIO_File.h +++ b/src/Backups/BackupIO_File.h @@ -38,6 +38,8 @@ public: void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, bool copy_encrypted, UInt64 start_pos, UInt64 length) override; + void copyFile(const String & destination, const String & source, size_t size) override; + void removeFile(const String & file_name) override; void removeFiles(const Strings & file_names) override; diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 8bb2f895e38..9688d7f0730 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -249,6 +249,24 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src BackupWriterDefault::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length); } +void BackupWriterS3::copyFile(const String & destination, const String & source, size_t size) +{ + LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination); + copyS3File( + client, + client, + /* src_bucket */ s3_uri.bucket, + /* src_key= */ fs::path(s3_uri.key) / source, + 0, + size, + s3_uri.bucket, + fs::path(s3_uri.key) / destination, + s3_settings.request_settings, + read_settings, + {}, + threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterS3")); +} + void BackupWriterS3::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) { copyDataToS3File(create_read_buffer, start_pos, length, client, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {}, diff --git a/src/Backups/BackupIO_S3.h b/src/Backups/BackupIO_S3.h index 4abcbedf89f..c00ce747ff5 100644 --- a/src/Backups/BackupIO_S3.h +++ b/src/Backups/BackupIO_S3.h @@ -49,6 +49,8 @@ public: void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, bool copy_encrypted, UInt64 start_pos, UInt64 length) override; + void copyFile(const String & destination, const String & source, size_t size) override; + void removeFile(const String & file_name) override; void removeFiles(const Strings & file_names) override; diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index bf1853828df..56c30fab5c2 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -362,10 +362,10 @@ void BackupImpl::writeBackupMetadata() *out << ""; *out << "" << xml << info.file_name << ""; + *out << "" << info.size << ""; if (info.size) { - *out << "" << info.size << ""; *out << "" << hexChecksum(info.checksum) << ""; if (info.base_size) { @@ -381,10 +381,6 @@ void BackupImpl::writeBackupMetadata() if (info.encrypted_by_disk) *out << "true"; } - else if (!info.reference_target.empty()) - *out << "" << xml << info.reference_target << ""; - else - *out << "" << info.size << ""; total_size += info.size; bool has_entry = !deduplicate_files || (info.size && (info.size != info.base_size) && (info.data_file_name.empty() || (info.data_file_name == info.file_name))); @@ -465,13 +461,6 @@ void BackupImpl::readBackupMetadata() BackupFileInfo info; info.file_name = getString(file_config, "name"); - info.reference_target = getString(file_config, "reference_target", ""); - if (!info.reference_target.empty()) - { - reference_files.emplace_back(std::move(info.file_name), std::move(info.reference_target)); - continue; - } - info.size = getUInt64(file_config, "size"); if (info.size) { @@ -521,14 +510,6 @@ void BackupImpl::readBackupMetadata() } } - for (auto & [source_file, target_file] : reference_files) - { - auto it = file_names.find(target_file); - if (it == file_names.end()) - throw Exception(ErrorCodes::BACKUP_ENTRY_NOT_FOUND, "Backup entry {} referenced by {} not found", target_file, source_file); - file_names.emplace(std::move(source_file), it->second); - } - uncompressed_size = size_of_entries + str.size(); compressed_size = uncompressed_size; if (!use_archive) @@ -954,6 +935,12 @@ void BackupImpl::writeFile(const BackupFileInfo & info, BackupEntryPtr entry) writer->copyDataToFile(info.data_file_name, create_read_buffer, info.base_size, info.size - info.base_size); } + if (!deduplicate_files) + { + for (const auto & reference : info.reference_sources) + writer->copyFile(reference, info.data_file_name, info.size - info.base_size); + } + { std::lock_guard lock{mutex}; ++num_entries; diff --git a/tests/integration/test_backup_restore_keeper_map/test.py b/tests/integration/test_backup_restore_keeper_map/test.py index 8343ad3177f..c401f482c3f 100644 --- a/tests/integration/test_backup_restore_keeper_map/test.py +++ b/tests/integration/test_backup_restore_keeper_map/test.py @@ -65,22 +65,24 @@ def new_backup_name(base_name): return f"Disk('backups', '{base_name}{backup_id_counter}')" -def test_on_cluster(): - node1.query_with_retry("CREATE DATABASE keeper_backup ON CLUSTER cluster") +@pytest.mark.parametrize("deduplicate_files", [0, 1]) +def test_on_cluster(deduplicate_files): + database_name = f"keeper_backup{deduplicate_files}" + node1.query_with_retry(f"CREATE DATABASE {database_name} ON CLUSTER cluster") node1.query_with_retry( - "CREATE TABLE keeper_backup.keeper1 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key" + f"CREATE TABLE {database_name}.keeper1 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/{database_name}/test_on_cluster1') PRIMARY KEY key" ) node1.query_with_retry( - "CREATE TABLE keeper_backup.keeper2 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key" + f"CREATE TABLE {database_name}.keeper2 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/{database_name}/test_on_cluster1') PRIMARY KEY key" ) node1.query_with_retry( - "CREATE TABLE keeper_backup.keeper3 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster2') PRIMARY KEY key" + f"CREATE TABLE {database_name}.keeper3 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/{database_name}/test_on_cluster2') PRIMARY KEY key" ) node1.query_with_retry( - "INSERT INTO keeper_backup.keeper2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5" + f"INSERT INTO {database_name}.keeper2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5" ) node1.query_with_retry( - "INSERT INTO keeper_backup.keeper3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5" + f"INSERT INTO {database_name}.keeper3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5" ) expected_result = "".join(f"{i}\ttest{i}\n" for i in range(5)) @@ -89,7 +91,7 @@ def test_on_cluster(): for node in [node1, node2, node3]: for i in range(1, 4): result = node.query_with_retry( - f"SELECT key, value FROM keeper_backup.keeper{i} ORDER BY key FORMAT TSV" + f"SELECT key, value FROM {database_name}.keeper{i} ORDER BY key FORMAT TSV" ) assert result == expected_result @@ -97,10 +99,10 @@ def test_on_cluster(): backup_name = new_backup_name("test_on_cluster") node1.query( - f"BACKUP DATABASE keeper_backup ON CLUSTER cluster TO {backup_name} SETTINGS async = false;" + f"BACKUP DATABASE {database_name} ON CLUSTER cluster TO {backup_name} SETTINGS async = false, deduplicate_files = {deduplicate_files};" ) - node1.query("DROP DATABASE keeper_backup ON CLUSTER cluster SYNC;") + node1.query(f"DROP DATABASE {database_name} ON CLUSTER cluster SYNC;") def apply_for_all_nodes(f): for node in [node1, node2, node3]: @@ -121,14 +123,14 @@ def test_on_cluster(): apply_for_all_nodes(lambda node: node.start_clickhouse()) node1.query( - f"RESTORE DATABASE keeper_backup ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;" + f"RESTORE DATABASE {database_name} ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;" ) verify_data() - node1.query("DROP TABLE keeper_backup.keeper3 ON CLUSTER cluster SYNC;") + node1.query(f"DROP TABLE {database_name}.keeper3 ON CLUSTER cluster SYNC;") node1.query( - f"RESTORE TABLE keeper_backup.keeper3 ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;" + f"RESTORE TABLE {database_name}.keeper3 ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;" ) verify_data() From 1356dc2eaa76ca63d7b6ff982201fe0bd26550bc Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 14 Nov 2023 14:55:21 +0000 Subject: [PATCH 053/192] Fixing style. --- src/Storages/StorageInput.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/StorageInput.cpp b/src/Storages/StorageInput.cpp index 7f1eeaedfb1..4c319ed9414 100644 --- a/src/Storages/StorageInput.cpp +++ b/src/Storages/StorageInput.cpp @@ -17,6 +17,7 @@ namespace DB namespace ErrorCodes { extern const int INVALID_USAGE_OF_INPUT; + extern const int LOGICAL_ERROR; } StorageInput::StorageInput(const StorageID & table_id, const ColumnsDescription & columns_) From 9df2775f08d5ab377ba3aa2dd05010329b67ef20 Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Tue, 14 Nov 2023 02:28:09 +0000 Subject: [PATCH 054/192] reduce timeout and setTimeout earlier. Signed-off-by: Jianfei Hu --- src/Coordination/KeeperContext.cpp | 7 ++++--- src/IO/S3/Credentials.cpp | 10 +++++----- tests/integration/helpers/keeper_config2.xml | 1 + .../integration/test_keeper_availability_zone/test.py | 6 ++++-- .../integration/test_keeper_four_word_command/test.py | 8 ++++---- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/Coordination/KeeperContext.cpp b/src/Coordination/KeeperContext.cpp index 9745a53d1ab..c3cb166abee 100644 --- a/src/Coordination/KeeperContext.cpp +++ b/src/Coordination/KeeperContext.cpp @@ -39,8 +39,9 @@ void KeeperContext::initialize(const Poco::Util::AbstractConfiguration & config, if (config.hasProperty("keeper_server.availability_zone")) { - auto keeper_az = config.getString("keeper_server.availability_zone.value"); - if (config.getBool("keeper_server.availability_zone.enable_auto_detection_on_cloud", false)) + auto keeper_az = config.getString("keeper_server.availability_zone.value", ""); + const auto auto_detect_for_cloud = config.getBool("keeper_server.availability_zone.enable_auto_detection_on_cloud", false); + if (keeper_az.empty() && auto_detect_for_cloud) { try { @@ -54,7 +55,7 @@ void KeeperContext::initialize(const Poco::Util::AbstractConfiguration & config, if (!keeper_az.empty()) { system_nodes_with_data[keeper_availability_zone_path] = keeper_az; - LOG_INFO(&Poco::Logger::get("KeeperContext"), "Initialize the KeeperContext with availability zone: '{}'.'. ", keeper_az); + LOG_INFO(&Poco::Logger::get("KeeperContext"), "Initialize the KeeperContext with availability zone: '{}'", keeper_az); } } diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index 4ba14572589..7d6ed094486 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -65,7 +65,7 @@ bool areCredentialsEmptyOrExpired(const Aws::Auth::AWSCredentials & credentials, } const char SSO_CREDENTIALS_PROVIDER_LOG_TAG[] = "SSOCredentialsProvider"; -const int AVAILABILITY_ZONE_REQUEST_TIMEOUT_SECONDS = 5; +const int AVAILABILITY_ZONE_REQUEST_TIMEOUT_SECONDS = 3; } @@ -241,11 +241,11 @@ String AWSEC2MetadataClient::getAvailabilityZoneOrException() { Poco::URI uri(getAWSMetadataEndpoint() + EC2_AVAILABILITY_ZONE_RESOURCE); Poco::Net::HTTPClientSession session(uri.getHost(), uri.getPort()); + session.setTimeout(Poco::Timespan(AVAILABILITY_ZONE_REQUEST_TIMEOUT_SECONDS, 0)); Poco::Net::HTTPResponse response; Poco::Net::HTTPRequest request(Poco::Net::HTTPRequest::HTTP_GET, uri.getPath()); session.sendRequest(request); - session.setTimeout(Poco::Timespan(AVAILABILITY_ZONE_REQUEST_TIMEOUT_SECONDS, 0)); std::istream & rs = session.receiveResponse(response); if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) @@ -287,17 +287,17 @@ String getRunningAvailabilityZoneImpl() auto aws_az = AWSEC2MetadataClient::getAvailabilityZoneOrException(); return aws_az; } - catch (const DB::Exception & aws_ex) + catch (const std::exception & aws_ex) { try { auto gcp_zone = getGCPAvailabilityZoneOrException(); return gcp_zone; } - catch (const DB::Exception & gcp_ex) + catch (const std::exception & gcp_ex) { throw DB::Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Failed to find the availability zone, tried AWS and GCP. AWS Error: {}\nGCP Error: {}", aws_ex.displayText(), gcp_ex.displayText()); + "Failed to find the availability zone, tried AWS and GCP. AWS Error: {}\nGCP Error: {}", aws_ex.what(), gcp_ex.what()); } } } diff --git a/tests/integration/helpers/keeper_config2.xml b/tests/integration/helpers/keeper_config2.xml index 0c58aaceb1c..2afff2f5e59 100644 --- a/tests/integration/helpers/keeper_config2.xml +++ b/tests/integration/helpers/keeper_config2.xml @@ -14,6 +14,7 @@ 2 az-zoo2 + 1 diff --git a/tests/integration/test_keeper_availability_zone/test.py b/tests/integration/test_keeper_availability_zone/test.py index b78e776f3c6..a2003f8539e 100644 --- a/tests/integration/test_keeper_availability_zone/test.py +++ b/tests/integration/test_keeper_availability_zone/test.py @@ -27,10 +27,12 @@ def test_get_availability_zone(): with KeeperClient.from_cluster(cluster, "zoo1") as client1: assert client1.get("/keeper/availability_zone") == "az-zoo1" + # Keeper2 set enable_auto_detection_on_cloud to true, but is ignored and az-zoo2 is used. with KeeperClient.from_cluster(cluster, "zoo2") as client2: assert client2.get("/keeper/availability_zone") == "az-zoo2" - + assert "availability_zone" in client2.ls("/keeper") + # keeper3 is not configured with availability_zone value. with KeeperClient.from_cluster(cluster, "zoo3") as client3: with pytest.raises(Exception): - client3.get("/keeper/availability_zone") \ No newline at end of file + client3.get("/keeper/availability_zone") diff --git a/tests/integration/test_keeper_four_word_command/test.py b/tests/integration/test_keeper_four_word_command/test.py index 25c4bc55327..71501133ae7 100644 --- a/tests/integration/test_keeper_four_word_command/test.py +++ b/tests/integration/test_keeper_four_word_command/test.py @@ -183,8 +183,8 @@ def test_cmd_mntr(started_cluster): # contains: # 10 nodes created by test # 3 nodes created by clickhouse "/clickhouse/task_queue/ddl" - # 1 root node, 4 keeper system nodes - assert int(result["zk_znode_count"]) == 15 + # 1 root node, 3 keeper system nodes + assert int(result["zk_znode_count"]) == 14 assert int(result["zk_watch_count"]) == 2 assert int(result["zk_ephemerals_count"]) == 2 assert int(result["zk_approximate_data_size"]) > 0 @@ -333,7 +333,7 @@ def test_cmd_srvr(started_cluster): assert int(result["Connections"]) == 1 assert int(result["Zxid"], 16) > 10 assert result["Mode"] == "leader" - assert result["Node count"] == "15" + assert result["Node count"] == "14" finally: destroy_zk_client(zk) @@ -373,7 +373,7 @@ def test_cmd_stat(started_cluster): assert int(result["Connections"]) == 1 assert int(result["Zxid"], 16) >= 10 assert result["Mode"] == "leader" - assert result["Node count"] == "15" + assert result["Node count"] == "14" # filter connection statistics cons = [n for n in data.split("\n") if "=" in n] From 64c2a696666d594783c1996c0910166cacba000f Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Tue, 14 Nov 2023 20:28:37 +0100 Subject: [PATCH 055/192] check performance --- base/poco/Net/src/HTTPSession.cpp | 32 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index 9ebbd7d04cd..97decded282 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -94,22 +94,24 @@ void HTTPSession::setTimeout(const Poco::Timespan& timeout) void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco::Timespan& sendTimeout, const Poco::Timespan& receiveTimeout) { _connectionTimeout = connectionTimeout; + _sendTimeout = sendTimeout; + _receiveTimeout = receiveTimeout; - if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) - { - _sendTimeout = sendTimeout; - - if (connected()) - _socket.setSendTimeout(_sendTimeout); - } - - if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) - { - _receiveTimeout = receiveTimeout; - - if (connected()) - _socket.setReceiveTimeout(_receiveTimeout); - } +// if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) +// { +// _sendTimeout = sendTimeout; +// +// if (connected()) +// _socket.setSendTimeout(_sendTimeout); +// } +// +// if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) +// { +// _receiveTimeout = receiveTimeout; +// +// if (connected()) +// _socket.setReceiveTimeout(_receiveTimeout); +// } } From cd909ffc48cc4fb6fb7bc23843659c9559c5921a Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 14 Nov 2023 20:13:09 +0000 Subject: [PATCH 056/192] Test RabbitMQ with secure connection --- .../compose/docker_compose_rabbitmq.yml | 6 ++- .../integration/runner/misc/rabbitmq.conf | 8 --- .../runner/misc/rabbitmq/ca-cert.pem | 32 ++++++++++++ .../runner/misc/rabbitmq/generate_certs.sh | 10 ++++ .../runner/misc/rabbitmq/rabbitmq.conf | 15 ++++++ .../runner/misc/rabbitmq/server-cert.pem | 33 ++++++++++++ .../runner/misc/rabbitmq/server-key.pem | 52 +++++++++++++++++++ .../integration/test_storage_rabbitmq/test.py | 28 +++++++--- 8 files changed, 168 insertions(+), 16 deletions(-) delete mode 100644 docker/test/integration/runner/misc/rabbitmq.conf create mode 100644 docker/test/integration/runner/misc/rabbitmq/ca-cert.pem create mode 100755 docker/test/integration/runner/misc/rabbitmq/generate_certs.sh create mode 100644 docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf create mode 100644 docker/test/integration/runner/misc/rabbitmq/server-cert.pem create mode 100644 docker/test/integration/runner/misc/rabbitmq/server-key.pem diff --git a/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml b/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml index 2db9fb589d2..61b21e0e3d9 100644 --- a/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml +++ b/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml @@ -6,9 +6,13 @@ services: hostname: rabbitmq1 expose: - ${RABBITMQ_PORT:-5672} + - ${RABBITMQ_SECURE_PORT:-5671} volumes: - type: ${RABBITMQ_LOGS_FS:-tmpfs} source: ${RABBITMQ_LOGS:-} target: /rabbitmq_logs/ - "${RABBITMQ_COOKIE_FILE}:/var/lib/rabbitmq/.erlang.cookie" - - /misc/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf \ No newline at end of file + - /misc/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf + - /misc/rabbitmq/ca-cert.pem:/etc/rabbitmq/ca-cert.pem + - /misc/rabbitmq/server-cert.pem:/etc/rabbitmq/server-cert.pem + - /misc/rabbitmq/server-key.pem:/etc/rabbitmq/server-key.pem diff --git a/docker/test/integration/runner/misc/rabbitmq.conf b/docker/test/integration/runner/misc/rabbitmq.conf deleted file mode 100644 index 3527c83880b..00000000000 --- a/docker/test/integration/runner/misc/rabbitmq.conf +++ /dev/null @@ -1,8 +0,0 @@ -loopback_users.guest = false -listeners.tcp.default = 5672 -default_pass = clickhouse -default_user = root -management.tcp.port = 15672 - -log.file = /rabbitmq_logs/rabbit.log -log.file.level = debug diff --git a/docker/test/integration/runner/misc/rabbitmq/ca-cert.pem b/docker/test/integration/runner/misc/rabbitmq/ca-cert.pem new file mode 100644 index 00000000000..4a7b88f7936 --- /dev/null +++ b/docker/test/integration/runner/misc/rabbitmq/ca-cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFhTCCA22gAwIBAgIUWhfjFfbwannH3KIqITDtgcvSItMwDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjMxMTE0 +MTgyODI2WhcNMzMxMTExMTgyODI2WjBSMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQsw +CQYDVQQDDAJjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAJfJegdC +gavNGYzSdva+5QMxGvqyLwZzjophMeyEzlW/Di4KFGPho+fVlVMB/EwaTRoBRLEu +SQusQwoFg71mGvUTOpgHzlsUz4vcVVFOiL4bJdzCWQKzdC8M8rUFoks9FMboVeSx +jhAnKAm/NpCLpm9VYnRjEq2KEbJp7VkPAHgZEXR7VABwCFvmDcztrfcWfmXxm6IH +o+AkF/nqdphLu7Q1yDQiF8Q8TuszuhqgQ7/1PrRcaSADrF15jJjQb05sILpGCT3e +lxJYId5RF0+fgTIqy03bAKB53+8V8cAkowI4rvPTmcFXhcG3rkDO6lyZixHhlpKi +PmXEzHh0kfsRjzkNBP0CKqPnu3D2iymROiPAH2cteaYe6jdD2HIjuVLk/TjX1ZFy +DlZCrJIwj0l8A2xAfLq8Gw5RSr0a9k5TiMD5nZtfd12Vd0K82vO32vmcjO2Igddc +VWccDDwUY/ZWV3uznkusOBrB8wba3ZsXA5hjJzs0KlTvQKPjX0y4lFMmZGbelwjt +pR5dRNLi5XTdMPzV0mAnvJhDTFEmME19Bh6AEsjuAz3gHUdwNTbSxUS3mF/hTL9k +v2wh5udUAOwqD1uEzqPJyG4JCJQozIDOEEZVixWqQ60b9wUHN8meqO4y9fxTdmHW +Vo5BAF1xEJhJJb0QY/O6GahPtWqb/Mr1rtPJAgMBAAGjUzBRMB0GA1UdDgQWBBSw +fQcOabXwX/v9F1hd2cmuIug56jAfBgNVHSMEGDAWgBSwfQcOabXwX/v9F1hd2cmu +Iug56jAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQAms8y6RVxl +mKSUbsU8JscYwOzcRUQJWETeIr4rtZvMHH+3vkdBU0yKxGpEm7U8J3+5oVTYPhbs +11ZAL+DvIZ6gT6pjDvECyVox1OkjNogz843fTMbNqjuuehjSKXwpMTy5/kmT2aLj +//nBi5UX1xo3RQ9vtmBwzZ3VFK99DFXraDOPS/yk43WV2uqdWsXCNvyEyCHmM1IB +9FQe2EFcO6s4/N+TarhIZ8Udhj5bl8d4eDd1yEckmTD4aHJBgMII2uEwrAxR5CT1 +tCqUKutvNrkXI5PIULvmy+Lwm7PJAC7grPtUHK6anSugpljd7bFj18fHH9APiC45 +Ou4OOK1BUZogCEo7rD36UlanxQO0GEzgDCVEoEdoe0WRdc6T9b4fM8vpQqwBdf9t +nkPB8oLCKerqqYwCiMuWm4BcRmExA7ypIkUCcluGO9/kTmdps3NqOvET9oLTjXuA +z5TPmaK5a3poKLoxBfv6WfRTgisOnMNTsjL1R8+xuhEn5hSlE2r3wAi8Cys9Z9PV +LhTj0SRTXILd2NW3lO8QfO0pGdjgk90GqkyUY9YjuiMVPvdUAFQsHm+0GEZEXjOD +Bw7tLSJQ4IKhfactg/Puxd15ahcWAxeelyED+w/zVGdHYblqbvfdtiGj370KVhoj +DL5HkdPa0IhTPqMBnmoVQ4C/WzKofXBjQQ== +-----END CERTIFICATE----- diff --git a/docker/test/integration/runner/misc/rabbitmq/generate_certs.sh b/docker/test/integration/runner/misc/rabbitmq/generate_certs.sh new file mode 100755 index 00000000000..442d2fe004f --- /dev/null +++ b/docker/test/integration/runner/misc/rabbitmq/generate_certs.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# 1. Generate CA's private key and self-signed certificate +openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout ca-key.pem -out ca-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=ca" + +# 2. Generate server's private key and certificate signing request (CSR) +openssl req -newkey rsa:4096 -nodes -batch -keyout server-key.pem -out server-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=server" + +# 3. Use CA's private key to sign server's CSR and get back the signed certificate +openssl x509 -req -days 3650 -in server-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -extfile server-ext.cnf -out server-cert.pem diff --git a/docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf b/docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf new file mode 100644 index 00000000000..307871ba589 --- /dev/null +++ b/docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf @@ -0,0 +1,15 @@ +loopback_users.guest = false +listeners.tcp.default = 5672 +default_pass = clickhouse +default_user = root +management.tcp.port = 15672 + +log.file = /rabbitmq_logs/rabbit.log +log.file.level = debug + +listeners.ssl.default = 5671 +ssl_options.verify = verify_none +ssl_options.fail_if_no_peer_cert = false +ssl_options.cacertfile = /etc/rabbitmq/ca_cert.pem +ssl_options.certfile = /etc/rabbitmq/server_cert.pem +ssl_options.keyfile = /etc/rabbitmq/server_key.pem diff --git a/docker/test/integration/runner/misc/rabbitmq/server-cert.pem b/docker/test/integration/runner/misc/rabbitmq/server-cert.pem new file mode 100644 index 00000000000..338de91aa0f --- /dev/null +++ b/docker/test/integration/runner/misc/rabbitmq/server-cert.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFpTCCA42gAwIBAgIUJvQslezZO09XgFGQCxOM6orIsWowDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjMxMTE0 +MTgyODI5WhcNMzMxMTExMTgyODI5WjBWMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8w +DQYDVQQDDAZzZXJ2ZXIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCe +o/K71WdKpVpdDvhaZy6wBVhFlu7j7DhfTSYvcPpAJfExmzO8JK3vh5/yGyAO1t79 +gAjqyXLMCZKw7ajM2rez9YnGYqaFi70BlTcU2KQ8LbFEYRc3cYNDmmWIKBpwpSri +We5SQrRLnDXqAn6T8FG5ejQ/t+1IUMrtZENB4lp8fBmEOJb5yr1TE++6EhiDBQho +cLDWWWP8b55kyZhqP/VgmId4lvboGMRKxbiRJ6/SPr/i/pteBD8jTYfbJr6ceXov +/p5yxIp61z5ry1anU7W3B8jTl/gj7SqtFdSnRajZ0DGJJAUKpiiJSCSlp5YB5Ub2 +eBBMHmdA5R1MuiU9TOA35nUW5wkhEOJXnBR/WCsYioVmn/+5dm6JPYiwp/TefYnr +x9iLbb/Tyx7MnXzeyvKg781SwmnvS6Blhtr0zhAW9szZz8cVHPBqFs6PzGs/5mwE +C+tM3Zp85aHd28nIT4NQLHdMDwVmGwmPdy4uavtYWMDhsuIyEU8hCZymiHhPnuHU +VbmfZ8GOTIzUgQAvZb0fL1Xow2Tf6XuARnvuU9weRttg9jSOqPuUENRsFXv0mU8M +EpQjrxry88Wfz7bBEjN5JHC16PB/Nu7zTGJ4/slThbxNv0bIONzvTBPbXrKnxw7Z +d9WhGJI+LQxRqLTynQe6yzDwIuW9LRdBNTp7CtQRwQIDAQABo28wbTArBgNVHREE +JDAigiBpbnRlZ3JhdGlvbi10ZXN0cy5jbGlja2hvdXNlLmNvbTAdBgNVHQ4EFgQU +54GvBUYWvMADpTz/zglwMlaJuskwHwYDVR0jBBgwFoAUsH0HDmm18F/7/RdYXdnJ +riLoOeowDQYJKoZIhvcNAQELBQADggIBADfNH6O6ay+xg0XmV6sR0n4j6PwL9Cnc +VjuCmHQbpFXfMvgCdfHvbtT0Y/pG7IoeKmrrm0JPvKa2E9Ht0j6ZnowQ2m9mJk8U +5Fd/PbC1I4KgVCw6HRSOcwqANJxOGe7RyN9PTZZ8fxzmzIR3FiQ2bXfr+LaotZOK +aVS8F8xCOzoMvL9LFls2YpEn20p/1EATIf2MFX3j9vKfcJVOyDJV4i5BMImStFLM +g3sdC96de/59yxt9khM0PNucU1ldNFs/kZVEcNSwGOAIgQEPwULJtDY+ZSWeROpX +EpWndN6zQsv1pdNvLtXsDXfi4YoH9QVaA/k4aFFJ08CjSZfMYmwyPOGsf/wqT65i +ADID2yb1A/FIIe/fM+d2gXHBVFBDmydJ1JCdCoYrEJgfWj1LO/0jLi34ZZ17Hu7F +D33fLARF9nlLzlUiWjcQlOjNoCM48AgG/3wHk4eiSfc/3PIJDuDGDa0NdtDeKKhH +XkP2ll4cMUH6EQ9KO1jHPmf5RokX4QJgH+ofO4U5XQFwc3lOyJzEQnED+wame7do +R7TE4F/OXhxLqA6DFkzXe89/kSCoAF9bjzmUn/ilrg8NXKKgprgHg4DJHgvCQVVC +34ab7Xj7msUm4D9vI+GAeUbUqnqCaWxDF6vCMT0Qq7iSVDxa/SV8TX8Vp2Zh+PSh +4m23Did+KjLq +-----END CERTIFICATE----- diff --git a/docker/test/integration/runner/misc/rabbitmq/server-key.pem b/docker/test/integration/runner/misc/rabbitmq/server-key.pem new file mode 100644 index 00000000000..92e93e8fba5 --- /dev/null +++ b/docker/test/integration/runner/misc/rabbitmq/server-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCeo/K71WdKpVpd +DvhaZy6wBVhFlu7j7DhfTSYvcPpAJfExmzO8JK3vh5/yGyAO1t79gAjqyXLMCZKw +7ajM2rez9YnGYqaFi70BlTcU2KQ8LbFEYRc3cYNDmmWIKBpwpSriWe5SQrRLnDXq +An6T8FG5ejQ/t+1IUMrtZENB4lp8fBmEOJb5yr1TE++6EhiDBQhocLDWWWP8b55k +yZhqP/VgmId4lvboGMRKxbiRJ6/SPr/i/pteBD8jTYfbJr6ceXov/p5yxIp61z5r +y1anU7W3B8jTl/gj7SqtFdSnRajZ0DGJJAUKpiiJSCSlp5YB5Ub2eBBMHmdA5R1M +uiU9TOA35nUW5wkhEOJXnBR/WCsYioVmn/+5dm6JPYiwp/TefYnrx9iLbb/Tyx7M +nXzeyvKg781SwmnvS6Blhtr0zhAW9szZz8cVHPBqFs6PzGs/5mwEC+tM3Zp85aHd +28nIT4NQLHdMDwVmGwmPdy4uavtYWMDhsuIyEU8hCZymiHhPnuHUVbmfZ8GOTIzU +gQAvZb0fL1Xow2Tf6XuARnvuU9weRttg9jSOqPuUENRsFXv0mU8MEpQjrxry88Wf +z7bBEjN5JHC16PB/Nu7zTGJ4/slThbxNv0bIONzvTBPbXrKnxw7Zd9WhGJI+LQxR +qLTynQe6yzDwIuW9LRdBNTp7CtQRwQIDAQABAoICAA0lev0T3z5xW36wueYL/PN7 +TehebKeYsMc9BngR/bsJKea5fN0PkRZzf865brusFMifLp3+WbQM6wocd8uaKHUS +WPuGu1P/04bpDap9lYajJriK7ziaAI2+osFYyXAiT954I2bPvk8xv8oHsOOjm7Iq +LWBGZrSCdX6cu3IfRu5f/mFVqzVCFtRmp4wc6ckZxquZAx6QQ9fsjAzAJBBSAoyh +t0BICmgLfWDQ582no0tiBdbS0J9G7NCJIUQI/uzKqFSH3iuWm/84DSUzsZemOT3U +uFDInDil885qK7g87pQ2S5SY1o4eXOebgeX0cFrx3CKaqocUUewv0HDGUEW3NDFs +KhUvlJZIFgk6bMend16U6kfRCUsjLA22Rfxzanl53cGVywCeIMirnLYuEu0TsxyK +CblBvyhcpjrGi7FQskzR+J9LpZPnmtn6TAb7JCAALRVHcAGKhGeh613SjPUfkWb0 +KpDps08x8MWGEAALuHbOK0nMLFm+PuMt7+krqCeJET+XM44GT+6ZstrDv0RufxUN ++pkLW7AsVZoXcFvaOWjuyBvX/f6UHCSfueo0mB3H80WoftDIfdhM+AI7/oBTYCBx +Z8BtW+g7Eq3pOUg/Um7S7Z2bybBWE14kpi95gRf3upEYPqHJUpJPdu20lk24iAt9 +LCXF4AjZBIdAuyJrYOJBAoIBAQDd/Bm14WvmBOablGLn6hmohi6M75D+/eQanlg9 +eJhXJUVd8FzOTjKi70EHWvkqswenNDbe/WGtImqG+9G+N/ol2qhi5xVSQ2XQmcVQ +U+k15Bzm9xKM0OqsStFvRgP1Cy6Ms3/jxr5JEEwUepmjvWTDGTlhTQASA/D7Uh2q +5HpPiHEVm4g5eTAYWeAbI6cGwVS0L4y6xkFGde37Kh2P8ZodWB+d3fglVu4Ok9Nf +wE2f8MK2ewQ0SbF/Nj2WjlVomvOvOJG/2CDLuiH/vc4YUvLAm8pNwvsmgtSh1Okt +E/HfXegrlPPEgw6owqoQFt+aGUITgEhiwEVAcYS0pXzzkQX5AoIBAQC28wJ8ueKr +fINpJM2pSc7WRDFduP5yGsRreSLBXLKMbvOlIVb3PaWp11Cg3+X5O90bPXYJ9mBI +WGR0g14/VD8edxs2D5TUZcP4/vKXGHaWRY9Z4A3jVpjzAxAaviNDHJ08tLXEMXZQ +lbA7dX8z6lpoQfwnPzjBwB01mVegwXPeIwIIfT/FmAiGzvSnAMXBGSGWRRdzof0M +/vPFbgllcQmM4AnEGcErCgFRpwcssO87T2jnvf6QVE5JCcnUcGIli1ThxCU9TRZM +5s6R7Nvk3/UjwcpRcqMtnGpTT2QXSnRwvWUfM+bKTwaxz4PjqKpgIc11kwJAjlxk +4CxYf1mDGLwJAoIBAGFJRTNS8ejDKRXyOE6PaGNVOz2FGLTILJoF34JBQfKfYQFE +gEfiOYry9Dr3AdBW2fnLhmi//3jTZoB2CHwnKDhC1h1STSPaadq8KZ+ExuZZbNlE +WxrfzJlpyNPNiZpxJht/54K57Vc0D0PCX2dFb82ZVm5wQqGinJBocpwcugX1NCpW +GaOmmw9xBCigvWjWffriA/kvPhhVQtEaqg4Vwoctwd18FG645Gf7HV4Pd3WrHIrA +6xzHV0T7To6XHpNTpYybbDT50ZW3o4LjellqsPz8yfK+izdbizjJiM+6t/w+uauw +Ag2Tqm8HsWSPwbtVaoIFbLPqs+8EUTaieFp+qnECggEAVuaTdd9uFfrtCNKchh8z +CoAV2uj2pAim6E3//k0j2qURQozVnFdCC6zk9aWkvYB8BGZrXUwUbAjgnp+P8xD3 +cmctG77G+STls66WWMMcAUFFWHGe5y/JMxVvXuSWJ1i+L4m/FVRRWPHhZjznkSdu +jjtZpOLY+N9igIU4JHn/qbKDUrj7w8X1tuMzPuiVBqYDWDe1bg2x/6xS6qLb/71z +xeDdgrKhGOqFud1XARmCaW/M6tdKxg/lp7fokOpZFHBcf2kGL1ogj6LK2HHj+ZGQ +Bc4VZh7H9/BmaPA7IP0S1kKAeBPVOp/TFD737Pm/BC7KQ2DzHusAZEI/jkHfqO/k +0QKCAQEAuiYLn9iLgk4uQO9oaSBGWKrJsR2L2dqI7IWU0X9xJlsQrJKcEeWg4LXt +djLsz0HrxZV/c+Pnh79hmFlBoEmH+hz32D/xd+/qrwwAcMkHAwMbznJu0IIuW2O9 +Uzma++7SvVmr9H0DkUwXFP3jn1A2n3uuI4czqtQ8N7GiH0UAWR5CsIP7azHvZTSj +s4Fzf8rTE6pNqVgQXjrVbI9H/h0uPP4alJbhnPba9mgB1cGmfBEnPkKgYNqSZse+ +95G2TlcK74sKBUSdBKqYBZ4ZUeTXV974Nva9guE9vzDQt1Cj6k0HWISVPUshPzIh +qrdHdxcM6yhA0Z0Gu6zj+Zsy4lU8gA== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 983e52ca294..837263bd70f 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -102,18 +102,32 @@ def rabbitmq_setup_teardown(): # Tests -def test_rabbitmq_select(rabbitmq_cluster): +@pytest.mark.parametrize( + "secure", + [ + pytest.param(0), + pytest.param(1), + ], +) +def test_rabbitmq_select(rabbitmq_cluster, secure): + port = 5672 + if secure: + port = 5671 + instance.query( """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ - SETTINGS rabbitmq_host_port = '{}:5672', + SETTINGS rabbitmq_host_port = '{}:{}', rabbitmq_exchange_name = 'select', rabbitmq_commit_on_select = 1, rabbitmq_format = 'JSONEachRow', - rabbitmq_row_delimiter = '\\n'; + rabbitmq_row_delimiter = '\\n', + rabbitmq_secure = {}; """.format( - rabbitmq_cluster.rabbitmq_host + rabbitmq_cluster.rabbitmq_host, + port, + secure ) ) @@ -3442,18 +3456,18 @@ def test_rabbitmq_handle_error_mode_stream(rabbitmq_cluster): rabbitmq_row_delimiter = '\\n', rabbitmq_handle_error_mode = 'stream'; - + CREATE TABLE test.errors (error Nullable(String), broken_message Nullable(String)) ENGINE = MergeTree() ORDER BY tuple(); CREATE MATERIALIZED VIEW test.errors_view TO test.errors AS SELECT _error as error, _raw_message as broken_message FROM test.rabbit where not isNull(_error); - + CREATE TABLE test.data (key UInt64, value UInt64) ENGINE = MergeTree() ORDER BY key; - + CREATE MATERIALIZED VIEW test.view TO test.data AS SELECT key, value FROM test.rabbit; """.format( From 369aaef92fb280c4b7ae4e5e04d0da2930718e3b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 14 Nov 2023 20:26:32 +0000 Subject: [PATCH 057/192] Automatic style fix --- tests/integration/test_storage_rabbitmq/test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 837263bd70f..adb7f59769a 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -125,9 +125,7 @@ def test_rabbitmq_select(rabbitmq_cluster, secure): rabbitmq_row_delimiter = '\\n', rabbitmq_secure = {}; """.format( - rabbitmq_cluster.rabbitmq_host, - port, - secure + rabbitmq_cluster.rabbitmq_host, port, secure ) ) From 07452b613a6b147c53530d6325fc9038ce58f675 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 14 Nov 2023 20:35:54 +0000 Subject: [PATCH 058/192] Fix certificate's file names --- docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf | 6 +++--- docker/test/integration/runner/misc/rabbitmq/server-ext.cnf | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 docker/test/integration/runner/misc/rabbitmq/server-ext.cnf diff --git a/docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf b/docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf index 307871ba589..258a282907a 100644 --- a/docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf +++ b/docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf @@ -10,6 +10,6 @@ log.file.level = debug listeners.ssl.default = 5671 ssl_options.verify = verify_none ssl_options.fail_if_no_peer_cert = false -ssl_options.cacertfile = /etc/rabbitmq/ca_cert.pem -ssl_options.certfile = /etc/rabbitmq/server_cert.pem -ssl_options.keyfile = /etc/rabbitmq/server_key.pem +ssl_options.cacertfile = /etc/rabbitmq/ca-cert.pem +ssl_options.certfile = /etc/rabbitmq/server-cert.pem +ssl_options.keyfile = /etc/rabbitmq/server-key.pem diff --git a/docker/test/integration/runner/misc/rabbitmq/server-ext.cnf b/docker/test/integration/runner/misc/rabbitmq/server-ext.cnf new file mode 100644 index 00000000000..49859873222 --- /dev/null +++ b/docker/test/integration/runner/misc/rabbitmq/server-ext.cnf @@ -0,0 +1 @@ +subjectAltName=DNS:integration-tests.clickhouse.com From 9d965368a2a337ef6cc2566462670e26fc9e2799 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 15 Nov 2023 08:36:24 +0000 Subject: [PATCH 059/192] Fix build --- src/Backups/RestoreCoordinationRemote.cpp | 2 +- src/Common/escapeForFileName.cpp | 5 ----- src/Common/escapeForFileName.h | 1 - 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp index b54231afcf7..60a83c580f0 100644 --- a/src/Backups/RestoreCoordinationRemote.cpp +++ b/src/Backups/RestoreCoordinationRemote.cpp @@ -245,7 +245,7 @@ bool RestoreCoordinationRemote::acquireInsertingDataForKeeperMap(const String & with_retries.renewZooKeeper(zk); /// we need to remove leading '/' from root_zk_path - auto normalized_root_zk_path = std::string_view{root_zk_path}.substr(1); + auto normalized_root_zk_path = root_zk_path.substr(1); std::string restore_lock_path = fs::path(zookeeper_path) / "keeper_map_tables" / escapeForFileName(normalized_root_zk_path); zk->createAncestors(restore_lock_path); auto code = zk->tryCreate(restore_lock_path, table_unique_id, zkutil::CreateMode::Persistent); diff --git a/src/Common/escapeForFileName.cpp b/src/Common/escapeForFileName.cpp index 790d46a93ec..a1f9bff28d0 100644 --- a/src/Common/escapeForFileName.cpp +++ b/src/Common/escapeForFileName.cpp @@ -6,11 +6,6 @@ namespace DB { std::string escapeForFileName(const std::string & s) -{ - return escapeForFileName(std::string_view{s}); -} - -std::string escapeForFileName(std::string_view s) { std::string res; const char * pos = s.data(); diff --git a/src/Common/escapeForFileName.h b/src/Common/escapeForFileName.h index 279275f55d5..9ae29650804 100644 --- a/src/Common/escapeForFileName.h +++ b/src/Common/escapeForFileName.h @@ -11,7 +11,6 @@ namespace DB */ std::string escapeForFileName(const std::string & s); -std::string escapeForFileName(std::string_view s); std::string unescapeForFileName(const std::string & s); } From 178d23b9511d953e84a2b0b7923699cfcb841cc6 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Wed, 15 Nov 2023 13:53:52 +0100 Subject: [PATCH 060/192] Concat with arbitrary types + tests --- src/Functions/concat.cpp | 64 ++++++++++--------- src/Functions/concatWithSeparator.cpp | 19 +++--- src/Functions/formatString.h | 13 ++-- .../0_stateless/00727_concat.reference | 51 ++++++++++++++- tests/queries/0_stateless/00727_concat.sql | 59 ++++++++++++++++- ...75_show_columns_called_from_clickhouse.sql | 2 +- 6 files changed, 157 insertions(+), 51 deletions(-) diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 350cbee58a3..37311e6c09b 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -1,16 +1,14 @@ #include #include -#include #include #include +#include #include #include -#include #include #include #include #include -#include #include "formatString.h" @@ -18,9 +16,9 @@ namespace DB { namespace ErrorCodes { - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_COLUMN; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int ILLEGAL_COLUMN; } using namespace GatherUtils; @@ -33,7 +31,7 @@ class ConcatImpl : public IFunction { public: static constexpr auto name = Name::name; - explicit ConcatImpl(ContextPtr context_) : context(context_) {} + explicit ConcatImpl(ContextPtr context_) : context(context_) { } static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } String getName() const override { return name; } @@ -68,8 +66,7 @@ public: /// For 3 and more arguments FormatStringImpl is much faster (up to 50-60%). if (arguments.size() == 2) return executeBinary(arguments, input_rows_count); - else - return executeFormatImpl(arguments, input_rows_count); + return executeFormatImpl(arguments, input_rows_count); } private: @@ -113,6 +110,7 @@ private: std::vector offsets(num_arguments); std::vector fixed_string_sizes(num_arguments); std::vector> constant_strings(num_arguments); + std::vector converted_col_ptrs(num_arguments); bool has_column_string = false; bool has_column_fixed_string = false; for (size_t i = 0; i < num_arguments; ++i) @@ -136,16 +134,27 @@ private: } else { - // An arbitrary type argument: converting it to a StringColumn as if `toString` was called - ColumnsWithTypeAndName args; - args.emplace_back(column, arguments[i].type, "tmp"); - const ColumnPtr converted_col_ptr = ConvertImplGenericToString::execute( - args, std::make_shared(), column->size()); - const ColumnString * converted_col_str = assert_cast(converted_col_ptr.get()); + // An arbitrary type argument: converting it to a StringColumn first + const auto serialization = arguments[i].type->getDefaultSerialization(); + ColumnString::MutablePtr converted_col_str = ColumnString::create(); + static FormatSettings format_settings; + + ColumnStringHelpers::WriteHelper write_helper(*converted_col_str, column->size()); + auto & write_buffer = write_helper.getWriteBuffer(); + for (size_t j = 0; j < column->size(); ++j) + { + serialization->serializeText(*column, j, write_buffer, format_settings); + write_helper.rowWritten(); + } + write_helper.finalize(); + // Same as the normal `ColumnString` branch has_column_string = true; data[i] = &converted_col_str->getChars(); offsets[i] = &converted_col_str->getOffsets(); + + // keep the refcounted-pointer around (to be able to use data/offsets later) + converted_col_ptrs[i] = std::move(converted_col_str); } } @@ -193,7 +202,7 @@ public: static constexpr auto name = "concat"; static FunctionOverloadResolverPtr create(ContextPtr context) { return std::make_unique(context); } - explicit ConcatOverloadResolver(ContextPtr context_) : context(context_) {} + explicit ConcatOverloadResolver(ContextPtr context_) : context(context_) { } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 0; } @@ -202,28 +211,25 @@ public: FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override { if (isArray(arguments.at(0).type)) - { return FunctionFactory::instance().getImpl("arrayConcat", context)->build(arguments); - } - else if (isMap(arguments.at(0).type)) - { + if (isMap(arguments.at(0).type)) return FunctionFactory::instance().getImpl("mapConcat", context)->build(arguments); - } - else if (isTuple(arguments.at(0).type)) - { + if (isTuple(arguments.at(0).type)) return FunctionFactory::instance().getImpl("tupleConcat", context)->build(arguments); - } - else - return std::make_unique( - FunctionConcat::create(context), collections::map(arguments, [](const auto & elem) { return elem.type; }), return_type); + return std::make_unique( + FunctionConcat::create(context), + collections::map(arguments, [](const auto & elem) { return elem.type; }), + return_type); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (arguments.size() < 2) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Number of arguments for function {} doesn't match: passed {}, should be at least 2.", - getName(), arguments.size()); + getName(), + arguments.size()); /// We always return Strings from concat, even if arguments were fixed strings. return std::make_shared(); diff --git a/src/Functions/concatWithSeparator.cpp b/src/Functions/concatWithSeparator.cpp index f0a7afbbaa7..f295d86943f 100644 --- a/src/Functions/concatWithSeparator.cpp +++ b/src/Functions/concatWithSeparator.cpp @@ -1,5 +1,5 @@ -#include #include +#include #include #include #include @@ -14,9 +14,9 @@ namespace DB { namespace ErrorCodes { - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_COLUMN; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int ILLEGAL_COLUMN; } namespace @@ -26,7 +26,7 @@ class ConcatWithSeparatorImpl : public IFunction { public: static constexpr auto name = Name::name; - explicit ConcatWithSeparatorImpl(ContextPtr context_) : context(context_) {} + explicit ConcatWithSeparatorImpl(ContextPtr context_) : context(context_) { } static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } @@ -113,8 +113,7 @@ public: else if (const ColumnConst * const_col = checkAndGetColumnConstStringOrFixedString(column.get())) constant_strings[2 * i] = const_col->getValue(); else - throw Exception(ErrorCodes::ILLEGAL_COLUMN, - "Illegal column {} of argument of function {}", column->getName(), getName()); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", column->getName(), getName()); } String pattern; @@ -156,14 +155,14 @@ using FunctionConcatWithSeparatorAssumeInjective = ConcatWithSeparatorImpl(FunctionDocumentation{ - .description=R"( + .description = R"( Returns the concatenation strings separated by string separator. Syntax: concatWithSeparator(sep, expr1, expr2, expr3...) )", .examples{{"concatWithSeparator", "SELECT concatWithSeparator('a', '1', '2', '3')", ""}}, .categories{"String"}}); factory.registerFunction(FunctionDocumentation{ - .description=R"( + .description = R"( Same as concatWithSeparator, the difference is that you need to ensure that concatWithSeparator(sep, expr1, expr2, expr3...) → result is injective, it will be used for optimization of GROUP BY. The function is named “injective” if it always returns different result for different values of arguments. In other words: different arguments never yield identical result. @@ -171,7 +170,7 @@ The function is named “injective” if it always returns different result for .examples{{"concatWithSeparatorAssumeInjective", "SELECT concatWithSeparatorAssumeInjective('a', '1', '2', '3')", ""}}, .categories{"String"}}); - /// Compatibility with Spark: + /// Compatibility with Spark and MySQL: factory.registerAlias("concat_ws", "concatWithSeparator", FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/formatString.h b/src/Functions/formatString.h index 30149e9a5b0..4bdb672caf4 100644 --- a/src/Functions/formatString.h +++ b/src/Functions/formatString.h @@ -1,18 +1,13 @@ #pragma once -#include -#include -#include -#include -#include -#include - - #include #include #include -#include #include +#include +#include +#include +#include namespace DB diff --git a/tests/queries/0_stateless/00727_concat.reference b/tests/queries/0_stateless/00727_concat.reference index af5626b4a11..4785f67bdd9 100644 --- a/tests/queries/0_stateless/00727_concat.reference +++ b/tests/queries/0_stateless/00727_concat.reference @@ -1 +1,50 @@ -Hello, world! +-- Const string + non-const arbitrary type +With 42 +With 43 +With 44 +With 45 +With 46 +With 47 +With 48 +With 49 +With 50 +With 51 +With 52 +With 53 +With 42.42 +With 43.43 +With 44 +With true +With false +With foo +With bar +With foo +With bar +With foo +With bar +With foo +With bar +With fae310ca-d52a-4923-9e9b-02bf67f4b009 +With 2023-11-14 +With 2123-11-14 +With 2023-11-14 05:50:12 +With 2023-11-14 05:50:12.123 +With hallo +With [\'foo\',\'bar\'] +With {"foo":"bar"} +With (42,\'foo\') +With {42:\'foo\'} +With 122.233.64.201 +With 2001:1:130f:2:3:9c0:876a:130b +With (42,43) +With [(0,0),(10,0),(10,10),(0,10)] +With [[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]] +With [[[(0,0),(10,0),(10,10),(0,10)]],[[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]]] +\N +-- Miscellaneous tests +Non-const strings +Three arguments test +3 arguments test with int type +Testing the alias +\N +\N diff --git a/tests/queries/0_stateless/00727_concat.sql b/tests/queries/0_stateless/00727_concat.sql index 800ebd5ec53..3119bd76c0c 100644 --- a/tests/queries/0_stateless/00727_concat.sql +++ b/tests/queries/0_stateless/00727_concat.sql @@ -1 +1,58 @@ -SELECT CONCAT('Hello', ', ', 'world!'); +-- Tags: no-fasttest +-- no-fasttest: json type needs rapidjson library, geo types need s2 geometry + +-- not tested here: (Simple)AggregateFunction, Nested + +SET allow_experimental_object_type = 1; + +SELECT '-- Const string + non-const arbitrary type'; +SELECT concat('With ', materialize(42 :: Int8)); +SELECT concat('With ', materialize(43 :: Int16)); +SELECT concat('With ', materialize(44 :: Int32)); +SELECT concat('With ', materialize(45 :: Int64)); +SELECT concat('With ', materialize(46 :: Int128)); +SELECT concat('With ', materialize(47 :: Int256)); +SELECT concat('With ', materialize(48 :: UInt8)); +SELECT concat('With ', materialize(49 :: UInt16)); +SELECT concat('With ', materialize(50 :: UInt32)); +SELECT concat('With ', materialize(51 :: UInt64)); +SELECT concat('With ', materialize(52 :: UInt128)); +SELECT concat('With ', materialize(53 :: UInt256)); +SELECT concat('With ', materialize(42.42 :: Float32)); +SELECT concat('With ', materialize(43.43 :: Float64)); +SELECT concat('With ', materialize(44.44 :: Decimal(2))); +SELECT concat('With ', materialize(true :: Bool)); +SELECT concat('With ', materialize(false :: Bool)); +SELECT concat('With ', materialize('foo' :: String)); +SELECT concat('With ', materialize('bar' :: FixedString(3))); +SELECT concat('With ', materialize('foo' :: Nullable(String))); +SELECT concat('With ', materialize('bar' :: Nullable(FixedString(3)))); +SELECT concat('With ', materialize('foo' :: LowCardinality(String))); +SELECT concat('With ', materialize('bar' :: LowCardinality(FixedString(3)))); +SELECT concat('With ', materialize('foo' :: LowCardinality(Nullable(String)))); +SELECT concat('With ', materialize('bar' :: LowCardinality(Nullable(FixedString(3))))); +SELECT concat('With ', materialize('fae310ca-d52a-4923-9e9b-02bf67f4b009' :: UUID)); +SELECT concat('With ', materialize('2023-11-14' :: Date)); +SELECT concat('With ', materialize('2123-11-14' :: Date32)); +SELECT concat('With ', materialize('2023-11-14 05:50:12' :: DateTime)); +SELECT concat('With ', materialize('2023-11-14 05:50:12.123' :: DateTime64(3))); +SELECT concat('With ', materialize('hallo' :: Enum('hallo' = 1))); +SELECT concat('With ', materialize(['foo', 'bar'] :: Array(String))); +SELECT concat('With ', materialize('{"foo": "bar"}' :: JSON)); +SELECT concat('With ', materialize((42, 'foo') :: Tuple(Int32, String))); +SELECT concat('With ', materialize(map(42, 'foo') :: Map(Int32, String))); +SELECT concat('With ', materialize('122.233.64.201' :: IPv4)); +SELECT concat('With ', materialize('2001:0001:130F:0002:0003:09C0:876A:130B' :: IPv6)); +SELECT concat('With ', materialize((42, 43) :: Point)); +SELECT concat('With ', materialize([(0,0),(10,0),(10,10),(0,10)] :: Ring)); +SELECT concat('With ', materialize([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]] :: Polygon)); +SELECT concat('With ', materialize([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]] :: MultiPolygon)); +SELECT concat('With ', materialize(NULL :: Nullable(UInt64))); + +SELECT '-- Miscellaneous tests'; +SELECT concat(materialize('Non-const'), materialize(' strings')); +SELECT concat('Three ', 'arguments', ' test'); +SELECT concat(materialize(3 :: Int64), ' arguments test', ' with int type'); +SELECT CONCAT('Testing the ', 'alias'); +SELECT concat(materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); +SELECT concat(42, materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); diff --git a/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql b/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql index 89073bd2943..752367517af 100644 --- a/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql +++ b/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql @@ -5,7 +5,7 @@ -- Tests the output of SHOW COLUMNS when called through the ClickHouse protocol. -- ----------------------------------------------------------------------------------- --- Please keep this test in-sync with 02775_show_columns_called_through_mysql.sql +-- Please keep this test in-sync with 02775_show_columns_called_from_mysql.expect -- ----------------------------------------------------------------------------------- DROP TABLE IF EXISTS tab; From 0aaea6e51ddbd13ad5becafd010e07dd652ab3c7 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Wed, 15 Nov 2023 14:42:38 +0100 Subject: [PATCH 061/192] Fix ColumnConst serialization issues, more tests --- src/Functions/concat.cpp | 9 +++---- .../0_stateless/00727_concat.reference | 17 ++++++++++--- tests/queries/0_stateless/00727_concat.sql | 24 +++++++++++++++---- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 37311e6c09b..346f96e4f03 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -135,6 +135,7 @@ private: else { // An arbitrary type argument: converting it to a StringColumn first + const auto full_column = column->convertToFullIfNeeded(); const auto serialization = arguments[i].type->getDefaultSerialization(); ColumnString::MutablePtr converted_col_str = ColumnString::create(); static FormatSettings format_settings; @@ -143,7 +144,7 @@ private: auto & write_buffer = write_helper.getWriteBuffer(); for (size_t j = 0; j < column->size(); ++j) { - serialization->serializeText(*column, j, write_buffer, format_settings); + serialization->serializeText(*full_column, j, write_buffer, format_settings); write_helper.rowWritten(); } write_helper.finalize(); @@ -210,11 +211,11 @@ public: FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override { - if (isArray(arguments.at(0).type)) + if (std::ranges::all_of(arguments, [](const auto & elem) { return isArray(elem.type); })) return FunctionFactory::instance().getImpl("arrayConcat", context)->build(arguments); - if (isMap(arguments.at(0).type)) + if (std::ranges::all_of(arguments, [](const auto & elem) { return isMap(elem.type); })) return FunctionFactory::instance().getImpl("mapConcat", context)->build(arguments); - if (isTuple(arguments.at(0).type)) + if (std::ranges::all_of(arguments, [](const auto & elem) { return isTuple(elem.type); })) return FunctionFactory::instance().getImpl("tupleConcat", context)->build(arguments); return std::make_unique( FunctionConcat::create(context), diff --git a/tests/queries/0_stateless/00727_concat.reference b/tests/queries/0_stateless/00727_concat.reference index 4785f67bdd9..9b6a8b3857b 100644 --- a/tests/queries/0_stateless/00727_concat.reference +++ b/tests/queries/0_stateless/00727_concat.reference @@ -24,6 +24,7 @@ With foo With bar With foo With bar +With 42 With fae310ca-d52a-4923-9e9b-02bf67f4b009 With 2023-11-14 With 2123-11-14 @@ -40,11 +41,21 @@ With (42,43) With [(0,0),(10,0),(10,10),(0,10)] With [[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]] With [[[(0,0),(10,0),(10,10),(0,10)]],[[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]]] +-- NULL arguments \N --- Miscellaneous tests +\N +\N +\N +\N +\N +\N +-- Various arguments tests Non-const strings +Two arguments test Three arguments test 3 arguments test with int type +42144 +42144255 +42144 +42144255 Testing the alias -\N -\N diff --git a/tests/queries/0_stateless/00727_concat.sql b/tests/queries/0_stateless/00727_concat.sql index 3119bd76c0c..ba76ff53884 100644 --- a/tests/queries/0_stateless/00727_concat.sql +++ b/tests/queries/0_stateless/00727_concat.sql @@ -4,6 +4,7 @@ -- not tested here: (Simple)AggregateFunction, Nested SET allow_experimental_object_type = 1; +SET allow_suspicious_low_cardinality_types=1; SELECT '-- Const string + non-const arbitrary type'; SELECT concat('With ', materialize(42 :: Int8)); @@ -31,6 +32,7 @@ SELECT concat('With ', materialize('foo' :: LowCardinality(String))); SELECT concat('With ', materialize('bar' :: LowCardinality(FixedString(3)))); SELECT concat('With ', materialize('foo' :: LowCardinality(Nullable(String)))); SELECT concat('With ', materialize('bar' :: LowCardinality(Nullable(FixedString(3))))); +SELECT concat('With ', materialize(42 :: LowCardinality(Nullable(UInt32)))); SELECT concat('With ', materialize('fae310ca-d52a-4923-9e9b-02bf67f4b009' :: UUID)); SELECT concat('With ', materialize('2023-11-14' :: Date)); SELECT concat('With ', materialize('2123-11-14' :: Date32)); @@ -47,12 +49,26 @@ SELECT concat('With ', materialize((42, 43) :: Point)); SELECT concat('With ', materialize([(0,0),(10,0),(10,10),(0,10)] :: Ring)); SELECT concat('With ', materialize([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]] :: Polygon)); SELECT concat('With ', materialize([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]] :: MultiPolygon)); -SELECT concat('With ', materialize(NULL :: Nullable(UInt64))); -SELECT '-- Miscellaneous tests'; +SELECT '-- NULL arguments'; +SELECT concat(NULL, NULL); +SELECT concat(NULL, materialize(NULL :: Nullable(UInt64))); +SELECT concat(materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); + +SELECT concat(42, materialize(NULL :: Nullable(UInt64))); +SELECT concat('42', materialize(NULL :: Nullable(UInt64))); + +SELECT concat(42, materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); +SELECT concat('42', materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); + +SELECT '-- Various arguments tests'; SELECT concat(materialize('Non-const'), materialize(' strings')); +SELECT concat('Two arguments ', 'test'); SELECT concat('Three ', 'arguments', ' test'); SELECT concat(materialize(3 :: Int64), ' arguments test', ' with int type'); +SELECT concat(materialize(42 :: Int32), materialize(144 :: UInt64)); +SELECT concat(materialize(42 :: Int32), materialize(144 :: UInt64), materialize(255 :: UInt32)); +SELECT concat(42, 144); +SELECT concat(42, 144, 255); + SELECT CONCAT('Testing the ', 'alias'); -SELECT concat(materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); -SELECT concat(42, materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); From 62378b421928268c6285cc1cd65b446a799291fb Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 15 Nov 2023 13:56:51 +0000 Subject: [PATCH 062/192] Fixing style. --- src/Storages/StorageInput.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageInput.h b/src/Storages/StorageInput.h index 9ac082a9add..82ddda4cdd1 100644 --- a/src/Storages/StorageInput.h +++ b/src/Storages/StorageInput.h @@ -10,7 +10,7 @@ namespace DB class StorageInput final : public IStorage { - friend class ReadFromInput; + friend class ReadFromInput; public: StorageInput(const StorageID & table_id, const ColumnsDescription & columns_); From c7cd4fa972893ad6dead0a6f3fe9038c2c61ebad Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Wed, 15 Nov 2023 14:57:34 +0100 Subject: [PATCH 063/192] Update concat docs --- .../sql-reference/functions/string-functions.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 4df987b5e2a..dc324e8e331 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -439,7 +439,7 @@ concat(s1, s2, ...) **Arguments** -Values of type String or FixedString. +Values of arbitrary types. If an argument is not a String or FixedString, it is converted to the String type using the default serialization. **Returned values** @@ -461,6 +461,20 @@ Result: └─────────────────────────────┠``` +**Example** + +```sql +SELECT concat(42, 144); +``` + +Result: + +```result +┌─concat(42, 144)─┠+│ 42144 │ +└─────────────────┠+``` + ## concatAssumeInjective Like [concat](#concat) but assumes that `concat(s1, s2, ...) → sn` is injective. Can be used for optimization of GROUP BY. From 1ea74cee3a5b664687fc8c059da0fed144fd9ea9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 15 Nov 2023 15:04:07 +0100 Subject: [PATCH 064/192] Early disconnect if there is authentication failure with interserver secret --- src/Server/TCPHandler.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index f929d0f5ff9..e7c40092077 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -588,6 +587,10 @@ void TCPHandler::runImpl() } catch (const Exception & e) { + /// Authentication failure with interserver secret. + if (e.code() == ErrorCodes::AUTHENTICATION_FAILED) + throw; + state.io.onException(); exception.reset(e.clone()); @@ -1717,7 +1720,18 @@ void TCPHandler::receiveQuery() { client_info.interface = ClientInfo::Interface::TCP_INTERSERVER; #if USE_SSL - String cluster_secret = server.context()->getCluster(cluster)->getSecret(); + + String cluster_secret; + try + { + cluster_secret = server.context()->getCluster(cluster)->getSecret(); + } + catch (const Exception & e) + { + auto exception = Exception::createRuntime(ErrorCodes::AUTHENTICATION_FAILED, e.message()); + session->onAuthenticationFailure(/* user_name= */ std::nullopt, socket().peerAddress(), exception); + throw exception; /// NOLINT + } if (salt.empty() || cluster_secret.empty()) { From 20cfe91ff967733bdbc34244759eb26d379c8869 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Wed, 15 Nov 2023 15:21:19 +0100 Subject: [PATCH 065/192] Remove unused error codes --- src/Functions/concat.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 346f96e4f03..081096b745e 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -16,9 +16,7 @@ namespace DB { namespace ErrorCodes { -extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -extern const int ILLEGAL_COLUMN; } using namespace GatherUtils; From a7543e3c7c36eee293c169c408a7ec2607fd86ad Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 15 Nov 2023 16:05:30 +0100 Subject: [PATCH 066/192] Fix test --- .../0_stateless/01555_system_distribution_queue_mask.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql index 3a90765226a..7ade1d24c59 100644 --- a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql +++ b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql @@ -17,7 +17,7 @@ system stop distributed sends dist_01555; insert into dist_01555 values (1)(2); -- since test_cluster_with_incorrect_pw contains incorrect password ignore error -system flush distributed dist_01555; -- { serverError 516 } +system flush distributed dist_01555; -- { clientError ATTEMPT_TO_READ_AFTER_EOF } select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1'), extract(last_exception, 'AUTHENTICATION_FAILED'), dateDiff('s', last_exception_time, now()) < 3600 from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV; drop table dist_01555; @@ -30,7 +30,7 @@ create table dist_01555 (key Int) Engine=Distributed(test_cluster_with_incorrect insert into dist_01555 values (1)(2); -- since test_cluster_with_incorrect_pw contains incorrect password ignore error -system flush distributed dist_01555; -- { serverError 516 } +system flush distributed dist_01555; -- { clientError ATTEMPT_TO_READ_AFTER_EOF } select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1'), extract(last_exception, 'AUTHENTICATION_FAILED'), dateDiff('s', last_exception_time, now()) < 3600 from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV; drop table dist_01555; From f21dd37d1838c2041a4dcd7671d616a3dca817a8 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Nov 2023 15:42:00 +0000 Subject: [PATCH 067/192] Some fixups --- .../en/sql-reference/functions/string-functions.md | 12 +++++++++--- src/Columns/ColumnStringHelpers.h | 2 +- src/Functions/concat.cpp | 14 ++++++-------- src/Functions/formatString.h | 10 +++++----- .../02775_show_columns_called_from_clickhouse.sql | 2 +- .../02775_show_columns_called_from_mysql.expect | 2 +- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index dc324e8e331..4b6e0356301 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -429,7 +429,7 @@ SELECT format('{} {}', 'Hello', 'World') ## concat -Concatenates the strings listed in the arguments without separator. +Concatenates the given arguments. **Syntax** @@ -439,7 +439,9 @@ concat(s1, s2, ...) **Arguments** -Values of arbitrary types. If an argument is not a String or FixedString, it is converted to the String type using the default serialization. +At least two values of arbitrary type. + +Arguments which are not of types [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md) are converted to strings using their default serialization. As this decreases performance, it is not recommended to use non-String/FixedString arguments. **Returned values** @@ -449,6 +451,8 @@ If any of arguments is `NULL`, the function returns `NULL`. **Example** +Query: + ``` sql SELECT concat('Hello, ', 'World!'); ``` @@ -461,7 +465,7 @@ Result: └─────────────────────────────┠``` -**Example** +Query: ```sql SELECT concat(42, 144); @@ -540,6 +544,8 @@ Concatenates the given strings with a given separator. concatWithSeparator(sep, expr1, expr2, expr3...) ``` +Alias: `concat_ws` + **Arguments** - sep — separator. Const [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md). diff --git a/src/Columns/ColumnStringHelpers.h b/src/Columns/ColumnStringHelpers.h index 851486e490a..97b52506ae0 100644 --- a/src/Columns/ColumnStringHelpers.h +++ b/src/Columns/ColumnStringHelpers.h @@ -62,7 +62,7 @@ public: return buffer; } - inline void rowWritten() + void rowWritten() { if constexpr (std::is_same_v) { diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 081096b745e..9aa6de5d219 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -91,7 +91,6 @@ private: else { /// Fallback: use generic implementation for not very important cases. - /// Concat of arbitrary types also goes here. return executeFormatImpl(arguments, input_rows_count); } @@ -108,7 +107,7 @@ private: std::vector offsets(num_arguments); std::vector fixed_string_sizes(num_arguments); std::vector> constant_strings(num_arguments); - std::vector converted_col_ptrs(num_arguments); + std::vector converted_col_ptrs(num_arguments); bool has_column_string = false; bool has_column_fixed_string = false; for (size_t i = 0; i < num_arguments; ++i) @@ -132,14 +131,13 @@ private: } else { - // An arbitrary type argument: converting it to a StringColumn first + /// A non-String/non-FixedString-type argument: use the default serialization to convert it to String const auto full_column = column->convertToFullIfNeeded(); const auto serialization = arguments[i].type->getDefaultSerialization(); - ColumnString::MutablePtr converted_col_str = ColumnString::create(); - static FormatSettings format_settings; - + auto converted_col_str = ColumnString::create(); ColumnStringHelpers::WriteHelper write_helper(*converted_col_str, column->size()); auto & write_buffer = write_helper.getWriteBuffer(); + FormatSettings format_settings; for (size_t j = 0; j < column->size(); ++j) { serialization->serializeText(*full_column, j, write_buffer, format_settings); @@ -147,12 +145,12 @@ private: } write_helper.finalize(); - // Same as the normal `ColumnString` branch + /// Same as the normal `ColumnString` branch has_column_string = true; data[i] = &converted_col_str->getChars(); offsets[i] = &converted_col_str->getOffsets(); - // keep the refcounted-pointer around (to be able to use data/offsets later) + /// Keep the refcounted-pointer alive converted_col_ptrs[i] = std::move(converted_col_str); } } diff --git a/src/Functions/formatString.h b/src/Functions/formatString.h index 4bdb672caf4..315e5c06227 100644 --- a/src/Functions/formatString.h +++ b/src/Functions/formatString.h @@ -1,14 +1,14 @@ #pragma once +#include +#include +#include +#include + #include #include #include #include -#include -#include -#include -#include - namespace DB { diff --git a/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql b/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql index 752367517af..3bbcbb1a535 100644 --- a/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql +++ b/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql @@ -5,7 +5,7 @@ -- Tests the output of SHOW COLUMNS when called through the ClickHouse protocol. -- ----------------------------------------------------------------------------------- --- Please keep this test in-sync with 02775_show_columns_called_from_mysql.expect +-- Please keep this test in-sync with 02775_show_columns_called_from_clickhouse.expect -- ----------------------------------------------------------------------------------- DROP TABLE IF EXISTS tab; diff --git a/tests/queries/0_stateless/02775_show_columns_called_from_mysql.expect b/tests/queries/0_stateless/02775_show_columns_called_from_mysql.expect index bef5bd10ff3..8ba5774820e 100755 --- a/tests/queries/0_stateless/02775_show_columns_called_from_mysql.expect +++ b/tests/queries/0_stateless/02775_show_columns_called_from_mysql.expect @@ -6,7 +6,7 @@ # Tests the output of SHOW COLUMNS when called through the MySQL protocol. # ----------------------------------------------------------------------------------- -# Please keep this test in-sync with 02775_show_columns_called_through_clickhouse.sql +# Please keep this test in-sync with 02775_show_columns_called_from_clickhouse.sql # ----------------------------------------------------------------------------------- set basedir [file dirname $argv0] From e2b25aab0c4032c0b36f2124d02f9b4bfd36d242 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Nov 2023 15:53:38 +0000 Subject: [PATCH 068/192] Fixups, pt. II --- src/Functions/FunctionsConversion.h | 4 ++-- src/Functions/concat.cpp | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index e3ec7ebd320..d7c2c70884b 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -1247,9 +1247,9 @@ struct ConvertImplGenericToString FormatSettings format_settings; auto serialization = type.getDefaultSerialization(); - for (size_t i = 0; i < size; ++i) + for (size_t row = 0; row < size; ++row) { - serialization->serializeText(col_from, i, write_buffer, format_settings); + serialization->serializeText(col_from, row, write_buffer, format_settings); write_helper.rowWritten(); } diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 9aa6de5d219..f426f662868 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -132,15 +132,15 @@ private: else { /// A non-String/non-FixedString-type argument: use the default serialization to convert it to String - const auto full_column = column->convertToFullIfNeeded(); - const auto serialization = arguments[i].type->getDefaultSerialization(); + auto full_column = column->convertToFullIfNeeded(); + auto serialization = arguments[i].type->getDefaultSerialization(); auto converted_col_str = ColumnString::create(); ColumnStringHelpers::WriteHelper write_helper(*converted_col_str, column->size()); auto & write_buffer = write_helper.getWriteBuffer(); FormatSettings format_settings; - for (size_t j = 0; j < column->size(); ++j) + for (size_t row = 0; row < column->size(); ++row) { - serialization->serializeText(*full_column, j, write_buffer, format_settings); + serialization->serializeText(*full_column, row, write_buffer, format_settings); write_helper.rowWritten(); } write_helper.finalize(); @@ -150,7 +150,7 @@ private: data[i] = &converted_col_str->getChars(); offsets[i] = &converted_col_str->getOffsets(); - /// Keep the refcounted-pointer alive + /// Keep the pointer alive converted_col_ptrs[i] = std::move(converted_col_str); } } From 6e3e6383ba0ad5b317c8189ac5a654ee5bb9057b Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 15 Nov 2023 19:00:27 +0100 Subject: [PATCH 069/192] perf check 2 --- base/poco/Net/src/HTTPServerSession.cpp | 1 - base/poco/Net/src/HTTPSession.cpp | 32 +++++++++----------- src/Disks/ObjectStorages/S3/diskSettings.cpp | 2 +- src/IO/S3/PocoHTTPClient.cpp | 1 - 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/base/poco/Net/src/HTTPServerSession.cpp b/base/poco/Net/src/HTTPServerSession.cpp index f6d3c4e5b92..d4f2b24879e 100644 --- a/base/poco/Net/src/HTTPServerSession.cpp +++ b/base/poco/Net/src/HTTPServerSession.cpp @@ -26,7 +26,6 @@ HTTPServerSession::HTTPServerSession(const StreamSocket& socket, HTTPServerParam _maxKeepAliveRequests(pParams->getMaxKeepAliveRequests()) { setTimeout(pParams->getTimeout()); - this->socket().setReceiveTimeout(pParams->getTimeout()); } diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index 97decded282..9ebbd7d04cd 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -94,24 +94,22 @@ void HTTPSession::setTimeout(const Poco::Timespan& timeout) void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco::Timespan& sendTimeout, const Poco::Timespan& receiveTimeout) { _connectionTimeout = connectionTimeout; - _sendTimeout = sendTimeout; - _receiveTimeout = receiveTimeout; -// if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) -// { -// _sendTimeout = sendTimeout; -// -// if (connected()) -// _socket.setSendTimeout(_sendTimeout); -// } -// -// if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) -// { -// _receiveTimeout = receiveTimeout; -// -// if (connected()) -// _socket.setReceiveTimeout(_receiveTimeout); -// } + if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) + { + _sendTimeout = sendTimeout; + + if (connected()) + _socket.setSendTimeout(_sendTimeout); + } + + if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) + { + _receiveTimeout = receiveTimeout; + + if (connected()) + _socket.setReceiveTimeout(_receiveTimeout); + } } diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index b0384daab2d..0232a6eb070 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -67,7 +67,7 @@ std::unique_ptr getClient( config_prefix + ".http_keep_alive_timeout_ms", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT * 1000); client_configuration.http_connection_pool_size = config.getUInt(config_prefix + ".http_connection_pool_size", 1000); client_configuration.wait_on_pool_size_limit = false; - client_configuration.s3_use_adaptive_timeouts = config.getUInt( + client_configuration.s3_use_adaptive_timeouts = config.getBool( config_prefix + ".use_adaptive_timeouts", client_configuration.s3_use_adaptive_timeouts); /* diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 904e2324145..f681362e607 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include From f3f839205197a73cf3c4b40dd8d67077839701ba Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 15 Nov 2023 17:37:00 +0100 Subject: [PATCH 070/192] upgrade replication protocol --- src/Storages/MergeTree/DataPartsExchange.cpp | 83 ++++++++++++++----- src/Storages/MergeTree/MergeTreeSettings.h | 5 +- .../MergeTree/ReplicatedMergeTreeSink.cpp | 8 ++ tests/clickhouse-test | 3 + ...plication_protocol_wait_for_part.reference | 1 + ...916_replication_protocol_wait_for_part.sql | 23 +++++ 6 files changed, 101 insertions(+), 22 deletions(-) create mode 100644 tests/queries/0_stateless/02916_replication_protocol_wait_for_part.reference create mode 100644 tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 4545b2b98ae..7fd6f59ed69 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -65,8 +65,7 @@ constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID = 5; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_ZERO_COPY = 6; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION = 7; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION = 8; -// Reserved for ALTER PRIMARY KEY -// constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PRIMARY_KEY = 9; +constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE = 9; std::string getEndpointId(const std::string & node_id) { @@ -122,7 +121,7 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write MergeTreePartInfo::fromPartName(part_name, data.format_version); /// We pretend to work as older server version, to be sure that client will correctly process our version - response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION))}); + response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE))}); LOG_TRACE(log, "Sending part {}", part_name); @@ -140,6 +139,29 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write { part = findPart(part_name); + /// Ephemeral zero-copy lock may be lost for PreActive parts + /// do not expose PreActive parts + if (client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE) + { + bool part_is_ready = part->getState() != MergeTreeDataPartState::PreActive; + writeBinary(part_is_ready, out); + + if (!part_is_ready) + { + LOG_TRACE(log, "Part {} is in PreActive state, reply to the client that part is not ready yet", part_name); + return; + } + } + else + { + bool zero_copy_enabled = data.getSettings()->allow_remote_fs_zero_copy_replication; + if (part->getState() == MergeTreeDataPartState::PreActive && zero_copy_enabled) + { + /// report error, client will try again later, error message would be printed + throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in table", part_name); + } + } + CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedSend}; if (part->getDataPartStorage().isStoredOnRemoteDisk()) @@ -357,12 +379,8 @@ MergeTreeData::DataPartPtr Service::findPart(const String & name) /// determine the local state of the part, so queries for the parts in these states are completely normal. MergeTreeData::DataPartPtr part; - /// Ephemeral zero-copy lock may be lost for PreActive parts - bool zero_copy_enabled = data.getSettings()->allow_remote_fs_zero_copy_replication; - if (zero_copy_enabled) - part = data.getPartIfExists(name, {MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated}); - else - part = data.getPartIfExists(name, {MergeTreeDataPartState::PreActive, MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated}); + part = data.getPartIfExists(name, {MergeTreeDataPartState::PreActive, MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated}); + if (part) return part; @@ -424,7 +442,7 @@ std::pair Fetcher::fetchSelected { {"endpoint", endpoint_id}, {"part", part_name}, - {"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION)}, + {"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE)}, {"compress", "false"} }); @@ -482,17 +500,42 @@ std::pair Fetcher::fetchSelected creds.setPassword(password); } - std::unique_ptr in = std::make_unique( - uri, - Poco::Net::HTTPRequest::HTTP_POST, - nullptr, - timeouts, - creds, - DBMS_DEFAULT_BUFFER_SIZE, - 0, /* no redirects */ - static_cast(data_settings->replicated_max_parallel_fetches_for_host)); + std::unique_ptr in; + int server_protocol_version = 0; + bool part_is_ready = true; - int server_protocol_version = parse(in->getResponseCookie("server_protocol_version", "0")); + static const UInt32 part_not_ready_attempts = 5; + static const UInt32 wait_sleep_time_ms = 100; + + for (UInt32 attempt = 1; attempt <= part_not_ready_attempts; ++attempt) + { + in = std::make_unique( + uri, + Poco::Net::HTTPRequest::HTTP_POST, + nullptr, + timeouts, + creds, + DBMS_DEFAULT_BUFFER_SIZE, + 0, /* no redirects */ + static_cast(data_settings->replicated_max_parallel_fetches_for_host)); + + server_protocol_version = parse(in->getResponseCookie("server_protocol_version", "0")); + + if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE) + readBinary(part_is_ready, *in); + + if (part_is_ready) + break; + + sleepForMilliseconds(wait_sleep_time_ms); + + if (blocker.isCancelled()) + throw Exception(ErrorCodes::ABORTED, "Fetching of part was cancelled"); + } + + if (!part_is_ready) + throw Exception(ErrorCodes::ABORTED, "Part {} is still not ready in host {} after {} attempts, try another host", + part_name, host, part_not_ready_attempts); String remote_fs_metadata = parse(in->getResponseCookie("remote_fs_metadata", "")); diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 53876e77376..15c54ee3791 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -83,7 +83,8 @@ struct Settings; M(UInt64, max_delay_to_insert, 1, "Max delay of inserting data into MergeTree table in seconds, if there are a lot of unmerged parts in single partition.", 0) \ M(UInt64, min_delay_to_insert_ms, 10, "Min delay of inserting data into MergeTree table in milliseconds, if there are a lot of unmerged parts in single partition.", 0) \ M(UInt64, max_parts_in_total, 100000, "If more than this number active parts in all partitions in total, throw 'Too many parts ...' exception.", 0) \ - M(Bool, async_insert, false, "If true, data from INSERT query is stored in queue and later flushed to table in background.", 0) \ + M(Bool, async_insert, false, "If true, data from INSERT query is stored in queue and later flushed to table in background.", 0) \ + M(Milliseconds, sleep_before_commit_local_part_in_replicated_table_ms, 0, "For testing. Do not change it.", 0) \ \ /* Part removal settings. */ \ M(UInt64, simultaneous_parts_removal_limit, 0, "Maximum number of parts to remove during one CleanupThread iteration (0 means unlimited).", 0) \ @@ -121,7 +122,7 @@ struct Settings; M(UInt64, max_replicated_sends_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for replicated sends. Zero means unlimited.", 0) \ M(Milliseconds, wait_for_unique_parts_send_before_shutdown_ms, 0, "Before shutdown table will wait for required amount time for unique parts (exist only on current replica) to be fetched by other replicas (0 means disabled).", 0) \ M(Float, fault_probability_before_part_commit, 0, "For testing. Do not change it.", 0) \ - M(Float, fault_probability_after_part_commit, 0, "For testing. Do not change it.", 0) \ + M(Float, fault_probability_after_part_commit, 0, "For testing. Do not change it.", 0) \ \ /** Check delay of replicas settings. */ \ M(UInt64, min_relative_delay_to_measure, 120, "Calculate relative replica delay only if absolute delay is not less that this value.", 0) \ diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 7de5d46c66b..37f808824b5 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -924,6 +924,14 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: Coordination::Error multi_code = zookeeper->tryMultiNoThrow(ops, responses); /// 1 RTT if (multi_code == Coordination::Error::ZOK) { + auto sleep_before_commit_local_part_in_replicated_table_ms = storage.getSettings()->sleep_before_commit_local_part_in_replicated_table_ms; + if (sleep_before_commit_local_part_in_replicated_table_ms.totalMilliseconds()) + { + LOG_INFO(log, "committing part {}, triggered sleep_before_commit_local_part_in_replicated_table_ms {}", + part->name, sleep_before_commit_local_part_in_replicated_table_ms.totalMilliseconds()); + sleepForMilliseconds(sleep_before_commit_local_part_in_replicated_table_ms.totalMilliseconds()); + } + part->new_part_was_committed_to_zookeeper_after_rename_on_disk = true; transaction.commit(); storage.merge_selecting_task->schedule(); diff --git a/tests/clickhouse-test b/tests/clickhouse-test index cab7d7e79ff..048f848ff27 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -673,6 +673,9 @@ class MergeTreeSettingsRandomizer: "primary_key_compress_block_size": lambda: random.randint(8000, 100000), "replace_long_file_name_to_hash": lambda: random.randint(0, 1), "max_file_name_length": threshold_generator(0.3, 0.3, 0, 128), + "sleep_before_commit_local_part_in_replicated_table_ms": threshold_generator( + 0.3, 0.3, 0, 250 + ), } @staticmethod diff --git a/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.reference b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.reference new file mode 100644 index 00000000000..0cfbf08886f --- /dev/null +++ b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.reference @@ -0,0 +1 @@ +2 diff --git a/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql new file mode 100644 index 00000000000..ed9cfd00b45 --- /dev/null +++ b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql @@ -0,0 +1,23 @@ +-- Tags: no-replicated-database, no-fasttest +-- Tag no-replicated-database: different number of replicas + +create table tableIn (n int) + engine=ReplicatedMergeTree('/test/02916/{database}/table', '1') + order by tuple() + settings + storage_policy='s3_cache', + allow_remote_fs_zero_copy_replication=1, + sleep_before_commit_local_part_in_replicated_table_ms=50000; +create table tableOut (n int) + engine=ReplicatedMergeTree('/test/02916/{database}/table', '2') + order by tuple() + settings + storage_policy='s3_cache', + allow_remote_fs_zero_copy_replication=1; + +SET send_logs_level = 'error'; + +insert into tableIn values(1); +insert into tableIn values(2); +system sync replica tableOut; +select count() from tableOut; From d862dfdf9c753e17896f1c3a9d5cb01e71a5cee3 Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Wed, 15 Nov 2023 18:38:23 +0000 Subject: [PATCH 071/192] fix comments Signed-off-by: Jianfei Hu --- src/IO/S3/Credentials.cpp | 53 +++++++++++---------------------------- src/IO/S3/Credentials.h | 11 ++++---- 2 files changed, 21 insertions(+), 43 deletions(-) diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index 7d6ed094486..bc336634114 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -1,9 +1,4 @@ -#include -#include #include -#include -#include -#include "Common/Exception.h" #if USE_AWS_S3 @@ -21,22 +16,24 @@ # include # include - +# include # include # include +# include +# include # include # include -#include -#include - - -#include -#include -#include -#include -#include +# include +# include +# include +# include +# include +# include +# include +# include +# include namespace DB @@ -65,7 +62,7 @@ bool areCredentialsEmptyOrExpired(const Aws::Auth::AWSCredentials & credentials, } const char SSO_CREDENTIALS_PROVIDER_LOG_TAG[] = "SSOCredentialsProvider"; -const int AVAILABILITY_ZONE_REQUEST_TIMEOUT_SECONDS = 3; +constexpr int AVAILABILITY_ZONE_REQUEST_TIMEOUT_SECONDS = 3; } @@ -275,11 +272,11 @@ String getGCPAvailabilityZoneOrException() boost::split(zone_info, response_data, boost::is_any_of("/")); /// We expect GCP returns a string as "projects/123456789/zones/us-central1a". if (zone_info.size() != 4) - throw DB::Exception(ErrorCodes::GCP_ERROR, "Invalid format of GCP zone information, expect projects//zones/, got {}", response_data); + throw DB::Exception(ErrorCodes::GCP_ERROR, "Invalid format of GCP zone information, expect projects//zones/"); return zone_info[3]; } -String getRunningAvailabilityZoneImpl() +String getRunningAvailabilityZone() { LOG_INFO(&Poco::Logger::get("Application"), "Trying to detect the availability zone."); try @@ -302,26 +299,6 @@ String getRunningAvailabilityZoneImpl() } } -std::variant getRunningAvailabilityZoneImplOrException() -{ - try - { - return getRunningAvailabilityZoneImpl(); - } - catch (...) - { - return std::current_exception(); - } -} - -String getRunningAvailabilityZone() -{ - static auto az_or_exception = getRunningAvailabilityZoneImplOrException(); - if (const auto * az = std::get_if(&az_or_exception)) - return *az; - else - std::rethrow_exception(std::get(az_or_exception)); -} AWSEC2InstanceProfileConfigLoader::AWSEC2InstanceProfileConfigLoader(const std::shared_ptr & client_, bool use_secure_pull_) : client(client_) diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index a978679348f..b1666e13757 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -1,12 +1,13 @@ #pragma once -#include -#include -#include #include "config.h" #if USE_AWS_S3 +# include +# include +# include + # include # include # include @@ -22,7 +23,7 @@ namespace DB::S3 inline static constexpr uint64_t DEFAULT_EXPIRATION_WINDOW_SECONDS = 120; /// In GCP metadata service can be accessed via DNS regardless of IPv4 or IPv6. -static constexpr char GCP_METADATA_SERVICE_ENDPOINT[] = "http://metadata.google.internal"; +static inline constexpr char GCP_METADATA_SERVICE_ENDPOINT[] = "http://metadata.google.internal"; /// getRunningAvailabilityZone returns the availability zone of the underlying compute resources where the current process runs. String getRunningAvailabilityZone(); @@ -59,7 +60,7 @@ public: virtual Aws::String getCurrentRegion() const; - friend String getRunningAvailabilityZoneImpl(); + friend String getRunningAvailabilityZone(); private: std::pair getEC2MetadataToken(const std::string & user_agent_string) const; From d0398e3c1d1f55281e65633a4947c807d3c0c022 Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Wed, 15 Nov 2023 18:47:28 +0000 Subject: [PATCH 072/192] remove variant header Signed-off-by: Jianfei Hu --- src/IO/S3/Credentials.cpp | 1 - src/IO/S3/Credentials.h | 1 - 2 files changed, 2 deletions(-) diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index bc336634114..9ab21465593 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -21,7 +21,6 @@ # include # include -# include # include # include diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index b1666e13757..1f35443adf4 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -6,7 +6,6 @@ # include # include -# include # include # include From ea92dbb1c74c700b4df4172999d6ca504ff593bf Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Wed, 15 Nov 2023 19:18:38 +0000 Subject: [PATCH 073/192] fix build for non USE_S3 case Signed-off-by: Jianfei Hu --- src/IO/S3/Credentials.cpp | 6 ++++-- src/IO/S3/Credentials.h | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index 9ab21465593..e25f4551723 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -784,15 +784,17 @@ S3CredentialsProviderChain::S3CredentialsProviderChain( #else +# include + namespace DB { namespace S3 { -String getRunningAvailabilityZone() +std::string getRunningAvailabilityZone() { - throw Poco::Exception("Does not support availability zone detection for non-cloud environment"); + throw std::runtime_error("Does not support availability zone detection for non-cloud environment"); } } diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index 1f35443adf4..d8d103a847a 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -25,7 +25,7 @@ inline static constexpr uint64_t DEFAULT_EXPIRATION_WINDOW_SECONDS = 120; static inline constexpr char GCP_METADATA_SERVICE_ENDPOINT[] = "http://metadata.google.internal"; /// getRunningAvailabilityZone returns the availability zone of the underlying compute resources where the current process runs. -String getRunningAvailabilityZone(); +std::string getRunningAvailabilityZone(); class AWSEC2MetadataClient : public Aws::Internal::AWSHttpResourceClient { @@ -189,12 +189,14 @@ public: #else +# include + namespace DB { namespace S3 { -String getRunningAvailabilityZone(); +std::string getRunningAvailabilityZone(); } } From 3bbb329dd0f091b49a6dd771820110cde0e3a052 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 16 Nov 2023 00:13:05 +0100 Subject: [PATCH 074/192] Fix tests --- src/Server/TCPHandler.cpp | 15 +++++++++++++-- tests/queries/0_stateless/01119_session_log.sql | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index e7c40092077..884fc45f763 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -587,8 +587,19 @@ void TCPHandler::runImpl() } catch (const Exception & e) { - /// Authentication failure with interserver secret. - if (e.code() == ErrorCodes::AUTHENTICATION_FAILED) + /// Authentication failure with interserver secret + /// - early exit without trying to send the exception to the client. + /// Because the server should not try to skip (parse, decompress) the remaining packets sent by the client, + /// as it will lead to additional work and unneeded exposure to unauthenticated connections. + + /// Note that the exception AUTHENTICATION_FAILED can be here in two cases: + /// 1. The authentication in receiveHello is skipped with "interserver secret", + /// postponed to receiving the query, and then failed. + /// 2. Receiving exception from a query using a table function to authenticate with another server. + /// In this case, the user is already authenticated with this server, + /// is_interserver_mode is false, and we can send the exception to the client normally. + + if (is_interserver_mode && e.code() == ErrorCodes::AUTHENTICATION_FAILED) throw; state.io.onException(); diff --git a/tests/queries/0_stateless/01119_session_log.sql b/tests/queries/0_stateless/01119_session_log.sql index 55f6228797a..8f6967b89ec 100644 --- a/tests/queries/0_stateless/01119_session_log.sql +++ b/tests/queries/0_stateless/01119_session_log.sql @@ -4,7 +4,7 @@ select * from remote('127.0.0.2', system, one, 'default', ''); select * from remote('127.0.0.2', system, one, 'default', 'wrong password'); -- { serverError AUTHENTICATION_FAILED } select * from remote('127.0.0.2', system, one, 'nonexistsnt_user_1119', ''); -- { serverError AUTHENTICATION_FAILED } set receive_timeout=1; -select * from remote('127.0.0.2', system, one, ' INTERSERVER SECRET ', ''); -- { serverError NO_REMOTE_SHARD_AVAILABLE } +select * from remote('127.0.0.2', system, one, ' INTERSERVER SECRET ', ''); -- { serverError AUTHENTICATION_FAILED } set receive_timeout=300; select * from remote('127.0.0.2', system, one, ' ', ''); -- { serverError AUTHENTICATION_FAILED } From 09bec3c754698f2c7e3cf37a9b02018c7adddc33 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 16 Nov 2023 00:16:02 +0100 Subject: [PATCH 075/192] Fix integration test --- .../test_distributed_inter_server_secret/test.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index 4276fcffbf9..6e3f1e6e416 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -304,26 +304,20 @@ def test_secure_insert_buffer_async(): def test_secure_disagree(): - with pytest.raises( - QueryRuntimeException, match=".*Interserver authentication failed.*" - ): + with pytest.raises(QueryRuntimeException): n1.query("SELECT * FROM dist_secure_disagree") def test_secure_disagree_insert(): n1.query("TRUNCATE TABLE data") n1.query("INSERT INTO dist_secure_disagree SELECT * FROM numbers(2)") - with pytest.raises( - QueryRuntimeException, match=".*Interserver authentication failed.*" - ): + with pytest.raises(QueryRuntimeException): n1.query( "SYSTEM FLUSH DISTRIBUTED ON CLUSTER secure_disagree dist_secure_disagree" ) - # check the the connection will be re-established + # check that the connection will be re-established # IOW that we will not get "Unknown BlockInfo field" - with pytest.raises( - QueryRuntimeException, match=".*Interserver authentication failed.*" - ): + with pytest.raises(QueryRuntimeException): assert int(n1.query("SELECT count() FROM dist_secure_disagree")) == 0 From db666bf2bc063eac1668258f890a6766dc3c4430 Mon Sep 17 00:00:00 2001 From: johnnymatthews <9611008+johnnymatthews@users.noreply.github.com> Date: Wed, 15 Nov 2023 21:47:35 -0400 Subject: [PATCH 076/192] Disables RU intro section. --- docs/ru/introduction/_category_.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/ru/introduction/_category_.yml b/docs/ru/introduction/_category_.yml index 539f7ab97ed..b3e58207c12 100644 --- a/docs/ru/introduction/_category_.yml +++ b/docs/ru/introduction/_category_.yml @@ -2,6 +2,3 @@ position: 1 label: 'Введение' collapsible: true collapsed: true -link: - type: generated-index - title: Введение From f505181b0d8c485ee02d3e62fc057c22d5d188cc Mon Sep 17 00:00:00 2001 From: johnnymatthews <9611008+johnnymatthews@users.noreply.github.com> Date: Wed, 15 Nov 2023 22:08:10 -0400 Subject: [PATCH 077/192] Adds basics index page to RU introduction. --- docs/ru/introduction/index.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/ru/introduction/index.md diff --git a/docs/ru/introduction/index.md b/docs/ru/introduction/index.md new file mode 100644 index 00000000000..74a6e4dd135 --- /dev/null +++ b/docs/ru/introduction/index.md @@ -0,0 +1,13 @@ +--- +slug: /ru/introduction/ +sidebar_label: "Введение" +sidebar_position: 8 +--- + +# Введение + +Đ’ этом разделе ŃодержитŃŃŹ информация Đľ том, как начать Ń€Đ°Đ±ĐľŃ‚Ń Ń ClickHouse. + +- [Отличительные возможноŃти ClickHouse](./distinctive-features.md) +- [ПроизводительноŃŃ‚ŃŚ](./performance.md) +- [ĐŃтория ClickHouse](./history.md) From ef17d972ab64c90ff72a7c6c2beba61b522d6fcf Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Thu, 16 Nov 2023 05:18:50 +0000 Subject: [PATCH 078/192] Fix SET query formatting --- src/Parsers/ASTSetQuery.cpp | 48 ++++++++++++++++++- .../02916_set_formatting.reference | 11 +++++ .../0_stateless/02916_set_formatting.sql | 13 +++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02916_set_formatting.reference create mode 100644 tests/queries/0_stateless/02916_set_formatting.sql diff --git a/src/Parsers/ASTSetQuery.cpp b/src/Parsers/ASTSetQuery.cpp index e2c60e8369d..78161b865ee 100644 --- a/src/Parsers/ASTSetQuery.cpp +++ b/src/Parsers/ASTSetQuery.cpp @@ -4,11 +4,57 @@ #include #include #include +#include namespace DB { +class FieldVisitorToSetting : public StaticVisitor +{ +public: + template + String operator() (const T & x) const + { + FieldVisitorToString visitor; + return visitor(x); + } + + String operator() (const Map & x) const + { + WriteBufferFromOwnString wb; + + wb << '{'; + + auto it = x.begin(); + while (it != x.end()) + { + if (it != x.begin()) + wb << ", "; + wb << applyVisitor(*this, *it); + ++it; + } + wb << '}'; + + return wb.str(); + } + + String operator() (const Tuple & x) const + { + WriteBufferFromOwnString wb; + + for (auto it = x.begin(); it != x.end(); ++it) + { + if (it != x.begin()) + wb << ":"; + wb << applyVisitor(*this, *it); + } + + return wb.str(); + } +}; + + void ASTSetQuery::updateTreeHashImpl(SipHash & hash_state, bool /*ignore_aliases*/) const { for (const auto & change : changes) @@ -38,7 +84,7 @@ void ASTSetQuery::formatImpl(const FormatSettings & format, FormatState &, Forma if (!format.show_secrets && change.value.tryGet(custom) && custom.isSecret()) format.ostr << " = " << custom.toString(false); else - format.ostr << " = " << applyVisitor(FieldVisitorToString(), change.value); + format.ostr << " = " << applyVisitor(FieldVisitorToSetting(), change.value); } for (const auto & setting_name : default_settings) diff --git a/tests/queries/0_stateless/02916_set_formatting.reference b/tests/queries/0_stateless/02916_set_formatting.reference new file mode 100644 index 00000000000..34ff52365f9 --- /dev/null +++ b/tests/queries/0_stateless/02916_set_formatting.reference @@ -0,0 +1,11 @@ +SET additional_table_filters = {\'kjsnckjn\':\'ksanmn\', \'dkm\':\'dd\'} +SELECT v FROM t1 SETTINGS additional_table_filters = {\'default.t1\':\'s\'} +Row 1: +────── +statement: CREATE VIEW default.v1 +( + `v` UInt64 +) AS +SELECT v +FROM default.t1 +SETTINGS additional_table_filters = {'default.t1':'s != \'s1%\''} diff --git a/tests/queries/0_stateless/02916_set_formatting.sql b/tests/queries/0_stateless/02916_set_formatting.sql new file mode 100644 index 00000000000..10b875293f1 --- /dev/null +++ b/tests/queries/0_stateless/02916_set_formatting.sql @@ -0,0 +1,13 @@ +SELECT formatQuerySingleLine('set additional_table_filters = {\'kjsnckjn\': \'ksanmn\', \'dkm\': \'dd\'}'); +SELECT formatQuerySingleLine('SELECT v FROM t1 SETTINGS additional_table_filters = {\'default.t1\': \'s\'}'); + +DROP TABLE IF EXISTS t1; +DROP VIEW IF EXISTS v1; + +CREATE TABLE t1 (v UInt64, s String) ENGINE=MergeTree() ORDER BY v; +CREATE VIEW v1 (v UInt64) AS SELECT v FROM t1 SETTINGS additional_table_filters = {'default.t1': 's != \'s1%\''}; + +SHOW CREATE TABLE v1 FORMAT Vertical; + +DROP VIEW v1; +DROP TABLE t1; From 1d7eecaeece1bb2cdadf8c446bd8631592ecfb08 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Thu, 16 Nov 2023 06:08:12 +0000 Subject: [PATCH 079/192] Fix failure to start due to table dependency in joinGet --- src/Databases/DDLLoadingDependencyVisitor.cpp | 16 +++++------ .../02916_joinget_dependency.reference | 1 + .../0_stateless/02916_joinget_dependency.sh | 27 +++++++++++++++++++ 3 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/02916_joinget_dependency.reference create mode 100755 tests/queries/0_stateless/02916_joinget_dependency.sh diff --git a/src/Databases/DDLLoadingDependencyVisitor.cpp b/src/Databases/DDLLoadingDependencyVisitor.cpp index fc362dd8578..77a40f674fd 100644 --- a/src/Databases/DDLLoadingDependencyVisitor.cpp +++ b/src/Databases/DDLLoadingDependencyVisitor.cpp @@ -144,22 +144,22 @@ void DDLLoadingDependencyVisitor::extractTableNameFromArgument(const ASTFunction const auto * arg = function.arguments->as()->children[arg_idx].get(); - if (const auto * dict_function = arg->as()) + if (const auto * function_arg = arg->as()) { - if (!functionIsDictGet(dict_function->name)) + if (!functionIsJoinGet(function_arg->name) && !functionIsDictGet(function_arg->name)) return; - /// Get the dictionary name from `dict*` function. - const auto * literal_arg = dict_function->arguments->as()->children[0].get(); - const auto * dictionary_name = literal_arg->as(); + /// Get the dictionary name from `dict*` function or the table name from 'joinGet' function. + const auto * literal_arg = function_arg->arguments->as()->children[0].get(); + const auto * name = literal_arg->as(); - if (!dictionary_name) + if (!name) return; - if (dictionary_name->value.getType() != Field::Types::String) + if (name->value.getType() != Field::Types::String) return; - auto maybe_qualified_name = QualifiedTableName::tryParseFromString(dictionary_name->value.get()); + auto maybe_qualified_name = QualifiedTableName::tryParseFromString(name->value.get()); if (!maybe_qualified_name) return; diff --git a/tests/queries/0_stateless/02916_joinget_dependency.reference b/tests/queries/0_stateless/02916_joinget_dependency.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02916_joinget_dependency.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02916_joinget_dependency.sh b/tests/queries/0_stateless/02916_joinget_dependency.sh new file mode 100755 index 00000000000..6477ae8c967 --- /dev/null +++ b/tests/queries/0_stateless/02916_joinget_dependency.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# We test the dependency on the DROP + +$CLICKHOUSE_CLIENT -nm -q " + DROP TABLE IF EXISTS Sub_distributed; + DROP TABLE IF EXISTS Sub; + DROP TABLE IF EXISTS Mapping; + + CREATE TABLE Mapping (Id UInt64, RegionId UInt64) ENGINE = Join(ANY,LEFT,Id); + INSERT INTO Mapping VALUES (1,1); + CREATE TABLE Sub (Id UInt64, PropertyId UInt64) ENGINE = MergeTree() PRIMARY KEY (Id) ORDER BY (Id); + CREATE TABLE Sub_distributed (Id UInt64, PropertyId UInt64)ENGINE = Distributed('test_shard_localhost', $CLICKHOUSE_DATABASE, Sub, joinGet('$CLICKHOUSE_DATABASE.Mapping','RegionId',PropertyId));" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE Mapping; +" 2>&1 | grep -cm1 "HAVE_DEPENDENT_OBJECTS" + +$CLICKHOUSE_CLIENT -nm -q " + DROP TABLE Sub_distributed; + DROP TABLE Sub; + DROP TABLE Mapping; +" \ No newline at end of file From 7623153d3841d58f282915e24f9d032ee6d84e94 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Thu, 16 Nov 2023 07:01:49 +0000 Subject: [PATCH 080/192] Fix flattening existing Nested columns during ADD COLUMN --- src/Storages/AlterCommands.cpp | 21 +++++++++++++++---- .../02916_addcolumn_nested.reference | 3 +++ .../0_stateless/02916_addcolumn_nested.sql | 17 +++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/02916_addcolumn_nested.reference create mode 100644 tests/queries/0_stateless/02916_addcolumn_nested.sql diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 98bfa3b3f57..7eeaa2d4594 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -395,11 +395,24 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) column.ttl = ttl; - metadata.columns.add(column, after_column, first); - - /// Slow, because each time a list is copied if (context->getSettingsRef().flatten_nested) - metadata.columns.flattenNested(); + { + StorageInMemoryMetadata temporary_metadata; + temporary_metadata.columns.add(column, /*after_column*/ "", /*first*/ true); + temporary_metadata.columns.flattenNested(); + + const auto transformed_columns = temporary_metadata.columns.getAll(); + + for (auto it = transformed_columns.rbegin(); it != transformed_columns.rend(); it++) + { + const auto & transformed_column = temporary_metadata.columns.get(it->name); + metadata.columns.add(transformed_column, after_column, first); + } + } + else + { + metadata.columns.add(column, after_column, first); + } } else if (type == DROP_COLUMN) { diff --git a/tests/queries/0_stateless/02916_addcolumn_nested.reference b/tests/queries/0_stateless/02916_addcolumn_nested.reference new file mode 100644 index 00000000000..869d4336c62 --- /dev/null +++ b/tests/queries/0_stateless/02916_addcolumn_nested.reference @@ -0,0 +1,3 @@ +CREATE TABLE default.nested_table\n(\n `id` UInt64,\n `first` Nested(a Int8, b String)\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.nested_table\n(\n `id` UInt64,\n `second.c` Array(Int8),\n `second.d` Array(String),\n `first` Nested(a Int8, b String)\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.nested_table\n(\n `third` Nested(e Int8, f String),\n `id` UInt64,\n `second.c` Array(Int8),\n `second.d` Array(String),\n `first` Nested(a Int8, b String)\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02916_addcolumn_nested.sql b/tests/queries/0_stateless/02916_addcolumn_nested.sql new file mode 100644 index 00000000000..b23854824b5 --- /dev/null +++ b/tests/queries/0_stateless/02916_addcolumn_nested.sql @@ -0,0 +1,17 @@ +SET flatten_nested = 0; + +DROP TABLE IF EXISTS nested_table; +CREATE TABLE nested_table (id UInt64, first Nested(a Int8, b String)) ENGINE = MergeTree() ORDER BY id; +SHOW CREATE nested_table; + +SET flatten_nested = 1; + +ALTER TABLE nested_table ADD COLUMN second Nested(c Int8, d String) AFTER id; +SHOW CREATE nested_table; + +SET flatten_nested = 0; + +ALTER TABLE nested_table ADD COLUMN third Nested(e Int8, f String) FIRST; +SHOW CREATE nested_table; + +DROP TABLE nested_table; From cffc6611e000faa19cac35dccfc093d77ae7e315 Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Thu, 16 Nov 2023 07:05:41 +0000 Subject: [PATCH 081/192] Empty commit. From 69f214cdbcc85808d77893112f7c560285747a09 Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Thu, 16 Nov 2023 08:04:57 +0000 Subject: [PATCH 082/192] fix comments. Signed-off-by: Jianfei Hu --- src/IO/S3/Credentials.cpp | 32 +++++++++++++++++++------------- src/IO/S3/Credentials.h | 1 - 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index e25f4551723..73763853713 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -1,4 +1,15 @@ #include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNSUPPORTED_METHOD; +} + +} #if USE_AWS_S3 @@ -16,11 +27,9 @@ # include # include -# include # include # include -# include # include # include @@ -42,7 +51,6 @@ namespace ErrorCodes { extern const int AWS_ERROR; extern const int GCP_ERROR; - extern const int UNSUPPORTED_METHOD; } namespace S3 @@ -280,20 +288,20 @@ String getRunningAvailabilityZone() LOG_INFO(&Poco::Logger::get("Application"), "Trying to detect the availability zone."); try { - auto aws_az = AWSEC2MetadataClient::getAvailabilityZoneOrException(); - return aws_az; + return AWSEC2MetadataClient::getAvailabilityZoneOrException(); } - catch (const std::exception & aws_ex) + catch (...) { + auto aws_ex_msg = getExceptionMessage(std::current_exception(), false); try { - auto gcp_zone = getGCPAvailabilityZoneOrException(); - return gcp_zone; + return getGCPAvailabilityZoneOrException(); } - catch (const std::exception & gcp_ex) + catch (...) { + auto gcp_ex_msg = getExceptionMessage(std::current_exception(), false); throw DB::Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Failed to find the availability zone, tried AWS and GCP. AWS Error: {}\nGCP Error: {}", aws_ex.what(), gcp_ex.what()); + "Failed to find the availability zone, tried AWS and GCP. AWS Error: {}\nGCP Error: {}", aws_ex_msg, gcp_ex_msg); } } } @@ -784,8 +792,6 @@ S3CredentialsProviderChain::S3CredentialsProviderChain( #else -# include - namespace DB { @@ -794,7 +800,7 @@ namespace S3 std::string getRunningAvailabilityZone() { - throw std::runtime_error("Does not support availability zone detection for non-cloud environment"); + throw DB::Exception(ErrorCodes::UNSUPPORTED_METHOD, "Does not support availability zone detection for non-cloud environment"); } } diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index d8d103a847a..ad73de23486 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -4,7 +4,6 @@ #if USE_AWS_S3 -# include # include # include From 3b226a10feac18ae16836f5e627d48b413c12cc1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 16 Nov 2023 09:50:34 +0100 Subject: [PATCH 083/192] Fix test --- .../0_stateless/01555_system_distribution_queue_mask.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql index 7ade1d24c59..3a90765226a 100644 --- a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql +++ b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql @@ -17,7 +17,7 @@ system stop distributed sends dist_01555; insert into dist_01555 values (1)(2); -- since test_cluster_with_incorrect_pw contains incorrect password ignore error -system flush distributed dist_01555; -- { clientError ATTEMPT_TO_READ_AFTER_EOF } +system flush distributed dist_01555; -- { serverError 516 } select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1'), extract(last_exception, 'AUTHENTICATION_FAILED'), dateDiff('s', last_exception_time, now()) < 3600 from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV; drop table dist_01555; @@ -30,7 +30,7 @@ create table dist_01555 (key Int) Engine=Distributed(test_cluster_with_incorrect insert into dist_01555 values (1)(2); -- since test_cluster_with_incorrect_pw contains incorrect password ignore error -system flush distributed dist_01555; -- { clientError ATTEMPT_TO_READ_AFTER_EOF } +system flush distributed dist_01555; -- { serverError 516 } select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1'), extract(last_exception, 'AUTHENTICATION_FAILED'), dateDiff('s', last_exception_time, now()) < 3600 from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV; drop table dist_01555; From 61948e31714f1aa3fa8143a569a72403c5c70408 Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Thu, 16 Nov 2023 11:12:45 +0100 Subject: [PATCH 084/192] Update src/Core/Settings.h Co-authored-by: Nikita Taranov --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 34547aded9c..76016bc70fb 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -94,7 +94,7 @@ class IColumn; M(UInt64, s3_max_put_rps, 0, "Limit on S3 PUT request per second rate before throttling. Zero means unlimited.", 0) \ M(UInt64, s3_max_put_burst, 0, "Max number of requests that can be issued simultaneously before hitting request per second limit. By default (0) equals to `s3_max_put_rps`", 0) \ M(UInt64, s3_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ - M(Bool, s3_use_adaptive_timeouts, true, "When aggressive timeouts are enabled first two attempts are made with low receive and send timeout", 0) \ + M(Bool, s3_use_adaptive_timeouts, true, "When adaptive timeouts are enabled first two attempts are made with low receive and send timeout", 0) \ M(UInt64, azure_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ M(Bool, s3_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables.", 0) \ M(Bool, azure_truncate_on_insert, false, "Enables or disables truncate before insert in azure engine tables.", 0) \ From 649d734409fab1eb602026cd1e39b601a0b1673c Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Nov 2023 10:13:07 +0000 Subject: [PATCH 085/192] Bump gRPC to v1.56.3 --- contrib/grpc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grpc b/contrib/grpc index a08fe1a3407..bc110c3dc91 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit a08fe1a34075c93bb2d606dd608b9a3953288b81 +Subproject commit bc110c3dc91b77d1e54957871df54fd39f2a49d1 From a49db81b9f10aacee9528a02bea0f7b57dc532a7 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Nov 2023 10:15:11 +0000 Subject: [PATCH 086/192] Bump gRPC to v1.57.1 --- contrib/grpc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grpc b/contrib/grpc index bc110c3dc91..fd802577cc0 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit bc110c3dc91b77d1e54957871df54fd39f2a49d1 +Subproject commit fd802577cc06226428c99297d5be3a24f5e3ab96 From 1ba408eb0b2f37e2978376920920c7863a0325b8 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Nov 2023 10:38:03 +0000 Subject: [PATCH 087/192] Bump gRPC to v1.58.2 --- contrib/grpc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grpc b/contrib/grpc index fd802577cc0..2e45a02f2b2 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit fd802577cc06226428c99297d5be3a24f5e3ab96 +Subproject commit 2e45a02f2b24e3cc455d1793a469e1dbba894f94 From a250c2bb08eda91f50eebd5bffd5086ef3251b9f Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Nov 2023 10:42:41 +0000 Subject: [PATCH 088/192] Bump gRPC to v1.59.2 --- contrib/grpc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grpc b/contrib/grpc index 2e45a02f2b2..740e3dfd973 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit 2e45a02f2b24e3cc455d1793a469e1dbba894f94 +Subproject commit 740e3dfd97301a52ad8165b65285bcc149d9e817 From ac7fd357e4f75d9b16023450d8fe5df6f3e68cfa Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Nov 2023 10:47:10 +0000 Subject: [PATCH 089/192] Bump protobuf to v23.2 --- contrib/google-protobuf | 2 +- contrib/google-protobuf-cmake/CMakeLists.txt | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/contrib/google-protobuf b/contrib/google-protobuf index 089b89c8d41..5b179151990 160000 --- a/contrib/google-protobuf +++ b/contrib/google-protobuf @@ -1 +1 @@ -Subproject commit 089b89c8d4140f0d49fe4222b047a8ea814bc752 +Subproject commit 5b1791519907360781cfe3bebe1c79e5b1b0bcba diff --git a/contrib/google-protobuf-cmake/CMakeLists.txt b/contrib/google-protobuf-cmake/CMakeLists.txt index f6955a3d8ce..3b53ac822da 100644 --- a/contrib/google-protobuf-cmake/CMakeLists.txt +++ b/contrib/google-protobuf-cmake/CMakeLists.txt @@ -174,6 +174,8 @@ set(libprotobuf_files ${protobuf_source_dir}/src/google/protobuf/message.cc ${protobuf_source_dir}/src/google/protobuf/message_lite.cc ${protobuf_source_dir}/src/google/protobuf/parse_context.cc + ${protobuf_source_dir}/src/google/protobuf/port.cc + ${protobuf_source_dir}/src/google/protobuf/reflection_mode.cc ${protobuf_source_dir}/src/google/protobuf/reflection_ops.cc ${protobuf_source_dir}/src/google/protobuf/repeated_field.cc ${protobuf_source_dir}/src/google/protobuf/repeated_ptr_field.cc @@ -213,6 +215,7 @@ set(libprotoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/enum.cc ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/extension.cc ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field_generators/cord_field.cc ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field_generators/map_field.cc ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field_generators/message_field.cc @@ -299,6 +302,13 @@ set(libprotoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/python/pyi_generator.cc ${protobuf_source_dir}/src/google/protobuf/compiler/retention.cc ${protobuf_source_dir}/src/google/protobuf/compiler/ruby/ruby_generator.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/accessors/accessors.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/accessors/singular_bytes.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/context.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/generator.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/message.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/naming.cc ${protobuf_source_dir}/src/google/protobuf/compiler/subprocess.cc ${protobuf_source_dir}/src/google/protobuf/compiler/zip_writer.cc ) From ebd42187ad12ce2be24833820f59bb3d81def382 Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Thu, 16 Nov 2023 12:29:15 +0100 Subject: [PATCH 090/192] Update tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml Co-authored-by: Nikita Taranov --- .../configs/config.d/storage_conf.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml index f51b854de75..98c6f551be6 100644 --- a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml @@ -34,7 +34,7 @@ true 1 - 0 + 0 1 20000 From 7d37c0e07073b2a1909e80c4aea45fdc4a35be75 Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Thu, 16 Nov 2023 12:29:21 +0100 Subject: [PATCH 091/192] Update tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml Co-authored-by: Nikita Taranov --- .../configs/config.d/storage_conf.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml index 98c6f551be6..6303e9273fc 100644 --- a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml @@ -11,7 +11,7 @@ true 0 - 0 + 0 20000 From 4a1e207e7a5b02da2b2f6ea46edecd9fe6a9185c Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 16 Nov 2023 12:31:00 +0100 Subject: [PATCH 092/192] review notes --- base/poco/Net/src/HTTPSession.cpp | 34 +++++++++++++++++++++---------- src/IO/S3/PocoHTTPClient.cpp | 20 ++++++++++++++---- src/IO/S3/PocoHTTPClient.h | 6 +++--- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index 9ebbd7d04cd..d303a4c654b 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -93,22 +93,34 @@ void HTTPSession::setTimeout(const Poco::Timespan& timeout) void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco::Timespan& sendTimeout, const Poco::Timespan& receiveTimeout) { - _connectionTimeout = connectionTimeout; - - if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) + try { - _sendTimeout = sendTimeout; + _connectionTimeout = connectionTimeout; - if (connected()) - _socket.setSendTimeout(_sendTimeout); + if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) { + _sendTimeout = sendTimeout; + + if (connected()) + _socket.setSendTimeout(_sendTimeout); + } + + if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) { + _receiveTimeout = receiveTimeout; + + if (connected()) + _socket.setReceiveTimeout(_receiveTimeout); + } } - - if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) + catch (NetException &) { - _receiveTimeout = receiveTimeout; - if (connected()) - _socket.setReceiveTimeout(_receiveTimeout); +#ifndef NDEBUG + // mute exceptions in release + // just in case when changing settings on socket is not allowed + // however it should be OK for timeouts +#else + throw; +#endif } } diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index f681362e607..4a1b6def133 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -407,17 +407,29 @@ void PocoHTTPClient::makeRequestInternalImpl( /// This can lead to request signature difference on S3 side. if constexpr (pooled) session = makePooledHTTPSession( - target_uri, getTimeouts(method, first_attempt), http_connection_pool_size, wait_on_pool_size_limit, proxy_configuration); + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true), + http_connection_pool_size, + wait_on_pool_size_limit, + proxy_configuration); else - session = makeHTTPSession(target_uri, getTimeouts(method, first_attempt), proxy_configuration); + session = makeHTTPSession( + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true), + proxy_configuration); } else { if constexpr (pooled) session = makePooledHTTPSession( - target_uri, getTimeouts(method, first_attempt), http_connection_pool_size, wait_on_pool_size_limit); + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true), + http_connection_pool_size, + wait_on_pool_size_limit); else - session = makeHTTPSession(target_uri, getTimeouts(method, first_attempt)); + session = makeHTTPSession( + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true)); } /// In case of error this address will be written to logs diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 14c4fec5dd7..5178d75e7b6 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -55,7 +55,7 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration size_t http_connection_pool_size = 0; /// See PoolBase::BehaviourOnLimit bool wait_on_pool_size_limit = true; - bool s3_use_adaptive_timeouts = false; + bool s3_use_adaptive_timeouts = true; std::function error_report; @@ -171,7 +171,7 @@ private: Aws::Utils::RateLimits::RateLimiterInterface * readLimiter, Aws::Utils::RateLimits::RateLimiterInterface * writeLimiter) const; - ConnectionTimeouts getTimeouts(const String & method, bool first_attempt, bool first_byte = true) const; + ConnectionTimeouts getTimeouts(const String & method, bool first_attempt, bool first_byte) const; protected: static S3MetricKind getMetricKind(const Aws::Http::HttpRequest & request); @@ -182,7 +182,7 @@ protected: ConnectionTimeouts timeouts; const RemoteHostFilter & remote_host_filter; unsigned int s3_max_redirects; - bool s3_use_adaptive_timeouts = false; + bool s3_use_adaptive_timeouts = true; bool enable_s3_requests_logging; bool for_disk_s3; From e53e723be81d9ba76595afa7bbe67c1bf8764c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 16 Nov 2023 12:32:42 +0100 Subject: [PATCH 093/192] Apply same improvement to initializeAggregation --- src/Functions/initializeAggregation.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Functions/initializeAggregation.cpp b/src/Functions/initializeAggregation.cpp index 83df28808a1..eeeb03aeb30 100644 --- a/src/Functions/initializeAggregation.cpp +++ b/src/Functions/initializeAggregation.cpp @@ -141,10 +141,19 @@ ColumnPtr FunctionInitializeAggregation::executeImpl(const ColumnsWithTypeAndNam that->addBatch(0, input_rows_count, places.data(), 0, aggregate_arguments, arena.get()); } - for (size_t i = 0; i < input_rows_count; ++i) + if (agg_func.isState()) + { /// We should use insertMergeResultInto to insert result into ColumnAggregateFunction /// correctly if result contains AggregateFunction's states - agg_func.insertMergeResultInto(places[i], res_col, arena.get()); + for (size_t i = 0; i < input_rows_count; ++i) + agg_func.insertMergeResultInto(places[i], res_col, arena.get()); + } + else + { + for (size_t i = 0; i < input_rows_count; ++i) + agg_func.insertResultInto(places[i], res_col, arena.get()); + } + return result_holder; } From a0934253deff176da4c2eb21844cf31f3f7a3c61 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 16 Nov 2023 11:39:36 +0000 Subject: [PATCH 094/192] Bump protobuf to v24.4 --- contrib/google-protobuf | 2 +- contrib/google-protobuf-cmake/CMakeLists.txt | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/contrib/google-protobuf b/contrib/google-protobuf index 5b179151990..0862007f6ca 160000 --- a/contrib/google-protobuf +++ b/contrib/google-protobuf @@ -1 +1 @@ -Subproject commit 5b1791519907360781cfe3bebe1c79e5b1b0bcba +Subproject commit 0862007f6ca1f5723c58f10f0ca34f3f25a63b2e diff --git a/contrib/google-protobuf-cmake/CMakeLists.txt b/contrib/google-protobuf-cmake/CMakeLists.txt index 3b53ac822da..1ed4133270b 100644 --- a/contrib/google-protobuf-cmake/CMakeLists.txt +++ b/contrib/google-protobuf-cmake/CMakeLists.txt @@ -82,7 +82,6 @@ set(libprotobuf_lite_files ${protobuf_source_dir}/src/google/protobuf/any_lite.cc ${protobuf_source_dir}/src/google/protobuf/arena.cc ${protobuf_source_dir}/src/google/protobuf/arena_align.cc - ${protobuf_source_dir}/src/google/protobuf/arena_config.cc ${protobuf_source_dir}/src/google/protobuf/arenastring.cc ${protobuf_source_dir}/src/google/protobuf/arenaz_sampler.cc ${protobuf_source_dir}/src/google/protobuf/extension_set.cc @@ -131,17 +130,18 @@ set(libprotobuf_files ${protobuf_source_dir}/src/google/protobuf/any_lite.cc ${protobuf_source_dir}/src/google/protobuf/arena.cc ${protobuf_source_dir}/src/google/protobuf/arena_align.cc - ${protobuf_source_dir}/src/google/protobuf/arena_config.cc ${protobuf_source_dir}/src/google/protobuf/arenastring.cc ${protobuf_source_dir}/src/google/protobuf/arenaz_sampler.cc ${protobuf_source_dir}/src/google/protobuf/compiler/importer.cc ${protobuf_source_dir}/src/google/protobuf/compiler/parser.cc + ${protobuf_source_dir}/src/google/protobuf/cpp_features.pb.cc ${protobuf_source_dir}/src/google/protobuf/descriptor.cc ${protobuf_source_dir}/src/google/protobuf/descriptor.pb.cc ${protobuf_source_dir}/src/google/protobuf/descriptor_database.cc ${protobuf_source_dir}/src/google/protobuf/dynamic_message.cc ${protobuf_source_dir}/src/google/protobuf/extension_set.cc ${protobuf_source_dir}/src/google/protobuf/extension_set_heavy.cc + ${protobuf_source_dir}/src/google/protobuf/feature_resolver.cc ${protobuf_source_dir}/src/google/protobuf/generated_enum_util.cc ${protobuf_source_dir}/src/google/protobuf/generated_message_bases.cc ${protobuf_source_dir}/src/google/protobuf/generated_message_reflection.cc @@ -175,6 +175,7 @@ set(libprotobuf_files ${protobuf_source_dir}/src/google/protobuf/message_lite.cc ${protobuf_source_dir}/src/google/protobuf/parse_context.cc ${protobuf_source_dir}/src/google/protobuf/port.cc + ${protobuf_source_dir}/src/google/protobuf/raw_ptr.cc ${protobuf_source_dir}/src/google/protobuf/reflection_mode.cc ${protobuf_source_dir}/src/google/protobuf/reflection_ops.cc ${protobuf_source_dir}/src/google/protobuf/repeated_field.cc @@ -210,6 +211,7 @@ add_library(protobuf::libprotobuf ALIAS _libprotobuf) set(libprotoc_files + ${protobuf_source_dir}/src/google/protobuf/compiler/allowlists/editions.cc ${protobuf_source_dir}/src/google/protobuf/compiler/code_generator.cc ${protobuf_source_dir}/src/google/protobuf/compiler/command_line_interface.cc ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/enum.cc From a0840d36afbddad03c06c54fca17632917bd4d7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 16 Nov 2023 12:48:27 +0100 Subject: [PATCH 095/192] Apply the same to arrayReduce --- src/Functions/array/arrayReduce.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Functions/array/arrayReduce.cpp b/src/Functions/array/arrayReduce.cpp index ea087f4f9a8..46777ceb05c 100644 --- a/src/Functions/array/arrayReduce.cpp +++ b/src/Functions/array/arrayReduce.cpp @@ -182,10 +182,19 @@ ColumnPtr FunctionArrayReduce::executeImpl(const ColumnsWithTypeAndName & argume that->addBatchArray(0, input_rows_count, places.data(), 0, aggregate_arguments, offsets->data(), arena.get()); } - for (size_t i = 0; i < input_rows_count; ++i) - /// We should use insertMergeResultInto to insert result into ColumnAggregateFunction - /// correctly if result contains AggregateFunction's states - agg_func.insertMergeResultInto(places[i], res_col, arena.get()); + if (agg_func.isState()) + { + for (size_t i = 0; i < input_rows_count; ++i) + /// We should use insertMergeResultInto to insert result into ColumnAggregateFunction + /// correctly if result contains AggregateFunction's states + agg_func.insertMergeResultInto(places[i], res_col, arena.get()); + } + else + { + for (size_t i = 0; i < input_rows_count; ++i) + agg_func.insertResultInto(places[i], res_col, arena.get()); + } + return result_holder; } From e8c14562ab512d5704f2ec05f8e053346c6737fd Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 16 Nov 2023 11:50:25 +0000 Subject: [PATCH 096/192] Bump absl to HEAD --- contrib/abseil-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/abseil-cpp b/contrib/abseil-cpp index 5655528c418..3bd86026c93 160000 --- a/contrib/abseil-cpp +++ b/contrib/abseil-cpp @@ -1 +1 @@ -Subproject commit 5655528c41830f733160de4fb0b99073841bae9e +Subproject commit 3bd86026c93da5a40006fd53403dff9d5f5e30e3 From 504aeb987b220372f4a06a6e1dc8400d80eb3201 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 16 Nov 2023 12:13:13 +0000 Subject: [PATCH 097/192] Better messages --- src/Compression/CompressionCodecFPC.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Compression/CompressionCodecFPC.cpp b/src/Compression/CompressionCodecFPC.cpp index 5763d929f6c..2a6dfda7b5c 100644 --- a/src/Compression/CompressionCodecFPC.cpp +++ b/src/Compression/CompressionCodecFPC.cpp @@ -416,7 +416,7 @@ private: std::to_integer(bytes.front()) & MAX_ZERO_BYTE_COUNT); if (zero_byte_count1 > VALUE_SIZE || zero_byte_count2 > VALUE_SIZE) [[unlikely]] - throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Invalid compressed data"); + throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Invalid zero byte count(s): {} and {}", zero_byte_count1, zero_byte_count2); size_t tail_size1 = VALUE_SIZE - zero_byte_count1; size_t tail_size2 = VALUE_SIZE - zero_byte_count2; @@ -424,7 +424,7 @@ private: size_t expected_size = 0; if (__builtin_add_overflow(tail_size1, tail_size2, &expected_size) || __builtin_add_overflow(expected_size, 1, &expected_size)) [[unlikely]] - throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Invalid compressed data"); + throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Overflow occurred while calculating expected size"); if (bytes.size() < expected_size) [[unlikely]] throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Unexpected end of encoded sequence"); From 4d16c096a1c7464573c5101fd9068b9d451492dc Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 16 Nov 2023 12:09:13 +0000 Subject: [PATCH 098/192] Use ports from cluster --- tests/integration/helpers/cluster.py | 2 ++ tests/integration/test_storage_rabbitmq/test.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 729b30ba934..cbc511628f0 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -583,6 +583,7 @@ class ClickHouseCluster: self.rabbitmq_host = "rabbitmq1" self.rabbitmq_ip = None self.rabbitmq_port = 5672 + self.rabbitmq_secure_port = 5671 self.rabbitmq_dir = p.abspath(p.join(self.instances_dir, "rabbitmq")) self.rabbitmq_cookie_file = os.path.join(self.rabbitmq_dir, "erlang.cookie") self.rabbitmq_logs_dir = os.path.join(self.rabbitmq_dir, "logs") @@ -1316,6 +1317,7 @@ class ClickHouseCluster: self.with_rabbitmq = True env_variables["RABBITMQ_HOST"] = self.rabbitmq_host env_variables["RABBITMQ_PORT"] = str(self.rabbitmq_port) + env_variables["RABBITMQ_SECURE_PORT"] = str(self.rabbitmq_secure_port) env_variables["RABBITMQ_LOGS"] = self.rabbitmq_logs_dir env_variables["RABBITMQ_LOGS_FS"] = "bind" env_variables["RABBITMQ_COOKIE_FILE"] = self.rabbitmq_cookie_file diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index adb7f59769a..021cdf54af9 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -110,9 +110,9 @@ def rabbitmq_setup_teardown(): ], ) def test_rabbitmq_select(rabbitmq_cluster, secure): - port = 5672 + port = cluster.rabbitmq_port if secure: - port = 5671 + port = cluster.rabbitmq_secure_port instance.query( """ From bdf038191ac85bb9f38524a96480bbf6704d3a24 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 16 Nov 2023 13:05:37 +0000 Subject: [PATCH 099/192] better test_keeper_broken_logs --- tests/integration/test_keeper_broken_logs/test.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_keeper_broken_logs/test.py b/tests/integration/test_keeper_broken_logs/test.py index e283d946174..49b8d985ee8 100644 --- a/tests/integration/test_keeper_broken_logs/test.py +++ b/tests/integration/test_keeper_broken_logs/test.py @@ -1,13 +1,7 @@ import pytest from helpers.cluster import ClickHouseCluster import helpers.keeper_utils as keeper_utils -import random -import string -import os import time -from multiprocessing.dummy import Pool -from helpers.network import PartitionManager -from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( @@ -82,6 +76,13 @@ def test_single_node_broken_log(started_cluster): node1_conn.close() node1.stop_clickhouse() + + # wait until cluster stabilizes with a new leader + while not keeper_utils.is_leader( + started_cluster, node2 + ) and not keeper_utils.is_leader(started_cluster, node3): + time.sleep(1) + node1.exec_in_container( [ "truncate", From bb68321fc153be034fb0e2234b59bf6319cdd281 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 16 Nov 2023 09:02:11 +0000 Subject: [PATCH 100/192] More stable test_keeper_reconfig_replace_leader --- tests/integration/helpers/keeper_utils.py | 95 +++++++++++-------- .../test.py | 7 ++ 2 files changed, 65 insertions(+), 37 deletions(-) diff --git a/tests/integration/helpers/keeper_utils.py b/tests/integration/helpers/keeper_utils.py index 83d0f2969b7..1ca17e923e4 100644 --- a/tests/integration/helpers/keeper_utils.py +++ b/tests/integration/helpers/keeper_utils.py @@ -37,39 +37,59 @@ class KeeperException(Exception): class KeeperClient(object): SEPARATOR = b"\a\a\a\a\n" - def __init__(self, bin_path: str, host: str, port: int): + def __init__(self, bin_path: str, host: str, port: int, connection_tries=30): self.bin_path = bin_path self.host = host self.port = port - self.proc = subprocess.Popen( - [ - bin_path, - "keeper-client", - "--host", - host, - "--port", - str(port), - "--log-level", - "error", - "--tests-mode", - "--no-confirmation", - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) + retry_count = 0 - self.poller = select.epoll() - self.poller.register(self.proc.stdout) - self.poller.register(self.proc.stderr) + while True: + try: + self.proc = subprocess.Popen( + [ + bin_path, + "keeper-client", + "--host", + host, + "--port", + str(port), + "--log-level", + "error", + "--tests-mode", + "--no-confirmation", + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) - self._fd_nums = { - self.proc.stdout.fileno(): self.proc.stdout, - self.proc.stderr.fileno(): self.proc.stderr, - } + self.poller = select.epoll() + self.poller.register(self.proc.stdout) + self.poller.register(self.proc.stderr) - self.stopped = False + self._fd_nums = { + self.proc.stdout.fileno(): self.proc.stdout, + self.proc.stderr.fileno(): self.proc.stderr, + } + + self.stopped = False + + self.get("/keeper", 60.0) + break + except Exception as e: + retry_count += 1 + if ( + "All connection tries failed while connecting to ZooKeeper" + in str(e) + and retry_count < connection_tries + ): + print( + f"Got exception while connecting to Keeper: {e}\nWill reconnect, reconnect count = {retry_count}" + ) + time.sleep(1) + else: + raise def execute_query(self, query: str, timeout: float = 60.0) -> str: output = io.BytesIO() @@ -94,7 +114,7 @@ class KeeperClient(object): output.write(chunk) elif file == self.proc.stderr: - assert self.proc.stdout.readline() == self.SEPARATOR + self.proc.stdout.readline() raise KeeperException(self.proc.stderr.readline().strip().decode()) else: @@ -221,13 +241,12 @@ NOT_SERVING_REQUESTS_ERROR_MSG = "This instance is not currently serving request def wait_until_connected(cluster, node, port=9181, timeout=30.0): - elapsed = 0.0 + start = time.time() while send_4lw_cmd(cluster, node, "mntr", port) == NOT_SERVING_REQUESTS_ERROR_MSG: time.sleep(0.1) - elapsed += 0.1 - if elapsed >= timeout: + if time.time() - start > timeout: raise Exception( f"{timeout}s timeout while waiting for {node.name} to start serving requests" ) @@ -280,14 +299,16 @@ def wait_configs_equal(left_config: str, right_zk: KeeperClient, timeout: float Check whether get /keeper/config result in left_config is equal to get /keeper/config on right_zk ZK connection. """ - elapsed: float = 0.0 - while sorted(left_config.split("\n")) != sorted( - get_config_str(right_zk).split("\n") - ): + start = time.time() + left_config = sorted(left_config.split("\n")) + while True: + right_config = sorted(get_config_str(right_zk).split("\n")) + if left_config == right_config: + return + time.sleep(1) - elapsed += 1 - if elapsed >= timeout: + if time.time() - start > timeout: raise Exception( f"timeout while checking nodes configs to get equal. " - f"Left: {left_config}, right: {get_config_str(right_zk)}" + f"Left: {left_config}, right: {right_config}" ) diff --git a/tests/integration/test_keeper_reconfig_replace_leader/test.py b/tests/integration/test_keeper_reconfig_replace_leader/test.py index 4cdd48fcf7c..8e621eef279 100644 --- a/tests/integration/test_keeper_reconfig_replace_leader/test.py +++ b/tests/integration/test_keeper_reconfig_replace_leader/test.py @@ -3,6 +3,7 @@ import pytest from helpers.cluster import ClickHouseCluster, ClickHouseInstance from os.path import join, dirname, realpath +import time import helpers.keeper_utils as ku import typing as tp @@ -83,6 +84,12 @@ def test_reconfig_replace_leader(started_cluster): assert "node3" in config assert "node4" not in config + # wait until cluster stabilizes with a new leader + while not ku.is_leader(started_cluster, node2) and not ku.is_leader( + started_cluster, node3 + ): + time.sleep(1) + # additional 20s wait before removing leader ku.wait_configs_equal(config, zk2, timeout=50) From 85d363fb285b0b278e8cfc413c98a481d90c3476 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Thu, 16 Nov 2023 14:58:52 +0100 Subject: [PATCH 101/192] Update tests --- .../0_stateless/00727_concat.reference | 6 +++++ tests/queries/0_stateless/00727_concat.sql | 23 ++++++++++++++----- .../0_stateless/02233_interpolate_1.sql | 2 +- .../02389_analyzer_nested_lambda.reference | 2 +- .../02389_analyzer_nested_lambda.sql | 2 +- .../02521_analyzer_array_join_crash.reference | 2 +- .../02521_analyzer_array_join_crash.sql | 2 +- 7 files changed, 28 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/00727_concat.reference b/tests/queries/0_stateless/00727_concat.reference index 9b6a8b3857b..7c48ba97c2b 100644 --- a/tests/queries/0_stateless/00727_concat.reference +++ b/tests/queries/0_stateless/00727_concat.reference @@ -25,6 +25,7 @@ With bar With foo With bar With 42 +With 42 With fae310ca-d52a-4923-9e9b-02bf67f4b009 With 2023-11-14 With 2123-11-14 @@ -41,6 +42,11 @@ With (42,43) With [(0,0),(10,0),(10,10),(0,10)] With [[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]] With [[[(0,0),(10,0),(10,10),(0,10)]],[[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]]] +-- SimpleAggregateFunction +With 42 +With 4 +-- Nested +With [(\'foo\',\'qaz\'),(\'bar\',\'qux\')] -- NULL arguments \N \N diff --git a/tests/queries/0_stateless/00727_concat.sql b/tests/queries/0_stateless/00727_concat.sql index ba76ff53884..7d901514aea 100644 --- a/tests/queries/0_stateless/00727_concat.sql +++ b/tests/queries/0_stateless/00727_concat.sql @@ -1,8 +1,6 @@ -- Tags: no-fasttest -- no-fasttest: json type needs rapidjson library, geo types need s2 geometry --- not tested here: (Simple)AggregateFunction, Nested - SET allow_experimental_object_type = 1; SET allow_suspicious_low_cardinality_types=1; @@ -33,11 +31,12 @@ SELECT concat('With ', materialize('bar' :: LowCardinality(FixedString(3)))); SELECT concat('With ', materialize('foo' :: LowCardinality(Nullable(String)))); SELECT concat('With ', materialize('bar' :: LowCardinality(Nullable(FixedString(3))))); SELECT concat('With ', materialize(42 :: LowCardinality(Nullable(UInt32)))); +SELECT concat('With ', materialize(42 :: LowCardinality(UInt32))); SELECT concat('With ', materialize('fae310ca-d52a-4923-9e9b-02bf67f4b009' :: UUID)); SELECT concat('With ', materialize('2023-11-14' :: Date)); SELECT concat('With ', materialize('2123-11-14' :: Date32)); -SELECT concat('With ', materialize('2023-11-14 05:50:12' :: DateTime)); -SELECT concat('With ', materialize('2023-11-14 05:50:12.123' :: DateTime64(3))); +SELECT concat('With ', materialize('2023-11-14 05:50:12' :: DateTime('Europe/Amsterdam'))); +SELECT concat('With ', materialize('2023-11-14 05:50:12.123' :: DateTime64(3, 'Europe/Amsterdam'))); SELECT concat('With ', materialize('hallo' :: Enum('hallo' = 1))); SELECT concat('With ', materialize(['foo', 'bar'] :: Array(String))); SELECT concat('With ', materialize('{"foo": "bar"}' :: JSON)); @@ -50,14 +49,23 @@ SELECT concat('With ', materialize([(0,0),(10,0),(10,10),(0,10)] :: Ring)); SELECT concat('With ', materialize([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]] :: Polygon)); SELECT concat('With ', materialize([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]] :: MultiPolygon)); +SELECT '-- SimpleAggregateFunction'; +CREATE OR REPLACE TABLE concat_saf_test(x SimpleAggregateFunction(max, Int32)) ENGINE=MergeTree ORDER BY tuple(); +INSERT INTO concat_saf_test VALUES (42); +INSERT INTO concat_saf_test SELECT max(number) FROM numbers(5); +SELECT concat('With ', x) FROM concat_saf_test ORDER BY x DESC; + +SELECT '-- Nested'; +CREATE OR REPLACE TABLE concat_nested_test(kv Nested(k String, v String)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO concat_nested_test VALUES (['foo', 'bar'], ['qaz', 'qux']); +SELECT concat('With ', kv) FROM concat_nested_test; + SELECT '-- NULL arguments'; SELECT concat(NULL, NULL); SELECT concat(NULL, materialize(NULL :: Nullable(UInt64))); SELECT concat(materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); - SELECT concat(42, materialize(NULL :: Nullable(UInt64))); SELECT concat('42', materialize(NULL :: Nullable(UInt64))); - SELECT concat(42, materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); SELECT concat('42', materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); @@ -72,3 +80,6 @@ SELECT concat(42, 144); SELECT concat(42, 144, 255); SELECT CONCAT('Testing the ', 'alias'); + +SELECT concat(); -- { serverError 42 } +SELECT concat(1); -- { serverError 42 } diff --git a/tests/queries/0_stateless/02233_interpolate_1.sql b/tests/queries/0_stateless/02233_interpolate_1.sql index 3d416b27f45..d589a18421b 100644 --- a/tests/queries/0_stateless/02233_interpolate_1.sql +++ b/tests/queries/0_stateless/02233_interpolate_1.sql @@ -26,7 +26,7 @@ SELECT n, source, inter FROM ( # Test INTERPOLATE with incompatible expression - should produce error SELECT n, source, inter FROM ( SELECT toFloat32(number % 10) AS n, 'original' AS source, number as inter FROM numbers(10) WHERE number % 3 = 1 -) ORDER BY n WITH FILL FROM 0 TO 11.51 STEP 0.5 INTERPOLATE (inter AS inter||'inter'); -- { serverError 44 } +) ORDER BY n WITH FILL FROM 0 TO 11.51 STEP 0.5 INTERPOLATE (inter AS reverse(inter)); -- { serverError 44 } # Test INTERPOLATE with column from WITH FILL expression - should produce error SELECT n, source, inter FROM ( diff --git a/tests/queries/0_stateless/02389_analyzer_nested_lambda.reference b/tests/queries/0_stateless/02389_analyzer_nested_lambda.reference index 935c53358c0..68eb282a6a1 100644 --- a/tests/queries/0_stateless/02389_analyzer_nested_lambda.reference +++ b/tests/queries/0_stateless/02389_analyzer_nested_lambda.reference @@ -117,5 +117,5 @@ SELECT arrayMap(x -> concat(concat(concat(concat(concat(toString(id), '___\0____ FROM test_table WHERE concat(concat(concat(toString(id), '___\0_______\0____'), toString(id)), concat(toString(id), NULL), toString(id)); SELECT '--'; -- -SELECT arrayMap(x -> concat(toString(id), arrayMap(x -> toString(1), [NULL])), [NULL]) FROM test_table; -- { serverError 44 }; +SELECT arrayMap(x -> splitByChar(toString(id), arrayMap(x -> toString(1), [NULL])), [NULL]) FROM test_table; -- { serverError 44 }; DROP TABLE test_table; diff --git a/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql b/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql index 8f8b5537da9..48e84246d1c 100644 --- a/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql +++ b/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql @@ -122,7 +122,7 @@ FROM test_table WHERE concat(concat(concat(toString(id), '___\0_______\0____'), SELECT '--'; -SELECT arrayMap(x -> concat(toString(id), arrayMap(x -> toString(1), [NULL])), [NULL]) FROM test_table; -- { serverError 44 }; +SELECT arrayMap(x -> splitByChar(toString(id), arrayMap(x -> toString(1), [NULL])), [NULL]) FROM test_table; -- { serverError 44 }; DROP TABLE test_table; diff --git a/tests/queries/0_stateless/02521_analyzer_array_join_crash.reference b/tests/queries/0_stateless/02521_analyzer_array_join_crash.reference index 59da8ccad1a..5e7728e0590 100644 --- a/tests/queries/0_stateless/02521_analyzer_array_join_crash.reference +++ b/tests/queries/0_stateless/02521_analyzer_array_join_crash.reference @@ -8,4 +8,4 @@ SELECT id, value_element, value FROM test_table ARRAY JOIN [[1,2,3]] AS value_el 0 [1,2,3] 3 SELECT value_element, value FROM test_table ARRAY JOIN [1048577] AS value_element, arrayMap(x -> value_element, ['']) AS value; 1048577 [1048577] -SELECT arrayFilter(x -> notEmpty(concat(x)), [NULL, NULL]) FROM system.one ARRAY JOIN [1048577] AS elem, arrayMap(x -> concat(x, elem, ''), ['']) AS unused; -- { serverError 44 } +SELECT arrayFilter(x -> notEmpty(concat(x)), [NULL, NULL]) FROM system.one ARRAY JOIN [1048577] AS elem, arrayMap(x -> splitByChar(x, elem), ['']) AS unused; -- { serverError 44 } diff --git a/tests/queries/0_stateless/02521_analyzer_array_join_crash.sql b/tests/queries/0_stateless/02521_analyzer_array_join_crash.sql index c7641a3bee0..53606e01ab7 100644 --- a/tests/queries/0_stateless/02521_analyzer_array_join_crash.sql +++ b/tests/queries/0_stateless/02521_analyzer_array_join_crash.sql @@ -17,7 +17,7 @@ SELECT id, value_element, value FROM test_table ARRAY JOIN [[1,2,3]] AS value_el SELECT value_element, value FROM test_table ARRAY JOIN [1048577] AS value_element, arrayMap(x -> value_element, ['']) AS value; -SELECT arrayFilter(x -> notEmpty(concat(x)), [NULL, NULL]) FROM system.one ARRAY JOIN [1048577] AS elem, arrayMap(x -> concat(x, elem, ''), ['']) AS unused; -- { serverError 44 } +SELECT arrayFilter(x -> notEmpty(concat(x)), [NULL, NULL]) FROM system.one ARRAY JOIN [1048577] AS elem, arrayMap(x -> splitByChar(x, elem), ['']) AS unused; -- { serverError 44 } -- { echoOff } From ddca2c2187d42b39fb139460cd32eb7e71adbce1 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 16 Nov 2023 14:29:53 +0100 Subject: [PATCH 102/192] server side waiting --- src/Storages/MergeTree/DataPartsExchange.cpp | 113 ++++++++---------- src/Storages/MergeTree/MergeTreeSettings.h | 4 +- tests/clickhouse-test | 2 +- ...916_replication_protocol_wait_for_part.sql | 7 +- 4 files changed, 57 insertions(+), 69 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 7fd6f59ed69..c39263a0b73 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -65,7 +65,6 @@ constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID = 5; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_ZERO_COPY = 6; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION = 7; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION = 8; -constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE = 9; std::string getEndpointId(const std::string & node_id) { @@ -121,7 +120,7 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write MergeTreePartInfo::fromPartName(part_name, data.format_version); /// We pretend to work as older server version, to be sure that client will correctly process our version - response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE))}); + response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION))}); LOG_TRACE(log, "Sending part {}", part_name); @@ -139,29 +138,6 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write { part = findPart(part_name); - /// Ephemeral zero-copy lock may be lost for PreActive parts - /// do not expose PreActive parts - if (client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE) - { - bool part_is_ready = part->getState() != MergeTreeDataPartState::PreActive; - writeBinary(part_is_ready, out); - - if (!part_is_ready) - { - LOG_TRACE(log, "Part {} is in PreActive state, reply to the client that part is not ready yet", part_name); - return; - } - } - else - { - bool zero_copy_enabled = data.getSettings()->allow_remote_fs_zero_copy_replication; - if (part->getState() == MergeTreeDataPartState::PreActive && zero_copy_enabled) - { - /// report error, client will try again later, error message would be printed - throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in table", part_name); - } - } - CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedSend}; if (part->getDataPartStorage().isStoredOnRemoteDisk()) @@ -373,6 +349,25 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( return data_checksums; } +bool wait_loop(UInt32 wait_timeout_ms, std::function pred) +{ + static const UInt32 loop_delay_ms = 5; + + /// this is sleep-based wait, it has to be short + chassert(wait_timeout_ms < 2000); + + if (pred()) + return true; + + Stopwatch timer; + while (!pred() && timer.elapsedMilliseconds() < wait_timeout_ms) + { + sleepForMilliseconds(loop_delay_ms); + } + + return pred(); +} + MergeTreeData::DataPartPtr Service::findPart(const String & name) { /// It is important to include Outdated parts here because remote replicas cannot reliably @@ -381,10 +376,26 @@ MergeTreeData::DataPartPtr Service::findPart(const String & name) part = data.getPartIfExists(name, {MergeTreeDataPartState::PreActive, MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated}); - if (part) + if (!part) + throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in table", name); + + bool zero_copy_enabled = data.getSettings()->allow_remote_fs_zero_copy_replication; + if (!zero_copy_enabled) return part; - throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in table", name); + /// Ephemeral zero-copy lock may be lost for PreActive parts + /// do not expose PreActive parts for zero-copy + + static const UInt32 wait_timeout_ms = 1000; + bool pred_result = wait_loop(wait_timeout_ms, [&] () { return part->getState() != MergeTreeDataPartState::PreActive; }); + + if (!pred_result) + throw Exception( + ErrorCodes::ABORTED, + "Part {} is in PreActive state for {} ms. Another host has to be asked.", + name, wait_timeout_ms); + + return part; } Fetcher::Fetcher(StorageReplicatedMergeTree & data_) @@ -442,7 +453,7 @@ std::pair Fetcher::fetchSelected { {"endpoint", endpoint_id}, {"part", part_name}, - {"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE)}, + {"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION)}, {"compress", "false"} }); @@ -500,43 +511,17 @@ std::pair Fetcher::fetchSelected creds.setPassword(password); } - std::unique_ptr in; - int server_protocol_version = 0; - bool part_is_ready = true; - - static const UInt32 part_not_ready_attempts = 5; - static const UInt32 wait_sleep_time_ms = 100; - - for (UInt32 attempt = 1; attempt <= part_not_ready_attempts; ++attempt) - { - in = std::make_unique( - uri, - Poco::Net::HTTPRequest::HTTP_POST, - nullptr, - timeouts, - creds, - DBMS_DEFAULT_BUFFER_SIZE, - 0, /* no redirects */ - static_cast(data_settings->replicated_max_parallel_fetches_for_host)); - - server_protocol_version = parse(in->getResponseCookie("server_protocol_version", "0")); - - if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE) - readBinary(part_is_ready, *in); - - if (part_is_ready) - break; - - sleepForMilliseconds(wait_sleep_time_ms); - - if (blocker.isCancelled()) - throw Exception(ErrorCodes::ABORTED, "Fetching of part was cancelled"); - } - - if (!part_is_ready) - throw Exception(ErrorCodes::ABORTED, "Part {} is still not ready in host {} after {} attempts, try another host", - part_name, host, part_not_ready_attempts); + std::unique_ptr in = std::make_unique( + uri, + Poco::Net::HTTPRequest::HTTP_POST, + nullptr, + timeouts, + creds, + DBMS_DEFAULT_BUFFER_SIZE, + 0, /* no redirects */ + static_cast(data_settings->replicated_max_parallel_fetches_for_host)); + int server_protocol_version = parse(in->getResponseCookie("server_protocol_version", "0")); String remote_fs_metadata = parse(in->getResponseCookie("remote_fs_metadata", "")); DiskPtr preffered_disk = disk; diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 15c54ee3791..41476bab5b1 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -83,7 +83,7 @@ struct Settings; M(UInt64, max_delay_to_insert, 1, "Max delay of inserting data into MergeTree table in seconds, if there are a lot of unmerged parts in single partition.", 0) \ M(UInt64, min_delay_to_insert_ms, 10, "Min delay of inserting data into MergeTree table in milliseconds, if there are a lot of unmerged parts in single partition.", 0) \ M(UInt64, max_parts_in_total, 100000, "If more than this number active parts in all partitions in total, throw 'Too many parts ...' exception.", 0) \ - M(Bool, async_insert, false, "If true, data from INSERT query is stored in queue and later flushed to table in background.", 0) \ + M(Bool, async_insert, false, "If true, data from INSERT query is stored in queue and later flushed to table in background.", 0) \ M(Milliseconds, sleep_before_commit_local_part_in_replicated_table_ms, 0, "For testing. Do not change it.", 0) \ \ /* Part removal settings. */ \ @@ -122,7 +122,7 @@ struct Settings; M(UInt64, max_replicated_sends_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for replicated sends. Zero means unlimited.", 0) \ M(Milliseconds, wait_for_unique_parts_send_before_shutdown_ms, 0, "Before shutdown table will wait for required amount time for unique parts (exist only on current replica) to be fetched by other replicas (0 means disabled).", 0) \ M(Float, fault_probability_before_part_commit, 0, "For testing. Do not change it.", 0) \ - M(Float, fault_probability_after_part_commit, 0, "For testing. Do not change it.", 0) \ + M(Float, fault_probability_after_part_commit, 0, "For testing. Do not change it.", 0) \ \ /** Check delay of replicas settings. */ \ M(UInt64, min_relative_delay_to_measure, 120, "Calculate relative replica delay only if absolute delay is not less that this value.", 0) \ diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 048f848ff27..053bd040bce 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -674,7 +674,7 @@ class MergeTreeSettingsRandomizer: "replace_long_file_name_to_hash": lambda: random.randint(0, 1), "max_file_name_length": threshold_generator(0.3, 0.3, 0, 128), "sleep_before_commit_local_part_in_replicated_table_ms": threshold_generator( - 0.3, 0.3, 0, 250 + 0.7, 0.7, 0, 100 ), } diff --git a/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql index ed9cfd00b45..97ef33f96e8 100644 --- a/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql +++ b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql @@ -7,7 +7,7 @@ create table tableIn (n int) settings storage_policy='s3_cache', allow_remote_fs_zero_copy_replication=1, - sleep_before_commit_local_part_in_replicated_table_ms=50000; + sleep_before_commit_local_part_in_replicated_table_ms=5000; create table tableOut (n int) engine=ReplicatedMergeTree('/test/02916/{database}/table', '2') order by tuple() @@ -15,9 +15,12 @@ create table tableOut (n int) storage_policy='s3_cache', allow_remote_fs_zero_copy_replication=1; -SET send_logs_level = 'error'; +SET send_logs_level='error'; insert into tableIn values(1); insert into tableIn values(2); system sync replica tableOut; select count() from tableOut; + +drop table tableIn +drop table tableOut From b13cd9792b599d3686a73f9114fadda7d0e090f0 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 16 Nov 2023 14:46:57 +0000 Subject: [PATCH 103/192] Fix cross build --- contrib/google-protobuf-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/google-protobuf-cmake/CMakeLists.txt b/contrib/google-protobuf-cmake/CMakeLists.txt index 1ed4133270b..89bdbb89eca 100644 --- a/contrib/google-protobuf-cmake/CMakeLists.txt +++ b/contrib/google-protobuf-cmake/CMakeLists.txt @@ -369,7 +369,7 @@ else () "-Dprotobuf_BUILD_PROTOC_BINARIES=1" "-DABSL_ROOT_DIR=${abseil_source_dir}" "-DABSL_ENABLE_INSTALL=0" - "${protobuf_source_dir}/cmake" + "${protobuf_source_dir}" WORKING_DIRECTORY "${PROTOC_BUILD_DIR}" COMMAND_ECHO STDOUT) From cc64397e9282091d073386a5fd912689257b1445 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 16 Nov 2023 19:21:58 +0300 Subject: [PATCH 104/192] Planner support transactions --- src/Planner/Planner.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 89166316261..12e8d795347 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -116,7 +116,7 @@ namespace void checkStoragesSupportTransactions(const PlannerContextPtr & planner_context) { const auto & query_context = planner_context->getQueryContext(); - if (query_context->getSettingsRef().throw_on_unsupported_query_inside_transaction) + if (!query_context->getSettingsRef().throw_on_unsupported_query_inside_transaction) return; if (!query_context->getCurrentTransaction()) @@ -130,13 +130,11 @@ void checkStoragesSupportTransactions(const PlannerContextPtr & planner_context) else if (auto * table_function_node = table_expression->as()) storage = table_function_node->getStorage(); - if (storage->supportsTransactions()) - continue; - - throw Exception(ErrorCodes::NOT_IMPLEMENTED, - "Storage {} (table {}) does not support transactions", - storage->getName(), - storage->getStorageID().getNameForLogs()); + if (storage && !storage->supportsTransactions()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "Storage {} (table {}) does not support transactions", + storage->getName(), + storage->getStorageID().getNameForLogs()); } } @@ -1333,9 +1331,9 @@ void Planner::buildPlanForQueryNode() query_node.getHaving() = {}; } - checkStoragesSupportTransactions(planner_context); collectSets(query_tree, *planner_context); collectTableExpressionData(query_tree, planner_context); + checkStoragesSupportTransactions(planner_context); if (!select_query_options.only_analyze) collectFiltersForAnalysis(query_tree, planner_context); From 00569baf4b9703891918be979f192d425dc81a2e Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 16 Nov 2023 19:23:55 +0300 Subject: [PATCH 105/192] Updated analyzer failed tests --- tests/analyzer_tech_debt.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/analyzer_tech_debt.txt b/tests/analyzer_tech_debt.txt index e155ee72ebb..d969b9e6fad 100644 --- a/tests/analyzer_tech_debt.txt +++ b/tests/analyzer_tech_debt.txt @@ -6,7 +6,6 @@ 01064_incremental_streaming_from_2_src_with_feedback 01083_expressions_in_engine_arguments 01155_rename_move_materialized_view -01173_transaction_control_queries 01214_test_storage_merge_aliases_with_where 01244_optimize_distributed_group_by_sharding_key 01268_mv_scalars @@ -30,7 +29,6 @@ 02139_MV_with_scalar_subquery 02174_cte_scalar_cache_mv 02302_s3_file_pruning -02345_implicit_transaction 02352_grouby_shadows_arg 02354_annoy 02428_parameterized_view From 49c58e76099b6a3771182c9bd093159054d707a0 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 16 Nov 2023 16:24:09 +0000 Subject: [PATCH 106/192] Disable RabbitMQ secure connection test in intergatiion test with TSAN --- tests/integration/test_storage_rabbitmq/test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 021cdf54af9..4f4d4dec02f 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -110,6 +110,9 @@ def rabbitmq_setup_teardown(): ], ) def test_rabbitmq_select(rabbitmq_cluster, secure): + if secure and instance.is_built_with_memory_sanitizer(): + pytest.skip("Data races: see https://github.com/ClickHouse/ClickHouse/issues/56866") + port = cluster.rabbitmq_port if secure: port = cluster.rabbitmq_secure_port From d03a1aab7bea331738d915d23b7353ccbbac9cf4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 16 Nov 2023 16:39:57 +0000 Subject: [PATCH 107/192] Automatic style fix --- tests/integration/test_storage_rabbitmq/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 4f4d4dec02f..cb34f7203d6 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -111,7 +111,9 @@ def rabbitmq_setup_teardown(): ) def test_rabbitmq_select(rabbitmq_cluster, secure): if secure and instance.is_built_with_memory_sanitizer(): - pytest.skip("Data races: see https://github.com/ClickHouse/ClickHouse/issues/56866") + pytest.skip( + "Data races: see https://github.com/ClickHouse/ClickHouse/issues/56866" + ) port = cluster.rabbitmq_port if secure: From 2ec96f9e9e737b2805f0408c3639c19760b445af Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 16 Nov 2023 17:56:06 +0100 Subject: [PATCH 108/192] Update 01052_window_view_proc_tumble_to_now.sh --- .../queries/0_stateless/01052_window_view_proc_tumble_to_now.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh b/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh index e75b7d9570b..4325ebeed24 100755 --- a/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh +++ b/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh @@ -16,7 +16,7 @@ DROP TABLE IF EXISTS wv; CREATE TABLE dst(count UInt64) Engine=MergeTree ORDER BY tuple(); CREATE TABLE mt(a Int32) ENGINE=MergeTree ORDER BY tuple(); -CREATE WINDOW VIEW wv TO dst AS SELECT count(a) AS count FROM mt GROUP BY tumble(now('US/Samoa'), INTERVAL '5' SECOND, 'US/Samoa') AS wid; +CREATE WINDOW VIEW wv TO dst AS SELECT count(a) AS count FROM mt GROUP BY tumble(now('US/Samoa'), INTERVAL '10' SECOND, 'US/Samoa') AS wid; INSERT INTO mt VALUES (1); EOF From 409f781c0907f4862ddf57d80c6e1adc1ad8b77e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 16 Nov 2023 18:10:26 +0100 Subject: [PATCH 109/192] Fix test --- tests/queries/0_stateless/01119_session_log.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01119_session_log.sql b/tests/queries/0_stateless/01119_session_log.sql index 8f6967b89ec..55f6228797a 100644 --- a/tests/queries/0_stateless/01119_session_log.sql +++ b/tests/queries/0_stateless/01119_session_log.sql @@ -4,7 +4,7 @@ select * from remote('127.0.0.2', system, one, 'default', ''); select * from remote('127.0.0.2', system, one, 'default', 'wrong password'); -- { serverError AUTHENTICATION_FAILED } select * from remote('127.0.0.2', system, one, 'nonexistsnt_user_1119', ''); -- { serverError AUTHENTICATION_FAILED } set receive_timeout=1; -select * from remote('127.0.0.2', system, one, ' INTERSERVER SECRET ', ''); -- { serverError AUTHENTICATION_FAILED } +select * from remote('127.0.0.2', system, one, ' INTERSERVER SECRET ', ''); -- { serverError NO_REMOTE_SHARD_AVAILABLE } set receive_timeout=300; select * from remote('127.0.0.2', system, one, ' ', ''); -- { serverError AUTHENTICATION_FAILED } From d24757bbbf6b4050429d5e2d4f1e7ecfd9fed93d Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 16 Nov 2023 18:11:40 +0100 Subject: [PATCH 110/192] Add assertions --- src/Interpreters/Cache/FileCache.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 5de24977db5..0591038fc1d 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -801,6 +801,11 @@ void FileCache::removePathIfExists(const String & path) void FileCache::removeAllReleasable() { assertInitialized(); + +#ifdef ABORT_ON_LOGICAL_ERROR + assertCacheCorrectness(); +#endif + metadata.removeAllKeys(/* if_releasable */true); if (stash) From d83cf03c30f5cde7d7442c6f01da04f8fb7c2a3a Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 16 Nov 2023 19:21:27 +0100 Subject: [PATCH 111/192] no randomization sleep_before_commit_local_part_in_replicated_table_ms --- tests/clickhouse-test | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 053bd040bce..cab7d7e79ff 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -673,9 +673,6 @@ class MergeTreeSettingsRandomizer: "primary_key_compress_block_size": lambda: random.randint(8000, 100000), "replace_long_file_name_to_hash": lambda: random.randint(0, 1), "max_file_name_length": threshold_generator(0.3, 0.3, 0, 128), - "sleep_before_commit_local_part_in_replicated_table_ms": threshold_generator( - 0.7, 0.7, 0, 100 - ), } @staticmethod From 28e0c51e3fdcf4692ad18d9e535a9ff4771d3955 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 16 Nov 2023 20:46:17 +0100 Subject: [PATCH 112/192] Update avg_weighted.xml (#56797) --- tests/performance/avg_weighted.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/performance/avg_weighted.xml b/tests/performance/avg_weighted.xml index 5aa89b08c35..edf3c19fdfa 100644 --- a/tests/performance/avg_weighted.xml +++ b/tests/performance/avg_weighted.xml @@ -27,10 +27,6 @@ SELECT avg(num_f) FROM perf_avg FORMAT Null SELECT avgWeighted(num_f, num) FROM perf_avg FORMAT Null SELECT avgWeighted(num_f, num_f) FROM perf_avg FORMAT Null - SELECT avgWeighted(num_f, num_u) FROM perf_avg FORMAT Null - SELECT avgWeighted(num_u, num_f) FROM perf_avg FORMAT Null - SELECT avgWeighted(num_u, num) FROM perf_avg FORMAT Null - SELECT avgWeighted(num_u, num_u) FROM perf_avg FORMAT Null SELECT avgWeighted(num_f, num_f) FROM perf_avg FORMAT Null SELECT avgWeighted(toNullable(num_f), num_f) FROM perf_avg FORMAT Null From e533abef75f9eb304ceea09a74afb37672728420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 16 Nov 2023 20:54:52 +0100 Subject: [PATCH 113/192] Resolve 01572_kill_window_function flakiness --- .../0_stateless/01572_kill_window_function.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/01572_kill_window_function.sh b/tests/queries/0_stateless/01572_kill_window_function.sh index 7103b7f7210..de6de3510a0 100755 --- a/tests/queries/0_stateless/01572_kill_window_function.sh +++ b/tests/queries/0_stateless/01572_kill_window_function.sh @@ -6,21 +6,20 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -o pipefail +function wait_for_query_to_start() +{ + while [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() FROM system.processes WHERE query_id = '$1'") == 0 ]]; do sleep 0.1; done +} + # Run a test query that takes very long to run. query_id="01572_kill_window_function-$CLICKHOUSE_DATABASE" -$CLICKHOUSE_CLIENT --query_id="$query_id" --query "SELECT count(1048575) OVER (PARTITION BY intDiv(NULL, number) ORDER BY number DESC NULLS FIRST ROWS BETWEEN CURRENT ROW AND 1048575 FOLLOWING) FROM numbers(255, 1048575)" >/dev/null 2>&1 & +$CLICKHOUSE_CLIENT --query_id="$query_id" --query "SELECT sum(number) OVER (PARTITION BY number % 10 ORDER BY number DESC NULLS FIRST ROWS BETWEEN CURRENT ROW AND 99999 FOLLOWING) FROM numbers(0, 10000000) format Null;" >/dev/null 2>&1 & client_pid=$! echo Started -# Use one query to both kill the test query and verify that it has started, -# because if we try to kill it before it starts, the test will fail. -while [ -z "$($CLICKHOUSE_CLIENT --query "kill query where query_id = '$query_id' and current_database = currentDatabase()")" ] -do - # If we don't yet see the query in the process list, the client should still - # be running. The query is very long. - kill -0 -- $client_pid - sleep 1 -done +wait_for_query_to_start $query_id + +$CLICKHOUSE_CLIENT --query "kill query where query_id = '$query_id' and current_database = currentDatabase() format Null" echo Sent kill request # Wait for the client to terminate. From ac3100747575e9f86493dfeb32a081734fac3a5c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 16 Nov 2023 21:00:39 +0100 Subject: [PATCH 114/192] Update DatabaseReplicated.cpp (#56796) --- src/Databases/DatabaseReplicated.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index a643eafdd14..5da20c42465 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -1083,12 +1083,14 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep } LOG_INFO(log, "All tables are created successfully"); - if (max_log_ptr_at_creation != 0) + chassert(max_log_ptr_at_creation || our_log_ptr); + UInt32 first_entry_to_mark_finished = new_replica ? max_log_ptr_at_creation : our_log_ptr; + if (first_entry_to_mark_finished) { /// If the replica is new and some of the queries applied during recovery /// where issued after the replica was created, then other nodes might be /// waiting for this node to notify them that the query was applied. - for (UInt32 ptr = max_log_ptr_at_creation; ptr <= max_log_ptr; ++ptr) + for (UInt32 ptr = first_entry_to_mark_finished; ptr <= max_log_ptr; ++ptr) { auto entry_name = DDLTaskBase::getLogEntryName(ptr); auto path = fs::path(zookeeper_path) / "log" / entry_name / "finished" / getFullReplicaName(); From 3765a1c77b586455565940b217d2a40f3d49333d Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Thu, 16 Nov 2023 16:07:49 -0400 Subject: [PATCH 115/192] Update random-functions.md --- docs/en/sql-reference/functions/random-functions.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/functions/random-functions.md b/docs/en/sql-reference/functions/random-functions.md index 13c29329f41..6fd31e8d25c 100644 --- a/docs/en/sql-reference/functions/random-functions.md +++ b/docs/en/sql-reference/functions/random-functions.md @@ -6,9 +6,9 @@ sidebar_label: Random Numbers # Functions for Generating Random Numbers -All functions in this section accept zero or one arguments. The only use of the argument (if provided) is to prevent prevent [common subexpression -elimination](../../sql-reference/functions/index.md#common-subexpression-elimination) such that two different execution of the same random -function in a query return different random values. +All functions in this section accept zero or one arguments. The only use of the argument (if provided) is to prevent [common subexpression +elimination](../../sql-reference/functions/index.md#common-subexpression-elimination) such that two different executions within a row of the same random +function return different random values. Related content - Blog: [Generating random data in ClickHouse](https://clickhouse.com/blog/generating-random-test-distribution-data-for-clickhouse) From 1abcb28624ee4d204e8e1723ad58e87d1d80be56 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 16 Nov 2023 23:32:17 +0100 Subject: [PATCH 116/192] Remove ctest --- CMakeLists.txt | 1 - cmake/add_check.cmake | 19 ------------------- src/CMakeLists.txt | 4 ---- tests/CMakeLists.txt | 26 -------------------------- 4 files changed, 50 deletions(-) delete mode 100644 cmake/add_check.cmake delete mode 100644 tests/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f05b2b78ce..4fe7a1e05e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -561,7 +561,6 @@ option(CHECK_LARGE_OBJECT_SIZES "Check that there are no large object files afte add_subdirectory (base) add_subdirectory (src) add_subdirectory (programs) -add_subdirectory (tests) add_subdirectory (utils) if (FUZZER) diff --git a/cmake/add_check.cmake b/cmake/add_check.cmake deleted file mode 100644 index ba30ee8676f..00000000000 --- a/cmake/add_check.cmake +++ /dev/null @@ -1,19 +0,0 @@ -# Adding test output on failure -enable_testing () - -if (NOT TARGET check) - if (CMAKE_CONFIGURATION_TYPES) - add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND} - --force-new-ctest-process --output-on-failure --build-config "$" - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) - else () - add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND} - --force-new-ctest-process --output-on-failure - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) - endif () -endif () - -macro (add_check target) - add_test (NAME test_${target} COMMAND ${target} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - add_dependencies (check ${target}) -endmacro (add_check) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5254743e154..3733295e9b4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -600,8 +600,6 @@ if (TARGET ch_rust::skim) dbms_target_link_libraries(PUBLIC ch_rust::skim) endif() -include ("${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake") - if (ENABLE_TESTS) macro (grep_gtest_sources BASE_DIR DST_VAR) # Cold match files that are not in tests/ directories @@ -645,6 +643,4 @@ if (ENABLE_TESTS) if (TARGET ch_contrib::parquet) target_link_libraries(unit_tests_dbms PRIVATE ch_contrib::parquet) endif() - - add_check(unit_tests_dbms) endif () diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt deleted file mode 100644 index 22c89aaafa7..00000000000 --- a/tests/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -enable_testing() - -# Run tests with "ninja check" or "make check" -if (TARGET check) - message (STATUS "Target check already exists") -else () - include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake) -endif () - -option (ENABLE_CLICKHOUSE_TEST "Install clickhouse-test script and relevant tests scenarios" OFF) - -if (ENABLE_CLICKHOUSE_TEST) - install (PROGRAMS clickhouse-test DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - install ( - DIRECTORY queries performance config - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse-test - USE_SOURCE_PERMISSIONS - COMPONENT clickhouse - PATTERN "CMakeLists.txt" EXCLUDE - PATTERN ".gitignore" EXCLUDE - ) -endif () - -if (ENABLE_TEST_INTEGRATION) - add_subdirectory (integration) -endif () From c72136b123685f6e4704b9d39122f637ee95ce0c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 08:07:32 +0100 Subject: [PATCH 117/192] Simpler CMake --- contrib/arrow-cmake/CMakeLists.txt | 16 ++++++++-------- contrib/azure-cmake/CMakeLists.txt | 3 +-- contrib/thrift-cmake/CMakeLists.txt | 6 ------ 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 71133451889..935fc886330 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -77,16 +77,16 @@ set(FLATBUFFERS_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/flatbuffers") set(FLATBUFFERS_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/flatbuffers") set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_SRC_DIR}/include") -# set flatbuffers CMake options -set(FLATBUFFERS_BUILD_FLATLIB ON CACHE BOOL "Enable the build of the flatbuffers library") -set(FLATBUFFERS_BUILD_SHAREDLIB OFF CACHE BOOL "Disable the build of the flatbuffers shared library") -set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "Skip flatbuffers tests") +set(FLATBUFFERS_SRCS + ${FLATBUFFERS_SRC_DIR}/src/idl_parser.cpp + ${FLATBUFFERS_SRC_DIR}/src/idl_gen_text.cpp + ${FLATBUFFERS_SRC_DIR}/src/reflection.cpp + ${FLATBUFFERS_SRC_DIR}/src/util.cpp) -add_subdirectory(${FLATBUFFERS_SRC_DIR} "${FLATBUFFERS_BINARY_DIR}") +add_library(_flatbuffers STATIC ${FLATBUFFERS_SRCS}) +target_include_directories(_flatbuffers PUBLIC ${FLATBUFFERS_INCLUDE_DIR}) +target_compile_definitions(_flatbuffers PRIVATE -DFLATBUFFERS_LOCALE_INDEPENDENT=0) -add_library(_flatbuffers INTERFACE) -target_link_libraries(_flatbuffers INTERFACE flatbuffers) -target_include_directories(_flatbuffers INTERFACE ${FLATBUFFERS_INCLUDE_DIR}) # === hdfs # NOTE: cannot use ch_contrib::hdfs since it's INCLUDE_DIRECTORIES does not includes trailing "hdfs/" diff --git a/contrib/azure-cmake/CMakeLists.txt b/contrib/azure-cmake/CMakeLists.txt index 7aba81259d3..bb44c993e79 100644 --- a/contrib/azure-cmake/CMakeLists.txt +++ b/contrib/azure-cmake/CMakeLists.txt @@ -48,9 +48,8 @@ set(AZURE_SDK_INCLUDES "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/inc/" ) -include("${AZURE_DIR}/cmake-modules/AzureTransportAdapters.cmake") - add_library(_azure_sdk ${AZURE_SDK_UNIFIED_SRC}) +target_compile_definitions(_azure_sdk PRIVATE BUILD_CURL_HTTP_TRANSPORT_ADAPTER) # Originally, on Windows azure-core is built with bcrypt and crypt32 by default if (TARGET OpenSSL::SSL) diff --git a/contrib/thrift-cmake/CMakeLists.txt b/contrib/thrift-cmake/CMakeLists.txt index d6aa6b9e5f2..89a444cfb83 100644 --- a/contrib/thrift-cmake/CMakeLists.txt +++ b/contrib/thrift-cmake/CMakeLists.txt @@ -47,8 +47,6 @@ set(thriftcpp_threads_SOURCES "${LIBRARY_DIR}/src/thrift/concurrency/Mutex.cpp" ) -include("${ClickHouse_SOURCE_DIR}/contrib/thrift/build/cmake/ConfigureChecks.cmake") # makes config.h - set (HAVE_ARPA_INET_H 1) set (HAVE_FCNTL_H 1) set (HAVE_GETOPT_H 1) @@ -81,10 +79,6 @@ if (OS_LINUX AND NOT USE_MUSL) set (STRERROR_R_CHAR_P 1) endif () -#set(PACKAGE ${PACKAGE_NAME}) -#set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") -#set(VERSION ${thrift_VERSION}) - # generate a config.h file configure_file("${CMAKE_CURRENT_SOURCE_DIR}/build/cmake/config.h.in" "${CMAKE_CURRENT_BINARY_DIR}/thrift/config.h") From 5496e2d6ac483beeccca03a2ed92b586e33f28dd Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Fri, 17 Nov 2023 10:24:47 +0300 Subject: [PATCH 118/192] test for #56790 --- .../00059_shard_global_in_mergetree.reference | 8 +++++++ .../00059_shard_global_in_mergetree.sql | 24 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/queries/0_stateless/00059_shard_global_in_mergetree.reference create mode 100644 tests/queries/0_stateless/00059_shard_global_in_mergetree.sql diff --git a/tests/queries/0_stateless/00059_shard_global_in_mergetree.reference b/tests/queries/0_stateless/00059_shard_global_in_mergetree.reference new file mode 100644 index 00000000000..829419dc759 --- /dev/null +++ b/tests/queries/0_stateless/00059_shard_global_in_mergetree.reference @@ -0,0 +1,8 @@ +20 +20 +20 +20 +20 +20 +20 +20 diff --git a/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql b/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql new file mode 100644 index 00000000000..b85560d2bea --- /dev/null +++ b/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql @@ -0,0 +1,24 @@ +-- Tags: shard + +-- test for #56790 + +CREATE TABLE test_local (x Int64) +ENGINE = MergeTree order by x as select * from numbers(10); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where 'XXX' global in (select 'XXX'); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * global in (select * from test_local); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * in (select * from test_local); + +set prefer_localhost_replica=0; + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where 'XXX' global in (select 'XXX'); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * global in (select * from test_local); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * in (select * from test_local); From c4f46c7ce575fbfcb52c69343534b9539b3f719d Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Fri, 17 Nov 2023 10:27:02 +0300 Subject: [PATCH 119/192] test for #56790 --- .../0_stateless/00059_shard_global_in_mergetree.reference | 1 - tests/queries/0_stateless/00059_shard_global_in_mergetree.sql | 2 -- 2 files changed, 3 deletions(-) diff --git a/tests/queries/0_stateless/00059_shard_global_in_mergetree.reference b/tests/queries/0_stateless/00059_shard_global_in_mergetree.reference index 829419dc759..208e649c056 100644 --- a/tests/queries/0_stateless/00059_shard_global_in_mergetree.reference +++ b/tests/queries/0_stateless/00059_shard_global_in_mergetree.reference @@ -5,4 +5,3 @@ 20 20 20 -20 diff --git a/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql b/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql index b85560d2bea..cbd4245a486 100644 --- a/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql +++ b/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql @@ -7,8 +7,6 @@ ENGINE = MergeTree order by x as select * from numbers(10); select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local); -select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local); - select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where 'XXX' global in (select 'XXX'); select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * global in (select * from test_local); From 224b282d947daf5275e9a38d1c62e8887eb44868 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 08:27:19 +0100 Subject: [PATCH 120/192] Remove garbage --- contrib/cassandra-cmake/CMakeLists.txt | 4 ---- contrib/qpl-cmake/CMakeLists.txt | 3 +-- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/contrib/cassandra-cmake/CMakeLists.txt b/contrib/cassandra-cmake/CMakeLists.txt index 32611e0e151..9e729c436d5 100644 --- a/contrib/cassandra-cmake/CMakeLists.txt +++ b/contrib/cassandra-cmake/CMakeLists.txt @@ -83,10 +83,6 @@ set(HAVE_MEMCPY 1) set(HAVE_LONG_LONG 1) set(HAVE_UINT16_T 1) -configure_file("${CASS_SRC_DIR}/third_party/sparsehash/config.h.cmake" "${CMAKE_CURRENT_BINARY_DIR}/sparsehash/internal/sparseconfig.h") - - - # Determine random availability if (OS_LINUX) #set (HAVE_GETRANDOM 1) - not on every Linux kernel diff --git a/contrib/qpl-cmake/CMakeLists.txt b/contrib/qpl-cmake/CMakeLists.txt index 4e6c66fe731..19501209b26 100644 --- a/contrib/qpl-cmake/CMakeLists.txt +++ b/contrib/qpl-cmake/CMakeLists.txt @@ -16,8 +16,7 @@ function(GetLibraryVersion _content _outputVar) SET(${_outputVar} ${CMAKE_MATCH_1} PARENT_SCOPE) endfunction() -FILE(READ "${QPL_PROJECT_DIR}/CMakeLists.txt" HEADER_CONTENT) -GetLibraryVersion("${HEADER_CONTENT}" QPL_VERSION) +set (QPL_VERSION 1.2.0) message(STATUS "Intel QPL version: ${QPL_VERSION}") From df3c066591758813c99422bab11f0811e1d487d3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 08:42:39 +0100 Subject: [PATCH 121/192] Remove more trash --- contrib/arrow-cmake/CMakeLists.txt | 1 - contrib/cassandra-cmake/CMakeLists.txt | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 935fc886330..96d1f4adda7 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -127,7 +127,6 @@ set(ORC_SRCS "${ORC_SOURCE_SRC_DIR}/BpackingDefault.hh" "${ORC_SOURCE_SRC_DIR}/ByteRLE.cc" "${ORC_SOURCE_SRC_DIR}/ByteRLE.hh" - "${ORC_SOURCE_SRC_DIR}/CMakeLists.txt" "${ORC_SOURCE_SRC_DIR}/ColumnPrinter.cc" "${ORC_SOURCE_SRC_DIR}/ColumnReader.cc" "${ORC_SOURCE_SRC_DIR}/ColumnReader.hh" diff --git a/contrib/cassandra-cmake/CMakeLists.txt b/contrib/cassandra-cmake/CMakeLists.txt index 9e729c436d5..0082364c130 100644 --- a/contrib/cassandra-cmake/CMakeLists.txt +++ b/contrib/cassandra-cmake/CMakeLists.txt @@ -68,8 +68,7 @@ list(APPEND INCLUDE_DIRS ${CASS_SRC_DIR}/third_party/hdr_histogram ${CASS_SRC_DIR}/third_party/http-parser ${CASS_SRC_DIR}/third_party/mt19937_64 - ${CASS_SRC_DIR}/third_party/rapidjson/rapidjson - ${CASS_SRC_DIR}/third_party/sparsehash/src) + ${CASS_SRC_DIR}/third_party/rapidjson/rapidjson) list(APPEND INCLUDE_DIRS ${CASS_INCLUDE_DIR} ${CASS_SRC_DIR}) @@ -112,17 +111,17 @@ configure_file( ${CASS_ROOT_DIR}/driver_config.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/driver_config.hpp) - add_library(_cassandra ${SOURCES} $ $ $) -target_link_libraries(_cassandra ch_contrib::zlib ch_contrib::minizip) +target_link_libraries(_cassandra ch_contrib::zlib ch_contrib::minizip ch_contrib::sparsehash) target_include_directories(_cassandra PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${INCLUDE_DIRS}) target_include_directories(_cassandra SYSTEM BEFORE PUBLIC ${CASS_INCLUDE_DIR}) target_compile_definitions(_cassandra PRIVATE CASS_BUILDING) +target_compile_definitions(_cassandra PRIVATE -DSPARSEHASH_HASH=std::hash -Dsparsehash=google) target_link_libraries(_cassandra ch_contrib::uv) From f73b3e10ac0c4094cbc907cad2533ea73ae916b6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 08:46:45 +0100 Subject: [PATCH 122/192] Ensure no new dependencies --- docker/packager/binary/build.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 6b6374d08c9..ad31397c8d9 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -22,6 +22,7 @@ if [ "$EXTRACT_TOOLCHAIN_DARWIN" = "1" ]; then fi fi + # Uncomment to debug ccache. Don't put ccache log in /output right away, or it # will be confusingly packed into the "performance" package. # export CCACHE_LOGFILE=/build/ccache.log @@ -32,6 +33,16 @@ mkdir -p /build/build_docker cd /build/build_docker rm -f CMakeCache.txt + +# We don't want to depend on any third-party CMake files. +# To check it, find and delete them. + +grep -o -P '"contrib/[^"]+"' ../.gitmodules | + grep -v -P 'llvm-project|abseil-cpp|qpl|grpc|corrosion' | + xargs -I@ find @ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | + xargs rm + + if [ -n "$MAKE_DEB" ]; then rm -rf /build/packages/root # NOTE: this is for backward compatibility with previous releases, From c7d8465897e6d51a71d9c28bec51ba676fb70fa3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 09:12:49 +0100 Subject: [PATCH 123/192] Ensure no new dependencies --- docker/packager/binary/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index ad31397c8d9..d469b359d1a 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -39,7 +39,7 @@ rm -f CMakeCache.txt grep -o -P '"contrib/[^"]+"' ../.gitmodules | grep -v -P 'llvm-project|abseil-cpp|qpl|grpc|corrosion' | - xargs -I@ find @ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | + xargs -I@ find ../@ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | xargs rm From f456ac97fe7c411f897a432e80055e7ed3599ad2 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Fri, 17 Nov 2023 04:19:22 -0400 Subject: [PATCH 124/192] fix currentdatabase issue --- .../00059_shard_global_in_mergetree.sql | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql b/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql index cbd4245a486..62eec6f324b 100644 --- a/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql +++ b/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql @@ -2,21 +2,24 @@ -- test for #56790 -CREATE TABLE test_local (x Int64) -ENGINE = MergeTree order by x as select * from numbers(10); +DROP TABLE IF EXISTS test_local; + +CREATE TABLE test_local (x Int64) ENGINE = MergeTree order by x as select * from numbers(10); select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local); select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where 'XXX' global in (select 'XXX'); -select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * global in (select * from test_local); +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * global in (select * from numbers(10)); -select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * in (select * from test_local); +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * in (select * from numbers(10)); set prefer_localhost_replica=0; select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where 'XXX' global in (select 'XXX'); -select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * global in (select * from test_local); +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * global in (select * from numbers(10)); -select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * in (select * from test_local); +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * in (select * from numbers(10)); + +DROP TABLE test_local; From 864dd32b05c2eb40baf545279b15cc7aacc5937a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 09:48:42 +0100 Subject: [PATCH 125/192] Remove garbage --- .../client_scripts/allin1_ssb.sh | 530 ------------------ .../client_scripts/client_stressing_test.py | 278 --------- .../client_scripts/queries_ssb.sql | 10 - .../client_scripts/run_ssb.sh | 6 - .../database_dir/deflate/config_deflate.xml | 49 -- .../deflate_s2/config_deflate_s2.xml | 49 -- .../database_dir/lz4/config_lz4.xml | 49 -- .../database_dir/lz4_s2/config_lz4_s2.xml | 49 -- .../database_dir/zstd/config_zstd.xml | 49 -- .../database_dir/zstd_s2/config_zstd_s2.xml | 49 -- 10 files changed, 1118 deletions(-) delete mode 100644 contrib/qpl-cmake/benchmark_sample/client_scripts/allin1_ssb.sh delete mode 100644 contrib/qpl-cmake/benchmark_sample/client_scripts/client_stressing_test.py delete mode 100644 contrib/qpl-cmake/benchmark_sample/client_scripts/queries_ssb.sql delete mode 100644 contrib/qpl-cmake/benchmark_sample/client_scripts/run_ssb.sh delete mode 100644 contrib/qpl-cmake/benchmark_sample/database_dir/deflate/config_deflate.xml delete mode 100644 contrib/qpl-cmake/benchmark_sample/database_dir/deflate_s2/config_deflate_s2.xml delete mode 100644 contrib/qpl-cmake/benchmark_sample/database_dir/lz4/config_lz4.xml delete mode 100644 contrib/qpl-cmake/benchmark_sample/database_dir/lz4_s2/config_lz4_s2.xml delete mode 100644 contrib/qpl-cmake/benchmark_sample/database_dir/zstd/config_zstd.xml delete mode 100644 contrib/qpl-cmake/benchmark_sample/database_dir/zstd_s2/config_zstd_s2.xml diff --git a/contrib/qpl-cmake/benchmark_sample/client_scripts/allin1_ssb.sh b/contrib/qpl-cmake/benchmark_sample/client_scripts/allin1_ssb.sh deleted file mode 100644 index 31017b565b6..00000000000 --- a/contrib/qpl-cmake/benchmark_sample/client_scripts/allin1_ssb.sh +++ /dev/null @@ -1,530 +0,0 @@ -#!/bin/bash -ckhost="localhost" -ckport=("9000" "9001" "9002" "9003") -WORKING_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." -OUTPUT_DIR="${WORKING_DIR}/output" -LOG_DIR="${OUTPUT_DIR}/log" -RAWDATA_DIR="${WORKING_DIR}/rawdata_dir" -database_dir="${WORKING_DIR}/database_dir" -CLIENT_SCRIPTS_DIR="${WORKING_DIR}/client_scripts" -LOG_PACK_FILE="$(date +%Y-%m-%d-%H-%M-%S)" -QUERY_FILE="queries_ssb.sql" -SERVER_BIND_CMD[0]="numactl -m 0 -N 0" -SERVER_BIND_CMD[1]="numactl -m 0 -N 0" -SERVER_BIND_CMD[2]="numactl -m 1 -N 1" -SERVER_BIND_CMD[3]="numactl -m 1 -N 1" -CLIENT_BIND_CMD="" -SSB_GEN_FACTOR=20 -TABLE_NAME="lineorder_flat" -TALBE_ROWS="119994608" -CODEC_CONFIG="lz4 deflate zstd" - -# define instance number -inst_num=$1 -if [ ! -n "$1" ]; then - echo "Please clarify instance number from 1,2,3 or 4" - exit 1 -else - echo "Benchmarking with instance number:$1" -fi - -if [ ! -d "$OUTPUT_DIR" ]; then -mkdir $OUTPUT_DIR -fi -if [ ! -d "$LOG_DIR" ]; then -mkdir $LOG_DIR -fi -if [ ! -d "$RAWDATA_DIR" ]; then -mkdir $RAWDATA_DIR -fi - -# define different directories -dir_server=("" "_s2" "_s3" "_s4") -ckreadSql=" - CREATE TABLE customer - ( - C_CUSTKEY UInt32, - C_NAME String, - C_ADDRESS String, - C_CITY LowCardinality(String), - C_NATION LowCardinality(String), - C_REGION LowCardinality(String), - C_PHONE String, - C_MKTSEGMENT LowCardinality(String) - ) - ENGINE = MergeTree ORDER BY (C_CUSTKEY); - - CREATE TABLE lineorder - ( - LO_ORDERKEY UInt32, - LO_LINENUMBER UInt8, - LO_CUSTKEY UInt32, - LO_PARTKEY UInt32, - LO_SUPPKEY UInt32, - LO_ORDERDATE Date, - LO_ORDERPRIORITY LowCardinality(String), - LO_SHIPPRIORITY UInt8, - LO_QUANTITY UInt8, - LO_EXTENDEDPRICE UInt32, - LO_ORDTOTALPRICE UInt32, - LO_DISCOUNT UInt8, - LO_REVENUE UInt32, - LO_SUPPLYCOST UInt32, - LO_TAX UInt8, - LO_COMMITDATE Date, - LO_SHIPMODE LowCardinality(String) - ) - ENGINE = MergeTree PARTITION BY toYear(LO_ORDERDATE) ORDER BY (LO_ORDERDATE, LO_ORDERKEY); - - CREATE TABLE part - ( - P_PARTKEY UInt32, - P_NAME String, - P_MFGR LowCardinality(String), - P_CATEGORY LowCardinality(String), - P_BRAND LowCardinality(String), - P_COLOR LowCardinality(String), - P_TYPE LowCardinality(String), - P_SIZE UInt8, - P_CONTAINER LowCardinality(String) - ) - ENGINE = MergeTree ORDER BY P_PARTKEY; - - CREATE TABLE supplier - ( - S_SUPPKEY UInt32, - S_NAME String, - S_ADDRESS String, - S_CITY LowCardinality(String), - S_NATION LowCardinality(String), - S_REGION LowCardinality(String), - S_PHONE String - ) - ENGINE = MergeTree ORDER BY S_SUPPKEY; -" -supplier_table=" - CREATE TABLE supplier - ( - S_SUPPKEY UInt32, - S_NAME String, - S_ADDRESS String, - S_CITY LowCardinality(String), - S_NATION LowCardinality(String), - S_REGION LowCardinality(String), - S_PHONE String - ) - ENGINE = MergeTree ORDER BY S_SUPPKEY; -" -part_table=" - CREATE TABLE part - ( - P_PARTKEY UInt32, - P_NAME String, - P_MFGR LowCardinality(String), - P_CATEGORY LowCardinality(String), - P_BRAND LowCardinality(String), - P_COLOR LowCardinality(String), - P_TYPE LowCardinality(String), - P_SIZE UInt8, - P_CONTAINER LowCardinality(String) - ) - ENGINE = MergeTree ORDER BY P_PARTKEY; -" -lineorder_table=" - CREATE TABLE lineorder - ( - LO_ORDERKEY UInt32, - LO_LINENUMBER UInt8, - LO_CUSTKEY UInt32, - LO_PARTKEY UInt32, - LO_SUPPKEY UInt32, - LO_ORDERDATE Date, - LO_ORDERPRIORITY LowCardinality(String), - LO_SHIPPRIORITY UInt8, - LO_QUANTITY UInt8, - LO_EXTENDEDPRICE UInt32, - LO_ORDTOTALPRICE UInt32, - LO_DISCOUNT UInt8, - LO_REVENUE UInt32, - LO_SUPPLYCOST UInt32, - LO_TAX UInt8, - LO_COMMITDATE Date, - LO_SHIPMODE LowCardinality(String) - ) - ENGINE = MergeTree PARTITION BY toYear(LO_ORDERDATE) ORDER BY (LO_ORDERDATE, LO_ORDERKEY); -" -customer_table=" - CREATE TABLE customer - ( - C_CUSTKEY UInt32, - C_NAME String, - C_ADDRESS String, - C_CITY LowCardinality(String), - C_NATION LowCardinality(String), - C_REGION LowCardinality(String), - C_PHONE String, - C_MKTSEGMENT LowCardinality(String) - ) - ENGINE = MergeTree ORDER BY (C_CUSTKEY); -" - -lineorder_flat_table=" - SET max_memory_usage = 20000000000; - CREATE TABLE lineorder_flat - ENGINE = MergeTree - PARTITION BY toYear(LO_ORDERDATE) - ORDER BY (LO_ORDERDATE, LO_ORDERKEY) AS - SELECT - l.LO_ORDERKEY AS LO_ORDERKEY, - l.LO_LINENUMBER AS LO_LINENUMBER, - l.LO_CUSTKEY AS LO_CUSTKEY, - l.LO_PARTKEY AS LO_PARTKEY, - l.LO_SUPPKEY AS LO_SUPPKEY, - l.LO_ORDERDATE AS LO_ORDERDATE, - l.LO_ORDERPRIORITY AS LO_ORDERPRIORITY, - l.LO_SHIPPRIORITY AS LO_SHIPPRIORITY, - l.LO_QUANTITY AS LO_QUANTITY, - l.LO_EXTENDEDPRICE AS LO_EXTENDEDPRICE, - l.LO_ORDTOTALPRICE AS LO_ORDTOTALPRICE, - l.LO_DISCOUNT AS LO_DISCOUNT, - l.LO_REVENUE AS LO_REVENUE, - l.LO_SUPPLYCOST AS LO_SUPPLYCOST, - l.LO_TAX AS LO_TAX, - l.LO_COMMITDATE AS LO_COMMITDATE, - l.LO_SHIPMODE AS LO_SHIPMODE, - c.C_NAME AS C_NAME, - c.C_ADDRESS AS C_ADDRESS, - c.C_CITY AS C_CITY, - c.C_NATION AS C_NATION, - c.C_REGION AS C_REGION, - c.C_PHONE AS C_PHONE, - c.C_MKTSEGMENT AS C_MKTSEGMENT, - s.S_NAME AS S_NAME, - s.S_ADDRESS AS S_ADDRESS, - s.S_CITY AS S_CITY, - s.S_NATION AS S_NATION, - s.S_REGION AS S_REGION, - s.S_PHONE AS S_PHONE, - p.P_NAME AS P_NAME, - p.P_MFGR AS P_MFGR, - p.P_CATEGORY AS P_CATEGORY, - p.P_BRAND AS P_BRAND, - p.P_COLOR AS P_COLOR, - p.P_TYPE AS P_TYPE, - p.P_SIZE AS P_SIZE, - p.P_CONTAINER AS P_CONTAINER - FROM lineorder AS l - INNER JOIN customer AS c ON c.C_CUSTKEY = l.LO_CUSTKEY - INNER JOIN supplier AS s ON s.S_SUPPKEY = l.LO_SUPPKEY - INNER JOIN part AS p ON p.P_PARTKEY = l.LO_PARTKEY; - show settings ilike 'max_memory_usage'; -" - -function insert_data(){ - echo "insert_data:$1" - create_table_prefix="clickhouse client --host ${ckhost} --port $2 --multiquery -q" - insert_data_prefix="clickhouse client --query " - case $1 in - all) - clickhouse client --host ${ckhost} --port $2 --multiquery -q"$ckreadSql" && { - ${insert_data_prefix} "INSERT INTO customer FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/customer.tbl --port=$2 - ${insert_data_prefix} "INSERT INTO part FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/part.tbl --port=$2 - ${insert_data_prefix} "INSERT INTO supplier FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/supplier.tbl --port=$2 - ${insert_data_prefix} "INSERT INTO lineorder FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/lineorder.tbl --port=$2 - } - ${create_table_prefix}"${lineorder_flat_table}" - ;; - customer) - echo ${create_table_prefix}\"${customer_table}\" - ${create_table_prefix}"${customer_table}" && { - echo "${insert_data_prefix} \"INSERT INTO $1 FORMAT CSV\" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2" - ${insert_data_prefix} "INSERT INTO $1 FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2 - } - ;; - part) - echo ${create_table_prefix}\"${part_table}\" - ${create_table_prefix}"${part_table}" && { - echo "${insert_data_prefix} \"INSERT INTO $1 FORMAT CSV\" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2" - ${insert_data_prefix} "INSERT INTO $1 FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2 - } - ;; - supplier) - echo ${create_table_prefix}"${supplier_table}" - ${create_table_prefix}"${supplier_table}" && { - echo "${insert_data_prefix} \"INSERT INTO $1 FORMAT CSV\" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2" - ${insert_data_prefix} "INSERT INTO $1 FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2 - } - ;; - lineorder) - echo ${create_table_prefix}"${lineorder_table}" - ${create_table_prefix}"${lineorder_table}" && { - echo "${insert_data_prefix} \"INSERT INTO $1 FORMAT CSV\" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2" - ${insert_data_prefix} "INSERT INTO $1 FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2 - } - ;; - lineorder_flat) - echo ${create_table_prefix}"${lineorder_flat_table}" - ${create_table_prefix}"${lineorder_flat_table}" - return 0 - ;; - *) - exit 0 - ;; - - esac -} - -function check_sql(){ - select_sql="select * from "$1" limit 1" - clickhouse client --host ${ckhost} --port $2 --multiquery -q"${select_sql}" -} - -function check_table(){ - checknum=0 - source_tables="customer part supplier lineorder lineorder_flat" - test_tables=${1:-${source_tables}} - echo "Checking table data required in server..." - for i in $(seq 0 $[inst_num-1]) - do - for j in `echo ${test_tables}` - do - check_sql $j ${ckport[i]} &> /dev/null || { - let checknum+=1 && insert_data "$j" ${ckport[i]} - } - done - done - - for i in $(seq 0 $[inst_num-1]) - do - echo "clickhouse client --host ${ckhost} --port ${ckport[i]} -m -q\"select count() from ${TABLE_NAME};\"" - var=$(clickhouse client --host ${ckhost} --port ${ckport[i]} -m -q"select count() from ${TABLE_NAME};") - if [ $var -eq $TALBE_ROWS ];then - echo "Instance_${i} Table data integrity check OK -> Rows:$var" - else - echo "Instance_${i} Table data integrity check Failed -> Rows:$var" - exit 1 - fi - done - if [ $checknum -gt 0 ];then - echo "Need sleep 10s after first table data insertion...$checknum" - sleep 10 - fi -} - -function check_instance(){ -instance_alive=0 -for i in {1..10} -do - sleep 1 - netstat -nltp | grep ${1} > /dev/null - if [ $? -ne 1 ];then - instance_alive=1 - break - fi - -done - -if [ $instance_alive -eq 0 ];then - echo "check_instance -> clickhouse server instance faild to launch due to 10s timeout!" - exit 1 -else - echo "check_instance -> clickhouse server instance launch successfully!" -fi -} - -function start_clickhouse_for_insertion(){ - echo "start_clickhouse_for_insertion" - for i in $(seq 0 $[inst_num-1]) - do - echo "cd ${database_dir}/$1${dir_server[i]}" - echo "${SERVER_BIND_CMD[i]} clickhouse server -C config_${1}${dir_server[i]}.xml >&${LOG_DIR}/${1}_${i}_server_log& > /dev/null" - - cd ${database_dir}/$1${dir_server[i]} - ${SERVER_BIND_CMD[i]} clickhouse server -C config_${1}${dir_server[i]}.xml >&${LOG_DIR}/${1}_${i}_server_log& > /dev/null - check_instance ${ckport[i]} - done -} - -function start_clickhouse_for_stressing(){ - echo "start_clickhouse_for_stressing" - for i in $(seq 0 $[inst_num-1]) - do - echo "cd ${database_dir}/$1${dir_server[i]}" - echo "${SERVER_BIND_CMD[i]} clickhouse server -C config_${1}${dir_server[i]}.xml >&/dev/null&" - - cd ${database_dir}/$1${dir_server[i]} - ${SERVER_BIND_CMD[i]} clickhouse server -C config_${1}${dir_server[i]}.xml >&/dev/null& - check_instance ${ckport[i]} - done -} -yum -y install git make gcc sudo net-tools &> /dev/null -pip3 install clickhouse_driver numpy &> /dev/null -test -d ${RAWDATA_DIR}/ssb-dbgen || git clone https://github.com/vadimtk/ssb-dbgen.git ${RAWDATA_DIR}/ssb-dbgen && cd ${RAWDATA_DIR}/ssb-dbgen - -if [ ! -f ${RAWDATA_DIR}/ssb-dbgen/dbgen ];then - make && { - test -f ${RAWDATA_DIR}/ssb-dbgen/customer.tbl || echo y |./dbgen -s ${SSB_GEN_FACTOR} -T c - test -f ${RAWDATA_DIR}/ssb-dbgen/part.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T p - test -f ${RAWDATA_DIR}/ssb-dbgen/supplier.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T s - test -f ${RAWDATA_DIR}/ssb-dbgen/date.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T d - test -f ${RAWDATA_DIR}/ssb-dbgen/lineorder.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T l - } -else - test -f ${RAWDATA_DIR}/ssb-dbgen/customer.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T c - test -f ${RAWDATA_DIR}/ssb-dbgen/part.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T p - test -f ${RAWDATA_DIR}/ssb-dbgen/supplier.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T s - test -f ${RAWDATA_DIR}/ssb-dbgen/date.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T d - test -f ${RAWDATA_DIR}/ssb-dbgen/lineorder.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T l - -fi - -filenum=`find ${RAWDATA_DIR}/ssb-dbgen/ -name "*.tbl" | wc -l` - -if [ $filenum -ne 5 ];then - echo "generate ssb data file *.tbl faild" - exit 1 -fi - -function kill_instance(){ -instance_alive=1 -for i in {1..2} -do - pkill clickhouse && sleep 5 - instance_alive=0 - for i in $(seq 0 $[inst_num-1]) - do - netstat -nltp | grep ${ckport[i]} > /dev/null - if [ $? -ne 1 ];then - instance_alive=1 - break; - fi - done - if [ $instance_alive -eq 0 ];then - break; - fi -done -if [ $instance_alive -eq 0 ];then - echo "kill_instance OK!" -else - echo "kill_instance Failed -> clickhouse server instance still alive due to 10s timeout" - exit 1 -fi -} - -function run_test(){ -is_xml=0 -for i in $(seq 0 $[inst_num-1]) -do - if [ -f ${database_dir}/${1}${dir_server[i]}/config_${1}${dir_server[i]}.xml ]; then - is_xml=$[is_xml+1] - fi -done -if [ $is_xml -eq $inst_num ];then - echo "Benchmark with $inst_num instance" - start_clickhouse_for_insertion ${1} - - for i in $(seq 0 $[inst_num-1]) - do - clickhouse client --host ${ckhost} --port ${ckport[i]} -m -q"show databases;" >/dev/null - done - - if [ $? -eq 0 ];then - check_table - fi - kill_instance - - if [ $1 == "deflate" ];then - test -f ${LOG_DIR}/${1}_server_log && deflatemsg=`cat ${LOG_DIR}/${1}_server_log | grep DeflateJobHWPool` - if [ -n "$deflatemsg" ];then - echo ------------------------------------------------------ - echo $deflatemsg - echo ------------------------------------------------------ - fi - fi - echo "Check table data required in server_${1} -> Done! " - - start_clickhouse_for_stressing ${1} - for i in $(seq 0 $[inst_num-1]) - do - clickhouse client --host ${ckhost} --port ${ckport[i]} -m -q"show databases;" >/dev/null - done - if [ $? -eq 0 ];then - test -d ${CLIENT_SCRIPTS_DIR} && cd ${CLIENT_SCRIPTS_DIR} - echo "Client stressing... " - echo "${CLIENT_BIND_CMD} python3 client_stressing_test.py ${QUERY_FILE} $inst_num &> ${LOG_DIR}/${1}.log" - ${CLIENT_BIND_CMD} python3 client_stressing_test.py ${QUERY_FILE} $inst_num &> ${LOG_DIR}/${1}.log - echo "Completed client stressing, checking log... " - finish_log=`grep "Finished" ${LOG_DIR}/${1}.log | wc -l` - if [ $finish_log -eq 1 ] ;then - kill_instance - test -f ${LOG_DIR}/${1}.log && echo "${1}.log ===> ${LOG_DIR}/${1}.log" - else - kill_instance - echo "No find 'Finished' in client log -> Performance test may fail" - exit 1 - - fi - - else - echo "${1} clickhouse server start fail" - exit 1 - fi -else - echo "clickhouse server start fail -> Please check xml files required in ${database_dir} for each instance" - exit 1 - -fi -} -function clear_log(){ - if [ -d "$LOG_DIR" ]; then - cd ${LOG_DIR} && rm -rf * - fi -} - -function gather_log_for_codec(){ - cd ${OUTPUT_DIR} && mkdir -p ${LOG_PACK_FILE}/${1} - cp -rf ${LOG_DIR} ${OUTPUT_DIR}/${LOG_PACK_FILE}/${1} -} - -function pack_log(){ - if [ -e "${OUTPUT_DIR}/run.log" ]; then - cp ${OUTPUT_DIR}/run.log ${OUTPUT_DIR}/${LOG_PACK_FILE}/ - fi - echo "Please check all log information in ${OUTPUT_DIR}/${LOG_PACK_FILE}" -} - -function setup_check(){ - - iax_dev_num=`accel-config list | grep iax | wc -l` - if [ $iax_dev_num -eq 0 ] ;then - iax_dev_num=`accel-config list | grep iax | wc -l` - if [ $iax_dev_num -eq 0 ] ;then - echo "No IAA devices available -> Please check IAA hardware setup manually!" - exit 1 - else - echo "IAA enabled devices number:$iax_dev_num" - fi - else - echo "IAA enabled devices number:$iax_dev_num" - fi - libaccel_version=`accel-config -v` - clickhouser_version=`clickhouse server --version` - kernel_dxd_log=`dmesg | grep dxd` - echo "libaccel_version:$libaccel_version" - echo "clickhouser_version:$clickhouser_version" - echo -e "idxd section in kernel log:\n$kernel_dxd_log" -} - -setup_check -export CLICKHOUSE_WATCHDOG_ENABLE=0 -for i in ${CODEC_CONFIG[@]} -do - clear_log - codec=${i} - echo "run test------------$codec" - run_test $codec - gather_log_for_codec $codec -done - -pack_log -echo "Done." \ No newline at end of file diff --git a/contrib/qpl-cmake/benchmark_sample/client_scripts/client_stressing_test.py b/contrib/qpl-cmake/benchmark_sample/client_scripts/client_stressing_test.py deleted file mode 100644 index f12381a198c..00000000000 --- a/contrib/qpl-cmake/benchmark_sample/client_scripts/client_stressing_test.py +++ /dev/null @@ -1,278 +0,0 @@ -from operator import eq -import os -import random -import time -import sys -from clickhouse_driver import Client -import numpy as np -import subprocess -import multiprocessing -from multiprocessing import Manager - -warmup_runs = 10 -calculated_runs = 10 -seconds = 30 -max_instances_number = 8 -retest_number = 3 -retest_tolerance = 10 - - -def checkInt(str): - try: - int(str) - return True - except ValueError: - return False - - -def setup_client(index): - if index < 4: - port_idx = index - else: - port_idx = index + 4 - client = Client( - host="localhost", - database="default", - user="default", - password="", - port="900%d" % port_idx, - ) - union_mode_query = "SET union_default_mode='DISTINCT'" - client.execute(union_mode_query) - return client - - -def warm_client(clientN, clientL, query, loop): - for c_idx in range(clientN): - for _ in range(loop): - clientL[c_idx].execute(query) - - -def read_queries(queries_list): - queries = list() - queries_id = list() - with open(queries_list, "r") as f: - for line in f: - line = line.rstrip() - line = line.split("$") - queries_id.append(line[0]) - queries.append(line[1]) - return queries_id, queries - - -def run_task(client, cname, query, loop, query_latency): - start_time = time.time() - for i in range(loop): - client.execute(query) - query_latency.append(client.last_query.elapsed) - - end_time = time.time() - p95 = np.percentile(query_latency, 95) - print( - "CLIENT: {0} end. -> P95: %f, qps: %f".format(cname) - % (p95, loop / (end_time - start_time)) - ) - - -def run_multi_clients(clientN, clientList, query, loop): - client_pids = {} - start_time = time.time() - manager = multiprocessing.Manager() - query_latency_list0 = manager.list() - query_latency_list1 = manager.list() - query_latency_list2 = manager.list() - query_latency_list3 = manager.list() - query_latency_list4 = manager.list() - query_latency_list5 = manager.list() - query_latency_list6 = manager.list() - query_latency_list7 = manager.list() - - for c_idx in range(clientN): - client_name = "Role_%d" % c_idx - if c_idx == 0: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list0), - ) - elif c_idx == 1: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list1), - ) - elif c_idx == 2: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list2), - ) - elif c_idx == 3: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list3), - ) - elif c_idx == 4: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list4), - ) - elif c_idx == 5: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list5), - ) - elif c_idx == 6: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list6), - ) - elif c_idx == 7: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list7), - ) - else: - print("ERROR: CLIENT number dismatch!!") - exit() - print("CLIENT: %s start" % client_name) - client_pids[c_idx].start() - - for c_idx in range(clientN): - client_pids[c_idx].join() - end_time = time.time() - totalT = end_time - start_time - - query_latencyTotal = list() - for item in query_latency_list0: - query_latencyTotal.append(item) - for item in query_latency_list1: - query_latencyTotal.append(item) - for item in query_latency_list2: - query_latencyTotal.append(item) - for item in query_latency_list3: - query_latencyTotal.append(item) - for item in query_latency_list4: - query_latencyTotal.append(item) - for item in query_latency_list5: - query_latencyTotal.append(item) - for item in query_latency_list6: - query_latencyTotal.append(item) - for item in query_latency_list7: - query_latencyTotal.append(item) - - totalP95 = np.percentile(query_latencyTotal, 95) * 1000 - return totalT, totalP95 - - -def run_task_caculated(client, cname, query, loop): - query_latency = list() - start_time = time.time() - for i in range(loop): - client.execute(query) - query_latency.append(client.last_query.elapsed) - end_time = time.time() - p95 = np.percentile(query_latency, 95) - - -def run_multi_clients_caculated(clientN, clientList, query, loop): - client_pids = {} - start_time = time.time() - for c_idx in range(clientN): - client_name = "Role_%d" % c_idx - client_pids[c_idx] = multiprocessing.Process( - target=run_task_caculated, - args=(clientList[c_idx], client_name, query, loop), - ) - client_pids[c_idx].start() - for c_idx in range(clientN): - client_pids[c_idx].join() - end_time = time.time() - totalT = end_time - start_time - return totalT - - -if __name__ == "__main__": - client_number = 1 - queries = list() - queries_id = list() - - if len(sys.argv) != 3: - print( - "usage: python3 client_stressing_test.py [queries_file_path] [client_number]" - ) - sys.exit() - else: - queries_list = sys.argv[1] - client_number = int(sys.argv[2]) - print( - "queries_file_path: %s, client_number: %d" % (queries_list, client_number) - ) - if not os.path.isfile(queries_list) or not os.access(queries_list, os.R_OK): - print("please check the right path for queries file") - sys.exit() - if ( - not checkInt(sys.argv[2]) - or int(sys.argv[2]) > max_instances_number - or int(sys.argv[2]) < 1 - ): - print("client_number should be in [1~%d]" % max_instances_number) - sys.exit() - - client_list = {} - queries_id, queries = read_queries(queries_list) - - for c_idx in range(client_number): - client_list[c_idx] = setup_client(c_idx) - # clear cache - os.system("sync; echo 3 > /proc/sys/vm/drop_caches") - - print("###Polit Run Begin") - for i in queries: - warm_client(client_number, client_list, i, 1) - print("###Polit Run End -> Start stressing....") - - query_index = 0 - for q in queries: - print( - "\n###START -> Index: %d, ID: %s, Query: %s" - % (query_index, queries_id[query_index], q) - ) - warm_client(client_number, client_list, q, warmup_runs) - print("###Warm Done!") - for j in range(0, retest_number): - totalT = run_multi_clients_caculated( - client_number, client_list, q, calculated_runs - ) - curr_loop = int(seconds * calculated_runs / totalT) + 1 - print( - "###Calculation Done! -> loopN: %d, expected seconds:%d" - % (curr_loop, seconds) - ) - - print("###Stress Running! -> %d iterations......" % curr_loop) - - totalT, totalP95 = run_multi_clients( - client_number, client_list, q, curr_loop - ) - - if totalT > (seconds - retest_tolerance) and totalT < ( - seconds + retest_tolerance - ): - break - else: - print( - "###totalT:%d is far way from expected seconds:%d. Run again ->j:%d!" - % (totalT, seconds, j) - ) - - print( - "###Completed! -> ID: %s, clientN: %d, totalT: %.2f s, latencyAVG: %.2f ms, P95: %.2f ms, QPS_Final: %.2f" - % ( - queries_id[query_index], - client_number, - totalT, - totalT * 1000 / (curr_loop * client_number), - totalP95, - ((curr_loop * client_number) / totalT), - ) - ) - query_index += 1 - print("###Finished!") diff --git a/contrib/qpl-cmake/benchmark_sample/client_scripts/queries_ssb.sql b/contrib/qpl-cmake/benchmark_sample/client_scripts/queries_ssb.sql deleted file mode 100644 index abf2df6503a..00000000000 --- a/contrib/qpl-cmake/benchmark_sample/client_scripts/queries_ssb.sql +++ /dev/null @@ -1,10 +0,0 @@ -Q1.1$SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE toYear(LO_ORDERDATE) = 1993 AND LO_DISCOUNT BETWEEN 1 AND 3 AND LO_QUANTITY < 25; -Q2.1$SELECT sum(LO_REVENUE),toYear(LO_ORDERDATE) AS year,P_BRAND FROM lineorder_flat WHERE P_CATEGORY = 'MFGR#12' AND S_REGION = 'AMERICA' GROUP BY year,P_BRAND ORDER BY year,P_BRAND; -Q2.2$SELECT sum(LO_REVENUE),toYear(LO_ORDERDATE) AS year,P_BRAND FROM lineorder_flat WHERE P_BRAND >= 'MFGR#2221' AND P_BRAND <= 'MFGR#2228' AND S_REGION = 'ASIA' GROUP BY year,P_BRAND ORDER BY year,P_BRAND; -Q2.3$SELECT sum(LO_REVENUE),toYear(LO_ORDERDATE) AS year,P_BRAND FROM lineorder_flat WHERE P_BRAND = 'MFGR#2239' AND S_REGION = 'EUROPE' GROUP BY year,P_BRAND ORDER BY year,P_BRAND; -Q3.1$SELECT C_NATION,S_NATION,toYear(LO_ORDERDATE) AS year,sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE C_REGION = 'ASIA' AND S_REGION = 'ASIA' AND year >= 1992 AND year <= 1997 GROUP BY C_NATION,S_NATION,year ORDER BY year ASC,revenue DESC; -Q3.2$SELECT C_CITY,S_CITY,toYear(LO_ORDERDATE) AS year,sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE C_NATION = 'UNITED STATES' AND S_NATION = 'UNITED STATES' AND year >= 1992 AND year <= 1997 GROUP BY C_CITY,S_CITY,year ORDER BY year ASC,revenue DESC; -Q3.3$SELECT C_CITY,S_CITY,toYear(LO_ORDERDATE) AS year,sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE (C_CITY = 'UNITED KI1' OR C_CITY = 'UNITED KI5') AND (S_CITY = 'UNITED KI1' OR S_CITY = 'UNITED KI5') AND year >= 1992 AND year <= 1997 GROUP BY C_CITY,S_CITY,year ORDER BY year ASC,revenue DESC; -Q4.1$SELECT toYear(LO_ORDERDATE) AS year,C_NATION,sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') GROUP BY year,C_NATION ORDER BY year ASC,C_NATION ASC; -Q4.2$SELECT toYear(LO_ORDERDATE) AS year,S_NATION,P_CATEGORY,sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (year = 1997 OR year = 1998) AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') GROUP BY year,S_NATION,P_CATEGORY ORDER BY year ASC,S_NATION ASC,P_CATEGORY ASC; -Q4.3$SELECT toYear(LO_ORDERDATE) AS year,S_CITY,P_BRAND,sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE S_NATION = 'UNITED STATES' AND (year = 1997 OR year = 1998) AND P_CATEGORY = 'MFGR#14' GROUP BY year,S_CITY,P_BRAND ORDER BY year ASC,S_CITY ASC,P_BRAND ASC; diff --git a/contrib/qpl-cmake/benchmark_sample/client_scripts/run_ssb.sh b/contrib/qpl-cmake/benchmark_sample/client_scripts/run_ssb.sh deleted file mode 100644 index 6067b1058f2..00000000000 --- a/contrib/qpl-cmake/benchmark_sample/client_scripts/run_ssb.sh +++ /dev/null @@ -1,6 +0,0 @@ -WORKING_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." -if [ ! -d "${WORKING_DIR}/output" ]; then -mkdir ${WORKING_DIR}/output -fi -bash allin1_ssb.sh 2 > ${WORKING_DIR}/output/run.log -echo "Please check log in: ${WORKING_DIR}/output/run.log" \ No newline at end of file diff --git a/contrib/qpl-cmake/benchmark_sample/database_dir/deflate/config_deflate.xml b/contrib/qpl-cmake/benchmark_sample/database_dir/deflate/config_deflate.xml deleted file mode 100644 index ab77a9cdcbe..00000000000 --- a/contrib/qpl-cmake/benchmark_sample/database_dir/deflate/config_deflate.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - trace - true - - - 8123 - 9000 - 9004 - - ./ - - 8589934592 - 5368709120 - true - - - - deflate_qpl - - - - - - - - - ::/0 - - - default - default - 1 - - - - - - - - - - - diff --git a/contrib/qpl-cmake/benchmark_sample/database_dir/deflate_s2/config_deflate_s2.xml b/contrib/qpl-cmake/benchmark_sample/database_dir/deflate_s2/config_deflate_s2.xml deleted file mode 100644 index b71456486f5..00000000000 --- a/contrib/qpl-cmake/benchmark_sample/database_dir/deflate_s2/config_deflate_s2.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - trace - true - - - 8124 - 9001 - 9005 - - ./ - - 8589934592 - 5368709120 - true - - - - deflate_qpl - - - - - - - - - ::/0 - - - default - default - 1 - - - - - - - - - - - diff --git a/contrib/qpl-cmake/benchmark_sample/database_dir/lz4/config_lz4.xml b/contrib/qpl-cmake/benchmark_sample/database_dir/lz4/config_lz4.xml deleted file mode 100644 index f4dc59b60aa..00000000000 --- a/contrib/qpl-cmake/benchmark_sample/database_dir/lz4/config_lz4.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - trace - true - - - 8123 - 9000 - 9004 - - ./ - - 8589934592 - 5368709120 - true - - - - lz4 - - - - - - - - - ::/0 - - - default - default - 1 - - - - - - - - - - - diff --git a/contrib/qpl-cmake/benchmark_sample/database_dir/lz4_s2/config_lz4_s2.xml b/contrib/qpl-cmake/benchmark_sample/database_dir/lz4_s2/config_lz4_s2.xml deleted file mode 100644 index 357db8942d7..00000000000 --- a/contrib/qpl-cmake/benchmark_sample/database_dir/lz4_s2/config_lz4_s2.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - trace - true - - - 8124 - 9001 - 9005 - - ./ - - 8589934592 - 5368709120 - true - - - - lz4 - - - - - - - - - ::/0 - - - default - default - 1 - - - - - - - - - - - diff --git a/contrib/qpl-cmake/benchmark_sample/database_dir/zstd/config_zstd.xml b/contrib/qpl-cmake/benchmark_sample/database_dir/zstd/config_zstd.xml deleted file mode 100644 index 1c4c738edaf..00000000000 --- a/contrib/qpl-cmake/benchmark_sample/database_dir/zstd/config_zstd.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - trace - true - - - 8123 - 9000 - 9004 - - ./ - - 8589934592 - 5368709120 - true - - - - zstd - - - - - - - - - ::/0 - - - default - default - 1 - - - - - - - - - - - diff --git a/contrib/qpl-cmake/benchmark_sample/database_dir/zstd_s2/config_zstd_s2.xml b/contrib/qpl-cmake/benchmark_sample/database_dir/zstd_s2/config_zstd_s2.xml deleted file mode 100644 index f3db01b7739..00000000000 --- a/contrib/qpl-cmake/benchmark_sample/database_dir/zstd_s2/config_zstd_s2.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - trace - true - - - 8124 - 9001 - 9005 - - ./ - - 8589934592 - 5368709120 - true - - - - zstd - - - - - - - - - ::/0 - - - default - default - 1 - - - - - - - - - - - From c65607484e51a5e9aa8f59612e7817b899bb88ee Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 10:03:10 +0100 Subject: [PATCH 126/192] Remove garbage --- contrib/qpl-cmake/CMakeLists.txt | 464 ++++++++++++++++++++++++++++--- docker/packager/binary/build.sh | 2 +- 2 files changed, 419 insertions(+), 47 deletions(-) diff --git a/contrib/qpl-cmake/CMakeLists.txt b/contrib/qpl-cmake/CMakeLists.txt index 19501209b26..7a84048e16b 100644 --- a/contrib/qpl-cmake/CMakeLists.txt +++ b/contrib/qpl-cmake/CMakeLists.txt @@ -27,16 +27,422 @@ message(STATUS "Intel QPL version: ${QPL_VERSION}") # The qpl submodule comes with its own version of isal. It contains code which does not exist in upstream isal. It would be nice to link # only upstream isal (ch_contrib::isal) but at this point we can't. -include("${QPL_PROJECT_DIR}/cmake/CompileOptions.cmake") +# ========================================================================== +# Copyright (C) 2022 Intel Corporation +# +# SPDX-License-Identifier: MIT +# ========================================================================== + +set(QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS "-fno-exceptions;-fno-rtti") + +function(modify_standard_language_flag) + # Declaring function parameters + set(OPTIONS "") + set(ONE_VALUE_ARGS + LANGUAGE_NAME + FLAG_NAME + NEW_FLAG_VALUE) + set(MULTI_VALUE_ARGS "") + + # Parsing function parameters + cmake_parse_arguments(MODIFY + "${OPTIONS}" + "${ONE_VALUE_ARGS}" + "${MULTI_VALUE_ARGS}" + ${ARGN}) + + # Variables + set(FLAG_REGULAR_EXPRESSION "${MODIFY_FLAG_NAME}.*[ ]*") + set(NEW_VALUE "${MODIFY_FLAG_NAME}${MODIFY_NEW_FLAG_VALUE}") + + # Replacing specified flag with new value + string(REGEX REPLACE + ${FLAG_REGULAR_EXPRESSION} ${NEW_VALUE} + NEW_COMPILE_FLAGS + "${CMAKE_${MODIFY_LANGUAGE_NAME}_FLAGS}") + + # Returning the value + set(CMAKE_${MODIFY_LANGUAGE_NAME}_FLAGS ${NEW_COMPILE_FLAGS} PARENT_SCOPE) +endfunction() + +function(get_function_name_with_default_bit_width in_function_name bit_width out_function_name) + + if(in_function_name MATCHES ".*_i") + + string(REPLACE "_i" "" in_function_name ${in_function_name}) + + set(${out_function_name} "${in_function_name}_${bit_width}_i" PARENT_SCOPE) + + else() + + set(${out_function_name} "${in_function_name}_${bit_width}" PARENT_SCOPE) + + endif() + +endfunction() + +macro(get_list_of_supported_optimizations PLATFORMS_LIST) + list(APPEND PLATFORMS_LIST "") + list(APPEND PLATFORMS_LIST "px") + list(APPEND PLATFORMS_LIST "avx512") +endmacro(get_list_of_supported_optimizations) + +function(generate_unpack_kernel_arrays current_directory PLATFORMS_LIST) + list(APPEND UNPACK_POSTFIX_LIST "") + list(APPEND UNPACK_PRLE_POSTFIX_LIST "") + list(APPEND PACK_POSTFIX_LIST "") + list(APPEND PACK_INDEX_POSTFIX_LIST "") + list(APPEND SCAN_POSTFIX_LIST "") + list(APPEND DEFAULT_BIT_WIDTH_FUNCTIONS_LIST "") + list(APPEND DEFAULT_BIT_WIDTH_LIST "") + + #create list of functions that use only 8u 16u 32u postfixes + list(APPEND DEFAULT_BIT_WIDTH_FUNCTIONS_LIST "unpack_prle") + list(APPEND DEFAULT_BIT_WIDTH_FUNCTIONS_LIST "extract") + list(APPEND DEFAULT_BIT_WIDTH_FUNCTIONS_LIST "extract_i") + list(APPEND DEFAULT_BIT_WIDTH_FUNCTIONS_LIST "select") + list(APPEND DEFAULT_BIT_WIDTH_FUNCTIONS_LIST "select_i") + list(APPEND DEFAULT_BIT_WIDTH_FUNCTIONS_LIST "expand") + + #create default bit width list + list(APPEND DEFAULT_BIT_WIDTH_LIST "8u") + list(APPEND DEFAULT_BIT_WIDTH_LIST "16u") + list(APPEND DEFAULT_BIT_WIDTH_LIST "32u") + + #create scan kernel postfixes + list(APPEND SCAN_COMPARATOR_LIST "") + + list(APPEND SCAN_COMPARATOR_LIST "eq") + list(APPEND SCAN_COMPARATOR_LIST "ne") + list(APPEND SCAN_COMPARATOR_LIST "lt") + list(APPEND SCAN_COMPARATOR_LIST "le") + list(APPEND SCAN_COMPARATOR_LIST "gt") + list(APPEND SCAN_COMPARATOR_LIST "ge") + list(APPEND SCAN_COMPARATOR_LIST "range") + list(APPEND SCAN_COMPARATOR_LIST "not_range") + + foreach(SCAN_COMPARATOR IN LISTS SCAN_COMPARATOR_LIST) + list(APPEND SCAN_POSTFIX_LIST "_${SCAN_COMPARATOR}_8u") + list(APPEND SCAN_POSTFIX_LIST "_${SCAN_COMPARATOR}_16u8u") + list(APPEND SCAN_POSTFIX_LIST "_${SCAN_COMPARATOR}_32u8u") + endforeach() + + # create unpack kernel postfixes + foreach(input_width RANGE 1 32 1) + if(input_width LESS 8 OR input_width EQUAL 8) + list(APPEND UNPACK_POSTFIX_LIST "_${input_width}u8u") + + elseif(input_width LESS 16 OR input_width EQUAL 16) + list(APPEND UNPACK_POSTFIX_LIST "_${input_width}u16u") + + else() + list(APPEND UNPACK_POSTFIX_LIST "_${input_width}u32u") + endif() + endforeach() + + # create pack kernel postfixes + foreach(output_width RANGE 1 8 1) + list(APPEND PACK_POSTFIX_LIST "_8u${output_width}u") + endforeach() + + foreach(output_width RANGE 9 16 1) + list(APPEND PACK_POSTFIX_LIST "_16u${output_width}u") + endforeach() + + foreach(output_width RANGE 17 32 1) + list(APPEND PACK_POSTFIX_LIST "_32u${output_width}u") + endforeach() + + list(APPEND PACK_POSTFIX_LIST "_8u16u") + list(APPEND PACK_POSTFIX_LIST "_8u32u") + list(APPEND PACK_POSTFIX_LIST "_16u32u") + + # create pack index kernel postfixes + list(APPEND PACK_INDEX_POSTFIX_LIST "_nu") + list(APPEND PACK_INDEX_POSTFIX_LIST "_8u") + list(APPEND PACK_INDEX_POSTFIX_LIST "_8u16u") + list(APPEND PACK_INDEX_POSTFIX_LIST "_8u32u") + + # write to file + file(MAKE_DIRECTORY ${current_directory}/generated) + + foreach(PLATFORM_VALUE IN LISTS PLATFORMS_LIST) + set(directory "${current_directory}/generated") + set(PLATFORM_PREFIX "${PLATFORM_VALUE}_") + + # + # Write unpack table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}unpack.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}unpack.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}unpack.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}unpack.cpp "unpack_table_t ${PLATFORM_PREFIX}unpack_table = {\n") + + #write LE kernels + foreach(UNPACK_POSTFIX IN LISTS UNPACK_POSTFIX_LIST) + file(APPEND ${directory}/${PLATFORM_PREFIX}unpack.cpp "\t${PLATFORM_PREFIX}qplc_unpack${UNPACK_POSTFIX},\n") + endforeach() + + #write BE kernels + + #get last element of the list + set(LAST_ELEMENT "") + list(GET UNPACK_POSTFIX_LIST -1 LAST_ELEMENT) + + foreach(UNPACK_POSTFIX IN LISTS UNPACK_POSTFIX_LIST) + + if(UNPACK_POSTFIX STREQUAL LAST_ELEMENT) + file(APPEND ${directory}/${PLATFORM_PREFIX}unpack.cpp "\t${PLATFORM_PREFIX}qplc_unpack_be${UNPACK_POSTFIX}};\n") + else() + file(APPEND ${directory}/${PLATFORM_PREFIX}unpack.cpp "\t${PLATFORM_PREFIX}qplc_unpack_be${UNPACK_POSTFIX},\n") + endif() + endforeach() + + file(APPEND ${directory}/${PLATFORM_PREFIX}unpack.cpp "}\n") + + # + # Write pack table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}pack.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack.cpp "pack_table_t ${PLATFORM_PREFIX}pack_table = {\n") + + #write LE kernels + foreach(PACK_POSTFIX IN LISTS PACK_POSTFIX_LIST) + file(APPEND ${directory}/${PLATFORM_PREFIX}pack.cpp "\t${PLATFORM_PREFIX}qplc_pack${PACK_POSTFIX},\n") + endforeach() + + #write BE kernels + + #get last element of the list + set(LAST_ELEMENT "") + list(GET PACK_POSTFIX_LIST -1 LAST_ELEMENT) + + foreach(PACK_POSTFIX IN LISTS PACK_POSTFIX_LIST) + + if(PACK_POSTFIX STREQUAL LAST_ELEMENT) + file(APPEND ${directory}/${PLATFORM_PREFIX}pack.cpp "\t${PLATFORM_PREFIX}qplc_pack_be${PACK_POSTFIX}};\n") + else() + file(APPEND ${directory}/${PLATFORM_PREFIX}pack.cpp "\t${PLATFORM_PREFIX}qplc_pack_be${PACK_POSTFIX},\n") + endif() + endforeach() + + file(APPEND ${directory}/${PLATFORM_PREFIX}pack.cpp "}\n") + + # + # Write scan table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}scan.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}scan.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}scan.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}scan.cpp "scan_table_t ${PLATFORM_PREFIX}scan_table = {\n") + + #get last element of the list + set(LAST_ELEMENT "") + list(GET SCAN_POSTFIX_LIST -1 LAST_ELEMENT) + + foreach(SCAN_POSTFIX IN LISTS SCAN_POSTFIX_LIST) + + if(SCAN_POSTFIX STREQUAL LAST_ELEMENT) + file(APPEND ${directory}/${PLATFORM_PREFIX}scan.cpp "\t${PLATFORM_PREFIX}qplc_scan${SCAN_POSTFIX}};\n") + else() + file(APPEND ${directory}/${PLATFORM_PREFIX}scan.cpp "\t${PLATFORM_PREFIX}qplc_scan${SCAN_POSTFIX},\n") + endif() + endforeach() + + file(APPEND ${directory}/${PLATFORM_PREFIX}scan.cpp "}\n") + + # + # Write scan_i table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}scan_i.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}scan_i.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}scan_i.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}scan_i.cpp "scan_i_table_t ${PLATFORM_PREFIX}scan_i_table = {\n") + + #get last element of the list + set(LAST_ELEMENT "") + list(GET SCAN_POSTFIX_LIST -1 LAST_ELEMENT) + + foreach(SCAN_POSTFIX IN LISTS SCAN_POSTFIX_LIST) + + if(SCAN_POSTFIX STREQUAL LAST_ELEMENT) + file(APPEND ${directory}/${PLATFORM_PREFIX}scan_i.cpp "\t${PLATFORM_PREFIX}qplc_scan${SCAN_POSTFIX}_i};\n") + else() + file(APPEND ${directory}/${PLATFORM_PREFIX}scan_i.cpp "\t${PLATFORM_PREFIX}qplc_scan${SCAN_POSTFIX}_i,\n") + endif() + endforeach() + + file(APPEND ${directory}/${PLATFORM_PREFIX}scan_i.cpp "}\n") + + # + # Write pack_index table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}pack_index.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "pack_index_table_t ${PLATFORM_PREFIX}pack_index_table = {\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_bits_nu,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_index_8u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_index_8u16u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_index_8u32u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_bits_be_nu,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_index_8u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_index_be_8u16u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_index_be_8u32u};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "}\n") + + # + # Write default bit width functions + # + foreach(DEAULT_BIT_WIDTH_FUNCTION IN LISTS DEFAULT_BIT_WIDTH_FUNCTIONS_LIST) + file(WRITE ${directory}/${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}.cpp "${DEAULT_BIT_WIDTH_FUNCTION}_table_t ${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}_table = {\n") + + #get last element of the list + set(LAST_ELEMENT "") + list(GET DEFAULT_BIT_WIDTH_LIST -1 LAST_ELEMENT) + + foreach(BIT_WIDTH IN LISTS DEFAULT_BIT_WIDTH_LIST) + + set(FUNCTION_NAME "") + get_function_name_with_default_bit_width(${DEAULT_BIT_WIDTH_FUNCTION} ${BIT_WIDTH} FUNCTION_NAME) + + if(BIT_WIDTH STREQUAL LAST_ELEMENT) + file(APPEND ${directory}/${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}.cpp "\t${PLATFORM_PREFIX}qplc_${FUNCTION_NAME}};\n") + else() + file(APPEND ${directory}/${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}.cpp "\t${PLATFORM_PREFIX}qplc_${FUNCTION_NAME},\n") + endif() + endforeach() + + file(APPEND ${directory}/${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}.cpp "}\n") + endforeach() + + # + # Write aggregates table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}aggregates.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "aggregates_table_t ${PLATFORM_PREFIX}aggregates_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "\t${PLATFORM_PREFIX}qplc_bit_aggregates_8u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "\t${PLATFORM_PREFIX}qplc_aggregates_8u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "\t${PLATFORM_PREFIX}qplc_aggregates_16u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "\t${PLATFORM_PREFIX}qplc_aggregates_32u};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "}\n") + + # + # Write mem_copy functions table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "memory_copy_table_t ${PLATFORM_PREFIX}memory_copy_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "\t${PLATFORM_PREFIX}qplc_copy_8u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "\t${PLATFORM_PREFIX}qplc_copy_16u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "\t${PLATFORM_PREFIX}qplc_copy_32u};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "}\n") + + # + # Write mem_copy functions table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}zero.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}zero.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}zero.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}zero.cpp "zero_table_t ${PLATFORM_PREFIX}zero_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}zero.cpp "\t${PLATFORM_PREFIX}qplc_zero_8u};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}zero.cpp "}\n") + + # + # Write move functions table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}move.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}move.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}move.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}move.cpp "move_table_t ${PLATFORM_PREFIX}move_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}move.cpp "\t${PLATFORM_PREFIX}qplc_move_8u};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}move.cpp "}\n") + + # + # Write crc64 function table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}crc64.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}crc64.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}crc64.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}crc64.cpp "crc64_table_t ${PLATFORM_PREFIX}crc64_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}crc64.cpp "\t${PLATFORM_PREFIX}qplc_crc64};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}crc64.cpp "}\n") + + # + # Write xor_checksum function table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}xor_checksum.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}xor_checksum.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}xor_checksum.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}xor_checksum.cpp "xor_checksum_table_t ${PLATFORM_PREFIX}xor_checksum_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}xor_checksum.cpp "\t${PLATFORM_PREFIX}qplc_xor_checksum_8u};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}xor_checksum.cpp "}\n") + + # + # Write deflate functions table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}deflate.cpp "#include \"deflate_slow_icf.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "#include \"deflate_hash_table.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "#include \"deflate_histogram.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "deflate_table_t ${PLATFORM_PREFIX}deflate_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "\t reinterpret_cast(&${PLATFORM_PREFIX}slow_deflate_icf_body),\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "\t reinterpret_cast(&${PLATFORM_PREFIX}deflate_histogram_reset),\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "\t reinterpret_cast(&${PLATFORM_PREFIX}deflate_hash_table_reset)};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "}\n") + + # + # Write deflate fix functions table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}deflate_fix.cpp "#include \"deflate_slow.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate_fix.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate_fix.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate_fix.cpp "deflate_fix_table_t ${PLATFORM_PREFIX}deflate_fix_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate_fix.cpp "\t reinterpret_cast(&${PLATFORM_PREFIX}slow_deflate_body)};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate_fix.cpp "}\n") + + # + # Write setup_dictionary functions table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}setup_dictionary.cpp "#include \"deflate_slow_utils.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}setup_dictionary.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}setup_dictionary.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}setup_dictionary.cpp "setup_dictionary_table_t ${PLATFORM_PREFIX}setup_dictionary_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}setup_dictionary.cpp "\t reinterpret_cast(&${PLATFORM_PREFIX}setup_dictionary)};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}setup_dictionary.cpp "}\n") + + endforeach() +endfunction() -# check nasm compiler -include(CheckLanguage) -check_language(ASM_NASM) -if(NOT CMAKE_ASM_NASM_COMPILER) - message(FATAL_ERROR "Please install NASM from 'https://www.nasm.us/' because NASM compiler can not be found!") -endif() -# [SUBDIR]isal enable_language(ASM_NASM) set(ISAL_C_SRC ${QPL_SRC_DIR}/isal/igzip/adler32_base.c @@ -106,11 +512,6 @@ set_target_properties(isal PROPERTIES CXX_STANDARD 11 C_STANDARD 99) -target_compile_options(isal PRIVATE - "$<$:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}>" - "$<$:>" - "$<$:>") - # AS_FEATURE_LEVEL=10 means "Check SIMD capabilities of the target system at runtime and use up to AVX512 if available". # HAVE_KNOWS_AVX512 means rely on AVX512 being available on the target system. target_compile_options(isal_asm PRIVATE "-I${QPL_SRC_DIR}/isal/include/" @@ -163,15 +564,7 @@ foreach(PLATFORM_ID IN LISTS PLATFORMS_LIST) PUBLIC $ PRIVATE $) - set_target_properties(qplcore_${PLATFORM_ID} PROPERTIES - $<$:C_STANDARD 17>) - - target_compile_options(qplcore_${PLATFORM_ID} - PRIVATE ${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS} - PRIVATE "$<$:>" - PRIVATE "$<$:-O3;-D_FORTIFY_SOURCE=2>") - - # Set specific compiler options and/or definitions based on a platform + # Set specific compiler options and/or definitions based on a platform if (${PLATFORM_ID} MATCHES "avx512") target_compile_definitions(qplcore_${PLATFORM_ID} PRIVATE PLATFORM=2) target_compile_options(qplcore_${PLATFORM_ID} PRIVATE -march=skylake-avx512) @@ -220,10 +613,7 @@ set_target_properties(qplcore_sw_dispatcher PROPERTIES CXX_STANDARD 17) target_compile_definitions(qplcore_sw_dispatcher PUBLIC -DQPL_LIB) target_compile_options(qplcore_sw_dispatcher - PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}; - ${QPL_LINUX_TOOLCHAIN_DYNAMIC_LIBRARY_FLAGS}; - $<$:-O3;-D_FORTIFY_SOURCE=2>> - PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}>) + PRIVATE ${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}) # [SUBDIR]core-iaa file(GLOB HW_PATH_SRC ${QPL_SRC_DIR}/core-iaa/sources/aecs/*.c @@ -248,14 +638,6 @@ target_include_directories(core_iaa PRIVATE $ # own_checkers.h PRIVATE $) -set_target_properties(core_iaa PROPERTIES - $<$:C_STANDARD 17> - CXX_STANDARD 17) - -target_compile_options(core_iaa - PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}; - $<$:-O3;-D_FORTIFY_SOURCE=2>>) - target_compile_features(core_iaa PRIVATE c_std_11) target_compile_definitions(core_iaa PRIVATE QPL_BADARG_CHECK @@ -285,10 +667,7 @@ set_property(GLOBAL APPEND PROPERTY QPL_LIB_DEPS $) target_compile_options(middle_layer_lib - PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}; - ${QPL_LINUX_TOOLCHAIN_DYNAMIC_LIBRARY_FLAGS}; - $<$:-O3;-D_FORTIFY_SOURCE=2>> - PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}>) + PRIVATE ${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}) target_compile_definitions(middle_layer_lib PUBLIC QPL_VERSION="${QPL_VERSION}" @@ -323,15 +702,8 @@ target_include_directories(_qpl PRIVATE $ PRIVATE $) -set_target_properties(_qpl PROPERTIES - $<$:C_STANDARD 17> - CXX_STANDARD 17) - target_compile_options(_qpl - PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}; - ${QPL_LINUX_TOOLCHAIN_DYNAMIC_LIBRARY_FLAGS}; - $<$:-O3;-D_FORTIFY_SOURCE=2>> - PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}>) + PRIVATE ${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}) target_compile_definitions(_qpl PRIVATE -DQPL_LIB diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index d469b359d1a..42bfb48db70 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -38,7 +38,7 @@ rm -f CMakeCache.txt # To check it, find and delete them. grep -o -P '"contrib/[^"]+"' ../.gitmodules | - grep -v -P 'llvm-project|abseil-cpp|qpl|grpc|corrosion' | + grep -v -P 'llvm-project|abseil-cpp|grpc|corrosion' | xargs -I@ find ../@ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | xargs rm From 7dda3b2353b22639a0304219c01beed16407c6eb Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 17 Nov 2023 10:11:15 +0000 Subject: [PATCH 127/192] Review comments --- src/Backups/BackupCoordinationFileInfos.cpp | 65 ++++++++++++--------- src/Backups/BackupFileInfo.h | 7 ++- src/Backups/BackupImpl.cpp | 29 ++++++--- src/Backups/BackupImpl.h | 7 +-- 4 files changed, 65 insertions(+), 43 deletions(-) diff --git a/src/Backups/BackupCoordinationFileInfos.cpp b/src/Backups/BackupCoordinationFileInfos.cpp index b17b755b966..3280064a5d7 100644 --- a/src/Backups/BackupCoordinationFileInfos.cpp +++ b/src/Backups/BackupCoordinationFileInfos.cpp @@ -50,6 +50,21 @@ BackupFileInfo BackupCoordinationFileInfos::getFileInfoByDataFileIndex(size_t da return *(file_infos_for_all_hosts[data_file_index]); } +namespace +{ + +/// copy all the file infos that are shared between reference target and source +void copyFileInfoToReference(const BackupFileInfo & target, BackupFileInfo & reference) +{ + reference.size = target.size; + reference.checksum = target.checksum; + reference.base_size = target.base_size; + reference.base_checksum = target.base_checksum; + reference.encrypted_by_disk = target.encrypted_by_disk; +} + +} + void BackupCoordinationFileInfos::prepare() const { if (prepared) @@ -78,11 +93,24 @@ void BackupCoordinationFileInfos::prepare() const num_files = 0; total_size_of_files = 0; + std::vector unresolved_references; + std::unordered_map file_name_to_info; + + const auto handle_unresolved_references = [&](const auto & try_resolve_reference) + { + for (auto * reference : unresolved_references) + { + if (!try_resolve_reference(*reference)) + throw DB::Exception( + ErrorCodes::LOGICAL_ERROR, + "Couldn't resolve reference {} with target {}", + reference->file_name, + reference->reference_target); + } + }; + if (plain_backup) { - std::vector unresolved_references; - std::unordered_map file_name_to_info; - const auto try_resolve_reference = [&](BackupFileInfo & reference) { auto it = file_name_to_info.find(reference.reference_target); @@ -91,10 +119,9 @@ void BackupCoordinationFileInfos::prepare() const return false; auto & target_info = it->second; - target_info->reference_sources.push_back(reference.file_name); - reference.size = target_info->size; + target_info->data_file_copies.push_back(reference.file_name); + copyFileInfoToReference(*target_info, reference); total_size_of_files += reference.size; - reference.checksum = target_info->checksum; return true; }; @@ -118,23 +145,12 @@ void BackupCoordinationFileInfos::prepare() const } } - for (auto * reference : unresolved_references) - { - if (!try_resolve_reference(*reference)) - throw DB::Exception( - ErrorCodes::LOGICAL_ERROR, - "Couldn't resolve reference {} with target {}", - reference->file_name, - reference->reference_target); - } + handle_unresolved_references(try_resolve_reference); num_files = file_infos_for_all_hosts.size(); } else { - std::vector unresolved_references; - std::unordered_map file_name_to_info; - const auto try_resolve_reference = [&](BackupFileInfo & reference) { auto it = file_name_to_info.find(reference.reference_target); @@ -143,8 +159,7 @@ void BackupCoordinationFileInfos::prepare() const return false; auto & target_info = it->second; - reference.size = target_info->size; - reference.checksum = target_info->checksum; + copyFileInfoToReference(*target_info, reference); reference.data_file_name = target_info->data_file_name; reference.data_file_index = target_info->data_file_index; return true; @@ -195,15 +210,7 @@ void BackupCoordinationFileInfos::prepare() const file_name_to_info.emplace(info.file_name, &info); } - for (auto * reference : unresolved_references) - { - if (!try_resolve_reference(*reference)) - throw DB::Exception( - ErrorCodes::LOGICAL_ERROR, - "Couldn't resolve reference {} with target {}", - reference->file_name, - reference->reference_target); - } + handle_unresolved_references(try_resolve_reference); num_files = file_infos_for_all_hosts.size(); } diff --git a/src/Backups/BackupFileInfo.h b/src/Backups/BackupFileInfo.h index 42bda3aa6ed..009fee091e0 100644 --- a/src/Backups/BackupFileInfo.h +++ b/src/Backups/BackupFileInfo.h @@ -42,9 +42,10 @@ struct BackupFileInfo /// Set if this file is just a reference to another file String reference_target; - /// List of files that are referencing this file - /// Used for plain backup which needs to resolve all references - Strings reference_sources; + /// (While writing a backup) if this list is not empty then after writing + /// `data_file_name` it should be copied to this list of destinations too. + /// This is used for plain backups. + Strings data_file_copies; struct LessByFileName { diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index 56c30fab5c2..61984d58889 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -24,6 +24,8 @@ #include #include +#include + namespace ProfileEvents { @@ -452,7 +454,6 @@ void BackupImpl::readBackupMetadata() size_of_entries = 0; const auto * contents = config_root->getNodeByPath("contents"); - std::vector> reference_files; for (const Poco::XML::Node * child = contents->firstChild(); child; child = child->nextSibling()) { if (child->nodeName() == "file") @@ -913,15 +914,20 @@ void BackupImpl::writeFile(const BackupFileInfo & info, BackupEntryPtr entry) /// NOTE: `mutex` must be unlocked during copying otherwise writing will be in one thread maximum and hence slow. - if (use_archive) + const auto write_info_to_archive = [&](const auto & file_name) { - LOG_TRACE(log, "Writing backup for file {} from {}: data file #{}, adding to archive", info.data_file_name, src_file_desc, info.data_file_index); - auto out = archive_writer->writeFile(info.data_file_name); + auto out = archive_writer->writeFile(file_name); auto read_buffer = entry->getReadBuffer(writer->getReadSettings()); if (info.base_size != 0) read_buffer->seek(info.base_size, SEEK_SET); copyData(*read_buffer, *out); out->finalize(); + }; + + if (use_archive) + { + LOG_TRACE(log, "Writing backup for file {} from {}: data file #{}, adding to archive", info.data_file_name, src_file_desc, info.data_file_index); + write_info_to_archive(info.data_file_name); } else if (src_disk && from_immutable_file) { @@ -935,11 +941,20 @@ void BackupImpl::writeFile(const BackupFileInfo & info, BackupEntryPtr entry) writer->copyDataToFile(info.data_file_name, create_read_buffer, info.base_size, info.size - info.base_size); } - if (!deduplicate_files) + std::function copy_file_inside_backup; + if (use_archive) { - for (const auto & reference : info.reference_sources) - writer->copyFile(reference, info.data_file_name, info.size - info.base_size); + copy_file_inside_backup = write_info_to_archive; } + else + { + copy_file_inside_backup = [&](const auto & data_file_copy) + { + writer->copyFile(data_file_copy, info.data_file_name, info.size - info.base_size); + }; + } + + std::ranges::for_each(info.data_file_copies, copy_file_inside_backup); { std::lock_guard lock{mutex}; diff --git a/src/Backups/BackupImpl.h b/src/Backups/BackupImpl.h index a4ab3d84d0c..6070db79aa6 100644 --- a/src/Backups/BackupImpl.h +++ b/src/Backups/BackupImpl.h @@ -72,11 +72,14 @@ public: Strings listFiles(const String & directory, bool recursive) const override; bool hasFiles(const String & directory) const override; bool fileExists(const String & file_name) const override; + bool fileExists(const SizeAndChecksum & size_and_checksum) const override; UInt64 getFileSize(const String & file_name) const override; UInt128 getFileChecksum(const String & file_name) const override; SizeAndChecksum getFileSizeAndChecksum(const String & file_name) const override; std::unique_ptr readFile(const String & file_name) const override; + std::unique_ptr readFile(const SizeAndChecksum & size_and_checksum) const override; size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const override; + size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const override; void writeFile(const BackupFileInfo & info, BackupEntryPtr entry) override; void finalizeWriting() override; bool supportsWritingInMultipleThreads() const override { return !use_archive; } @@ -111,10 +114,6 @@ private: std::unique_ptr readFileImpl(const SizeAndChecksum & size_and_checksum, bool read_encrypted) const; - bool fileExists(const SizeAndChecksum & size_and_checksum) const override; - std::unique_ptr readFile(const SizeAndChecksum & size_and_checksum) const override; - size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const override; - BackupInfo backup_info; const String backup_name_for_logging; const bool use_archive; From 9bcedf376436bb066db639a7ab7325cd71fb3b73 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 17 Nov 2023 10:27:19 +0000 Subject: [PATCH 128/192] Cleanup --- src/Backups/BackupIO_Default.cpp | 2 -- src/Common/ZooKeeper/ZooKeeper.cpp | 1 - src/Storages/StorageKeeperMap.cpp | 40 ++++++++++++++++-------------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/Backups/BackupIO_Default.cpp b/src/Backups/BackupIO_Default.cpp index 95f2c66b6b9..5ac522695ce 100644 --- a/src/Backups/BackupIO_Default.cpp +++ b/src/Backups/BackupIO_Default.cpp @@ -91,6 +91,4 @@ void BackupWriterDefault::copyFileFromDisk(const String & path_in_backup, DiskPt copyDataToFile(path_in_backup, create_read_buffer, start_pos, length); } - - } diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 8a97362aa96..436a4e14f14 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -385,7 +385,6 @@ void ZooKeeper::createAncestors(const std::string & path) size_t last_pos = path.rfind('/'); if (last_pos == std::string::npos || last_pos == 0) return; - std::string current_node = path.substr(0, last_pos); while (true) diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index 237b65c6a72..e3c960529de 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -763,28 +763,29 @@ void StorageKeeperMap::backupData(BackupEntriesCollector & backup_entries_collec auto post_collecting_task = [my_table_id = std::move(table_id), coordination, &backup_entries_collector, my_data_path_in_backup = data_path_in_backup, this] { auto path_with_data = coordination->getKeeperMapDataPath(zk_root_path); - if (path_with_data == my_data_path_in_backup) + if (path_with_data != my_data_path_in_backup) { - auto temp_disk = backup_entries_collector.getContext()->getGlobalTemporaryVolume()->getDisk(0); - auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef().max_compress_block_size; - - auto with_retries = std::make_shared - ( - &Poco::Logger::get(fmt::format("StorageKeeperMapBackup ({})", getStorageID().getNameForLogs())), - [&] { return getClient(); }, - WithRetries::KeeperSettings::fromContext(backup_entries_collector.getContext()), - [](WithRetries::FaultyKeeper &) {} - ); - - backup_entries_collector.addBackupEntries( - std::make_shared(this->zk_data_path, path_with_data, temp_disk, max_compress_block_size, std::move(with_retries)) - ->getBackupEntries()); + std::string source_path = fs::path(my_data_path_in_backup) / backup_data_filename; + std::string target_path = fs::path(path_with_data) / backup_data_filename; + backup_entries_collector.addBackupEntries({{source_path, std::make_shared(std::move(target_path))}}); return; } - std::string source_path = fs::path(my_data_path_in_backup) / backup_data_filename; - std::string target_path = fs::path(path_with_data) / backup_data_filename; - backup_entries_collector.addBackupEntries({{source_path, std::make_shared(std::move(target_path))}}); + auto temp_disk = backup_entries_collector.getContext()->getGlobalTemporaryVolume()->getDisk(0); + auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef().max_compress_block_size; + + auto with_retries = std::make_shared + ( + &Poco::Logger::get(fmt::format("StorageKeeperMapBackup ({})", getStorageID().getNameForLogs())), + [&] { return getClient(); }, + WithRetries::KeeperSettings::fromContext(backup_entries_collector.getContext()), + [](WithRetries::FaultyKeeper &) {} + ); + + backup_entries_collector.addBackupEntries( + std::make_shared( + this->zk_data_path, path_with_data, temp_disk, max_compress_block_size, std::move(with_retries)) + ->getBackupEntries()); }; backup_entries_collector.addPostTask(post_collecting_task); @@ -796,7 +797,8 @@ void StorageKeeperMap::restoreDataFromBackup(RestorerFromBackup & restorer, cons if (!backup->hasFiles(data_path_in_backup)) return; - auto table_id = toString(getStorageID().uuid); if (!restorer.getRestoreCoordination()->acquireInsertingDataForKeeperMap(zk_root_path, table_id)) + auto table_id = toString(getStorageID().uuid); + if (!restorer.getRestoreCoordination()->acquireInsertingDataForKeeperMap(zk_root_path, table_id)) { /// Other table is already restoring the data for this Keeper path. /// Tables defined on the same path share data From 4f441ec1319ac3f80a70f773f98ea99a80dfc407 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 11:32:27 +0100 Subject: [PATCH 129/192] Own CMake for Abseil --- contrib/abseil-cpp-cmake/CMakeLists.txt | 3451 ++++++++++++++++++++++- contrib/re2-cmake/CMakeLists.txt | 13 +- docker/packager/binary/build.sh | 2 +- 3 files changed, 3436 insertions(+), 30 deletions(-) diff --git a/contrib/abseil-cpp-cmake/CMakeLists.txt b/contrib/abseil-cpp-cmake/CMakeLists.txt index 2901daf32db..e84b4d46c4a 100644 --- a/contrib/abseil-cpp-cmake/CMakeLists.txt +++ b/contrib/abseil-cpp-cmake/CMakeLists.txt @@ -1,33 +1,3428 @@ set(ABSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp") +set(ABSL_COMMON_INCLUDE_DIRS "${ABSL_ROOT_DIR}") + +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +function(absl_cc_library) + cmake_parse_arguments(ABSL_CC_LIB + "DISABLE_INSTALL;PUBLIC;TESTONLY" + "NAME" + "HDRS;SRCS;COPTS;DEFINES;LINKOPTS;DEPS" + ${ARGN} + ) + + set(_NAME "absl_${ABSL_CC_LIB_NAME}") + + # Check if this is a header-only library + set(ABSL_CC_SRCS "${ABSL_CC_LIB_SRCS}") + foreach(src_file IN LISTS ABSL_CC_SRCS) + if(${src_file} MATCHES ".*\\.(h|inc)") + list(REMOVE_ITEM ABSL_CC_SRCS "${src_file}") + endif() + endforeach() + + if(ABSL_CC_SRCS STREQUAL "") + set(ABSL_CC_LIB_IS_INTERFACE 1) + else() + set(ABSL_CC_LIB_IS_INTERFACE 0) + endif() + + if(NOT ABSL_CC_LIB_IS_INTERFACE) + add_library(${_NAME} "") + target_sources(${_NAME} PRIVATE ${ABSL_CC_LIB_SRCS} ${ABSL_CC_LIB_HDRS}) + target_link_libraries(${_NAME} + PUBLIC ${ABSL_CC_LIB_DEPS} + PRIVATE + ${ABSL_CC_LIB_LINKOPTS} + ${ABSL_DEFAULT_LINKOPTS} + ) + + target_include_directories(${_NAME} + PUBLIC "${ABSL_COMMON_INCLUDE_DIRS}") + target_compile_options(${_NAME} + PRIVATE ${ABSL_CC_LIB_COPTS}) + target_compile_definitions(${_NAME} PUBLIC ${ABSL_CC_LIB_DEFINES}) + + else() + # Generating header-only library + add_library(${_NAME} INTERFACE) + target_include_directories(${_NAME} + INTERFACE "${ABSL_COMMON_INCLUDE_DIRS}") + + target_link_libraries(${_NAME} + INTERFACE + ${ABSL_CC_LIB_DEPS} + ${ABSL_CC_LIB_LINKOPTS} + ${ABSL_DEFAULT_LINKOPTS} + ) + target_compile_definitions(${_NAME} INTERFACE ${ABSL_CC_LIB_DEFINES}) + + endif() + + add_library(absl::${ABSL_CC_LIB_NAME} ALIAS ${_NAME}) +endfunction() + + +set(DIR ${ABSL_ROOT_DIR}/absl/algorithm) + +absl_cc_library( + NAME + algorithm + HDRS + "${DIR}/algorithm.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +absl_cc_library( + NAME + algorithm_container + HDRS + "${DIR}/container.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::algorithm + absl::core_headers + absl::meta + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/base) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + atomic_hook + HDRS + "${DIR}/internal/atomic_hook.h" + DEPS + absl::config + absl::core_headers + COPTS + ${ABSL_DEFAULT_COPTS} +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + errno_saver + HDRS + "${DIR}/internal/errno_saver.h" + DEPS + absl::config + COPTS + ${ABSL_DEFAULT_COPTS} +) + +absl_cc_library( + NAME + log_severity + HDRS + "${DIR}/log_severity.h" + SRCS + "${DIR}/log_severity.cc" + DEPS + absl::config + absl::core_headers + COPTS + ${ABSL_DEFAULT_COPTS} +) + +absl_cc_library( + NAME + nullability + HDRS + "${DIR}/nullability.h" + SRCS + "${DIR}/internal/nullability_impl.h" + DEPS + absl::core_headers + absl::type_traits + COPTS + ${ABSL_DEFAULT_COPTS} +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + raw_logging_internal + HDRS + "${DIR}/internal/raw_logging.h" + SRCS + "${DIR}/internal/raw_logging.cc" + DEPS + absl::atomic_hook + absl::config + absl::core_headers + absl::errno_saver + absl::log_severity + COPTS + ${ABSL_DEFAULT_COPTS} +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + spinlock_wait + HDRS + "${DIR}/internal/spinlock_wait.h" + SRCS + "${DIR}/internal/spinlock_akaros.inc" + "${DIR}/internal/spinlock_linux.inc" + "${DIR}/internal/spinlock_posix.inc" + "${DIR}/internal/spinlock_wait.cc" + "${DIR}/internal/spinlock_win32.inc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::core_headers + absl::errno_saver +) + +absl_cc_library( + NAME + config + HDRS + "${DIR}/config.h" + "${DIR}/options.h" + "${DIR}/policy_checks.h" + COPTS + ${ABSL_DEFAULT_COPTS} + PUBLIC +) + +absl_cc_library( + NAME + dynamic_annotations + HDRS + "${DIR}/dynamic_annotations.h" + SRCS + "${DIR}/internal/dynamic_annotations.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +absl_cc_library( + NAME + core_headers + HDRS + "${DIR}/attributes.h" + "${DIR}/const_init.h" + "${DIR}/macros.h" + "${DIR}/optimization.h" + "${DIR}/port.h" + "${DIR}/thread_annotations.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + malloc_internal + HDRS + "${DIR}/internal/direct_mmap.h" + "${DIR}/internal/low_level_alloc.h" + SRCS + "${DIR}/internal/low_level_alloc.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::base_internal + absl::config + absl::core_headers + absl::dynamic_annotations + absl::raw_logging_internal + Threads::Threads +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + base_internal + HDRS + "${DIR}/internal/hide_ptr.h" + "${DIR}/internal/identity.h" + "${DIR}/internal/inline_variable.h" + "${DIR}/internal/invoke.h" + "${DIR}/internal/scheduling_mode.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::type_traits +) + +absl_cc_library( + NAME + base + HDRS + "${DIR}/call_once.h" + "${DIR}/casts.h" + "${DIR}/internal/cycleclock.h" + "${DIR}/internal/cycleclock_config.h" + "${DIR}/internal/low_level_scheduling.h" + "${DIR}/internal/per_thread_tls.h" + "${DIR}/internal/spinlock.h" + "${DIR}/internal/sysinfo.h" + "${DIR}/internal/thread_identity.h" + "${DIR}/internal/tsan_mutex_interface.h" + "${DIR}/internal/unscaledcycleclock.h" + "${DIR}/internal/unscaledcycleclock_config.h" + SRCS + "${DIR}/internal/cycleclock.cc" + "${DIR}/internal/spinlock.cc" + "${DIR}/internal/sysinfo.cc" + "${DIR}/internal/thread_identity.cc" + "${DIR}/internal/unscaledcycleclock.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::atomic_hook + absl::base_internal + absl::config + absl::core_headers + absl::dynamic_annotations + absl::log_severity + absl::raw_logging_internal + absl::spinlock_wait + absl::type_traits + Threads::Threads + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + throw_delegate + HDRS + "${DIR}/internal/throw_delegate.h" + SRCS + "${DIR}/internal/throw_delegate.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + pretty_function + HDRS + "${DIR}/internal/pretty_function.h" + COPTS + ${ABSL_DEFAULT_COPTS} +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + endian + HDRS + "${DIR}/internal/endian.h" + "${DIR}/internal/unaligned_access.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::core_headers + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + scoped_set_env + SRCS + "${DIR}/internal/scoped_set_env.cc" + HDRS + "${DIR}/internal/scoped_set_env.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + strerror + SRCS + "${DIR}/internal/strerror.cc" + HDRS + "${DIR}/internal/strerror.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::errno_saver +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + fast_type_id + HDRS + "${DIR}/internal/fast_type_id.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +absl_cc_library( + NAME + prefetch + HDRS + "${DIR}/prefetch.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers +) + +set(DIR ${ABSL_ROOT_DIR}/absl/cleanup) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cleanup_internal + HDRS + "${DIR}/internal/cleanup.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::core_headers + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + cleanup + HDRS + "${DIR}/cleanup.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::cleanup_internal + absl::config + absl::core_headers + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/container) + +absl_cc_library( + NAME + btree + HDRS + "${DIR}/btree_map.h" + "${DIR}/btree_set.h" + "${DIR}/internal/btree.h" + "${DIR}/internal/btree_container.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::container_common + absl::common_policy_traits + absl::compare + absl::compressed_tuple + absl::container_memory + absl::cord + absl::core_headers + absl::layout + absl::memory + absl::raw_logging_internal + absl::strings + absl::throw_delegate + absl::type_traits + absl::utility +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + compressed_tuple + HDRS + "${DIR}/internal/compressed_tuple.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + fixed_array + HDRS + "${DIR}/fixed_array.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::compressed_tuple + absl::algorithm + absl::config + absl::core_headers + absl::dynamic_annotations + absl::throw_delegate + absl::memory + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + inlined_vector_internal + HDRS + "${DIR}/internal/inlined_vector.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::compressed_tuple + absl::core_headers + absl::memory + absl::span + absl::type_traits + PUBLIC +) + +absl_cc_library( + NAME + inlined_vector + HDRS + "${DIR}/inlined_vector.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::algorithm + absl::core_headers + absl::inlined_vector_internal + absl::throw_delegate + absl::memory + absl::type_traits + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + counting_allocator + HDRS + "${DIR}/internal/counting_allocator.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config +) + +absl_cc_library( + NAME + flat_hash_map + HDRS + "${DIR}/flat_hash_map.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::container_memory + absl::core_headers + absl::hash_function_defaults + absl::raw_hash_map + absl::algorithm_container + absl::memory + PUBLIC +) + +absl_cc_library( + NAME + flat_hash_set + HDRS + "${DIR}/flat_hash_set.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::container_memory + absl::hash_function_defaults + absl::raw_hash_set + absl::algorithm_container + absl::core_headers + absl::memory + PUBLIC +) + +absl_cc_library( + NAME + node_hash_map + HDRS + "${DIR}/node_hash_map.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::container_memory + absl::core_headers + absl::hash_function_defaults + absl::node_slot_policy + absl::raw_hash_map + absl::algorithm_container + absl::memory + PUBLIC +) + +absl_cc_library( + NAME + node_hash_set + HDRS + "${DIR}/node_hash_set.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::hash_function_defaults + absl::node_slot_policy + absl::raw_hash_set + absl::algorithm_container + absl::memory + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + container_memory + HDRS + "${DIR}/internal/container_memory.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::memory + absl::type_traits + absl::utility + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + hash_function_defaults + HDRS + "${DIR}/internal/hash_function_defaults.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::cord + absl::hash + absl::strings + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + hash_policy_traits + HDRS + "${DIR}/internal/hash_policy_traits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::common_policy_traits + absl::meta + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + common_policy_traits + HDRS + "${DIR}/internal/common_policy_traits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::meta + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + hashtablez_sampler + HDRS + "${DIR}/internal/hashtablez_sampler.h" + SRCS + "${DIR}/internal/hashtablez_sampler.cc" + "${DIR}/internal/hashtablez_sampler_force_weak_definition.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::exponential_biased + absl::raw_logging_internal + absl::sample_recorder + absl::synchronization + absl::time +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + hashtable_debug + HDRS + "${DIR}/internal/hashtable_debug.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::hashtable_debug_hooks +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + hashtable_debug_hooks + HDRS + "${DIR}/internal/hashtable_debug_hooks.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + node_slot_policy + HDRS + "${DIR}/internal/node_slot_policy.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + raw_hash_map + HDRS + "${DIR}/internal/raw_hash_map.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::container_memory + absl::raw_hash_set + absl::throw_delegate + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + container_common + HDRS + "${DIR}/internal/common.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + raw_hash_set + HDRS + "${DIR}/internal/raw_hash_set.h" + SRCS + "${DIR}/internal/raw_hash_set.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bits + absl::compressed_tuple + absl::config + absl::container_common + absl::container_memory + absl::core_headers + absl::dynamic_annotations + absl::endian + absl::hash + absl::hash_policy_traits + absl::hashtable_debug_hooks + absl::hashtablez_sampler + absl::memory + absl::meta + absl::optional + absl::prefetch + absl::raw_logging_internal + absl::utility + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + layout + HDRS + "${DIR}/internal/layout.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::meta + absl::strings + absl::span + absl::utility + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/crc) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + crc_cpu_detect + HDRS + "${DIR}/internal/cpu_detect.h" + SRCS + "${DIR}/internal/cpu_detect.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + crc_internal + HDRS + "${DIR}/internal/crc.h" + "${DIR}/internal/crc32_x86_arm_combined_simd.h" + SRCS + "${DIR}/internal/crc.cc" + "${DIR}/internal/crc_internal.h" + "${DIR}/internal/crc_x86_arm_combined.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::crc_cpu_detect + absl::config + absl::core_headers + absl::endian + absl::prefetch + absl::raw_logging_internal + absl::memory + absl::bits +) + +absl_cc_library( + NAME + crc32c + HDRS + "${DIR}/crc32c.h" + "${DIR}/internal/crc32c.h" + "${DIR}/internal/crc_memcpy.h" + SRCS + "${DIR}/crc32c.cc" + "${DIR}/internal/crc32c_inline.h" + "${DIR}/internal/crc_memcpy_fallback.cc" + "${DIR}/internal/crc_memcpy_x86_arm_combined.cc" + "${DIR}/internal/crc_non_temporal_memcpy.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::crc_cpu_detect + absl::crc_internal + absl::non_temporal_memcpy + absl::config + absl::core_headers + absl::endian + absl::prefetch + absl::str_format + absl::strings +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + non_temporal_arm_intrinsics + HDRS + "${DIR}/internal/non_temporal_arm_intrinsics.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + non_temporal_memcpy + HDRS + "${DIR}/internal/non_temporal_memcpy.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::non_temporal_arm_intrinsics + absl::config + absl::core_headers +) + +absl_cc_library( + NAME + crc_cord_state + HDRS + "${DIR}/internal/crc_cord_state.h" + SRCS + "${DIR}/internal/crc_cord_state.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::crc32c + absl::config + absl::strings +) + +set(DIR ${ABSL_ROOT_DIR}/absl/debugging) + +absl_cc_library( + NAME + stacktrace + HDRS + "${DIR}/stacktrace.h" + "${DIR}/internal/stacktrace_aarch64-inl.inc" + "${DIR}/internal/stacktrace_arm-inl.inc" + "${DIR}/internal/stacktrace_config.h" + "${DIR}/internal/stacktrace_emscripten-inl.inc" + "${DIR}/internal/stacktrace_generic-inl.inc" + "${DIR}/internal/stacktrace_powerpc-inl.inc" + "${DIR}/internal/stacktrace_riscv-inl.inc" + "${DIR}/internal/stacktrace_unimplemented-inl.inc" + "${DIR}/internal/stacktrace_win32-inl.inc" + "${DIR}/internal/stacktrace_x86-inl.inc" + SRCS + "${DIR}/stacktrace.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::debugging_internal + absl::config + absl::core_headers + absl::dynamic_annotations + absl::raw_logging_internal + PUBLIC +) + +absl_cc_library( + NAME + symbolize + HDRS + "${DIR}/symbolize.h" + "${DIR}/internal/symbolize.h" + SRCS + "${DIR}/symbolize.cc" + "${DIR}/symbolize_darwin.inc" + "${DIR}/symbolize_elf.inc" + "${DIR}/symbolize_emscripten.inc" + "${DIR}/symbolize_unimplemented.inc" + "${DIR}/symbolize_win32.inc" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::debugging_internal + absl::demangle_internal + absl::base + absl::config + absl::core_headers + absl::dynamic_annotations + absl::malloc_internal + absl::raw_logging_internal + absl::strings + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + examine_stack + HDRS + "${DIR}/internal/examine_stack.h" + SRCS + "${DIR}/internal/examine_stack.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::stacktrace + absl::symbolize + absl::config + absl::core_headers + absl::raw_logging_internal +) + +absl_cc_library( + NAME + failure_signal_handler + HDRS + "${DIR}/failure_signal_handler.h" + SRCS + "${DIR}/failure_signal_handler.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::examine_stack + absl::stacktrace + absl::base + absl::config + absl::core_headers + absl::raw_logging_internal + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + debugging_internal + HDRS + "${DIR}/internal/address_is_readable.h" + "${DIR}/internal/elf_mem_image.h" + "${DIR}/internal/vdso_support.h" + SRCS + "${DIR}/internal/address_is_readable.cc" + "${DIR}/internal/elf_mem_image.cc" + "${DIR}/internal/vdso_support.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::config + absl::dynamic_annotations + absl::errno_saver + absl::raw_logging_internal +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + demangle_internal + HDRS + "${DIR}/internal/demangle.h" + SRCS + "${DIR}/internal/demangle.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::core_headers + PUBLIC +) + +absl_cc_library( + NAME + leak_check + HDRS + "${DIR}/leak_check.h" + SRCS + "${DIR}/leak_check.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + PUBLIC +) + +# component target +absl_cc_library( + NAME + debugging + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::stacktrace + absl::leak_check + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/flags) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_path_util + HDRS + "${DIR}/internal/path_util.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::strings + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_program_name + SRCS + "${DIR}/internal/program_name.cc" + HDRS + "${DIR}/internal/program_name.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::flags_path_util + absl::strings + absl::synchronization + PUBLIC +) + +absl_cc_library( + NAME + flags_config + SRCS + "${DIR}/usage_config.cc" + HDRS + "${DIR}/config.h" + "${DIR}/usage_config.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_path_util + absl::flags_program_name + absl::core_headers + absl::strings + absl::synchronization +) + +absl_cc_library( + NAME + flags_marshalling + SRCS + "${DIR}/marshalling.cc" + HDRS + "${DIR}/marshalling.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_severity + absl::int128 + absl::optional + absl::strings + absl::str_format +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_commandlineflag_internal + SRCS + "${DIR}/internal/commandlineflag.cc" + HDRS + "${DIR}/internal/commandlineflag.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::dynamic_annotations + absl::fast_type_id +) + +absl_cc_library( + NAME + flags_commandlineflag + SRCS + "${DIR}/commandlineflag.cc" + HDRS + "${DIR}/commandlineflag.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::fast_type_id + absl::flags_commandlineflag_internal + absl::optional + absl::strings +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_private_handle_accessor + SRCS + "${DIR}/internal/private_handle_accessor.cc" + HDRS + "${DIR}/internal/private_handle_accessor.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_commandlineflag + absl::flags_commandlineflag_internal + absl::strings +) + +absl_cc_library( + NAME + flags_reflection + SRCS + "${DIR}/reflection.cc" + HDRS + "${DIR}/reflection.h" + "${DIR}/internal/registry.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_commandlineflag + absl::flags_private_handle_accessor + absl::flags_config + absl::strings + absl::synchronization + absl::flat_hash_map +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_internal + SRCS + "${DIR}/internal/flag.cc" + HDRS + "${DIR}/internal/flag.h" + "${DIR}/internal/sequence_lock.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base + absl::config + absl::flags_commandlineflag + absl::flags_commandlineflag_internal + absl::flags_config + absl::flags_marshalling + absl::synchronization + absl::meta + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + flags + SRCS + "${DIR}/flag.cc" + HDRS + "${DIR}/declare.h" + "${DIR}/flag.h" + "${DIR}/internal/flag_msvc.inc" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_commandlineflag + absl::flags_config + absl::flags_internal + absl::flags_reflection + absl::base + absl::core_headers + absl::strings +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_usage_internal + SRCS + "${DIR}/internal/usage.cc" + HDRS + "${DIR}/internal/usage.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_config + absl::flags + absl::flags_commandlineflag + absl::flags_internal + absl::flags_path_util + absl::flags_private_handle_accessor + absl::flags_program_name + absl::flags_reflection + absl::strings + absl::synchronization +) + +absl_cc_library( + NAME + flags_usage + SRCS + "${DIR}/usage.cc" + HDRS + "${DIR}/usage.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::flags_usage_internal + absl::raw_logging_internal + absl::strings + absl::synchronization +) + +absl_cc_library( + NAME + flags_parse + SRCS + "${DIR}/parse.cc" + HDRS + "${DIR}/internal/parse.h" + "${DIR}/parse.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::algorithm_container + absl::config + absl::core_headers + absl::flags_config + absl::flags + absl::flags_commandlineflag + absl::flags_commandlineflag_internal + absl::flags_internal + absl::flags_private_handle_accessor + absl::flags_program_name + absl::flags_reflection + absl::flags_usage + absl::strings + absl::synchronization +) + +set(DIR ${ABSL_ROOT_DIR}/absl/functional) + +absl_cc_library( + NAME + any_invocable + SRCS + "${DIR}/internal/any_invocable.h" + HDRS + "${DIR}/any_invocable.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::config + absl::core_headers + absl::type_traits + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + bind_front + SRCS + "${DIR}/internal/front_binder.h" + HDRS + "${DIR}/bind_front.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::compressed_tuple + PUBLIC +) + +absl_cc_library( + NAME + function_ref + SRCS + "${DIR}/internal/function_ref.h" + HDRS + "${DIR}/function_ref.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::core_headers + absl::any_invocable + absl::meta + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/hash) + +absl_cc_library( + NAME + hash + HDRS + "${DIR}/hash.h" + SRCS + "${DIR}/internal/hash.cc" + "${DIR}/internal/hash.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bits + absl::city + absl::config + absl::core_headers + absl::endian + absl::fixed_array + absl::function_ref + absl::meta + absl::int128 + absl::strings + absl::optional + absl::variant + absl::utility + absl::low_level_hash + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + city + HDRS + "${DIR}/internal/city.h" + SRCS + "${DIR}/internal/city.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::endian +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + low_level_hash + HDRS + "${DIR}/internal/low_level_hash.h" + SRCS + "${DIR}/internal/low_level_hash.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::endian + absl::int128 + absl::prefetch +) + +set(DIR ${ABSL_ROOT_DIR}/absl/log) + +# Internal targets +absl_cc_library( + NAME + log_internal_check_impl + SRCS + HDRS + "${DIR}/internal/check_impl.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::core_headers + absl::log_internal_check_op + absl::log_internal_conditions + absl::log_internal_message + absl::log_internal_strip +) + +absl_cc_library( + NAME + log_internal_check_op + SRCS + "${DIR}/internal/check_op.cc" + HDRS + "${DIR}/internal/check_op.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_internal_nullguard + absl::log_internal_nullstream + absl::log_internal_strip + absl::strings +) + +absl_cc_library( + NAME + log_internal_conditions + SRCS + "${DIR}/internal/conditions.cc" + HDRS + "${DIR}/internal/conditions.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::log_internal_voidify +) + +absl_cc_library( + NAME + log_internal_config + SRCS + HDRS + "${DIR}/internal/config.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers +) + +absl_cc_library( + NAME + log_internal_flags + SRCS + HDRS + "${DIR}/internal/flags.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::flags +) + +absl_cc_library( + NAME + log_internal_format + SRCS + "${DIR}/internal/log_format.cc" + HDRS + "${DIR}/internal/log_format.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_internal_append_truncated + absl::log_internal_config + absl::log_internal_globals + absl::log_severity + absl::strings + absl::str_format + absl::time + absl::span +) + +absl_cc_library( + NAME + log_internal_globals + SRCS + "${DIR}/internal/globals.cc" + HDRS + "${DIR}/internal/globals.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_severity + absl::raw_logging_internal + absl::strings + absl::time +) + +absl_cc_library( + NAME + log_internal_log_impl + SRCS + HDRS + "${DIR}/internal/log_impl.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::log_internal_conditions + absl::log_internal_message + absl::log_internal_strip +) + +absl_cc_library( + NAME + log_internal_proto + SRCS + "${DIR}/internal/proto.cc" + HDRS + "${DIR}/internal/proto.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::strings + absl::span +) + +absl_cc_library( + NAME + log_internal_message + SRCS + "${DIR}/internal/log_message.cc" + HDRS + "${DIR}/internal/log_message.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::errno_saver + absl::inlined_vector + absl::examine_stack + absl::log_internal_append_truncated + absl::log_internal_format + absl::log_internal_globals + absl::log_internal_proto + absl::log_internal_log_sink_set + absl::log_internal_nullguard + absl::log_globals + absl::log_entry + absl::log_severity + absl::log_sink + absl::log_sink_registry + absl::memory + absl::raw_logging_internal + absl::strings + absl::strerror + absl::time + absl::span +) + +absl_cc_library( + NAME + log_internal_log_sink_set + SRCS + "${DIR}/internal/log_sink_set.cc" + HDRS + "${DIR}/internal/log_sink_set.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + $<$:-llog> + DEPS + absl::base + absl::cleanup + absl::config + absl::core_headers + absl::log_internal_config + absl::log_internal_globals + absl::log_globals + absl::log_entry + absl::log_severity + absl::log_sink + absl::raw_logging_internal + absl::synchronization + absl::span + absl::strings +) + +absl_cc_library( + NAME + log_internal_nullguard + SRCS + "${DIR}/internal/nullguard.cc" + HDRS + "${DIR}/internal/nullguard.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers +) + +absl_cc_library( + NAME + log_internal_nullstream + SRCS + HDRS + "${DIR}/internal/nullstream.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_severity + absl::strings +) + +absl_cc_library( + NAME + log_internal_strip + SRCS + HDRS + "${DIR}/internal/strip.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::log_internal_message + absl::log_internal_nullstream + absl::log_severity +) + +absl_cc_library( + NAME + log_internal_voidify + SRCS + HDRS + "${DIR}/internal/voidify.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +absl_cc_library( + NAME + log_internal_append_truncated + SRCS + HDRS + "${DIR}/internal/append_truncated.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::strings + absl::span +) + +# Public targets +absl_cc_library( + NAME + absl_check + SRCS + HDRS + "${DIR}/absl_check.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::log_internal_check_impl + PUBLIC +) + +absl_cc_library( + NAME + absl_log + SRCS + HDRS + "${DIR}/absl_log.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::log_internal_log_impl + PUBLIC +) + +absl_cc_library( + NAME + check + SRCS + HDRS + "${DIR}/check.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::log_internal_check_impl + absl::core_headers + absl::log_internal_check_op + absl::log_internal_conditions + absl::log_internal_message + absl::log_internal_strip + PUBLIC +) + +absl_cc_library( + NAME + die_if_null + SRCS + "${DIR}/die_if_null.cc" + HDRS + "${DIR}/die_if_null.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log + absl::strings + PUBLIC +) + +absl_cc_library( + NAME + log_flags + SRCS + "${DIR}/flags.cc" + HDRS + "${DIR}/flags.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_globals + absl::log_severity + absl::log_internal_config + absl::log_internal_flags + absl::flags + absl::flags_marshalling + absl::strings + PUBLIC +) + +absl_cc_library( + NAME + log_globals + SRCS + "${DIR}/globals.cc" + HDRS + "${DIR}/globals.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::atomic_hook + absl::config + absl::core_headers + absl::hash + absl::log_severity + absl::raw_logging_internal + absl::strings +) + +absl_cc_library( + NAME + log_initialize + SRCS + "${DIR}/initialize.cc" + HDRS + "${DIR}/initialize.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::log_globals + absl::log_internal_globals + absl::time + PUBLIC +) + +absl_cc_library( + NAME + log + SRCS + HDRS + "${DIR}/log.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::log_internal_log_impl + PUBLIC +) + +absl_cc_library( + NAME + log_entry + SRCS + "${DIR}/log_entry.cc" + HDRS + "${DIR}/log_entry.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_internal_config + absl::log_severity + absl::span + absl::strings + absl::time + PUBLIC +) + +absl_cc_library( + NAME + log_sink + SRCS + "${DIR}/log_sink.cc" + HDRS + "${DIR}/log_sink.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::log_entry + PUBLIC +) + +absl_cc_library( + NAME + log_sink_registry + SRCS + HDRS + "${DIR}/log_sink_registry.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::log_sink + absl::log_internal_log_sink_set + PUBLIC +) + +absl_cc_library( + NAME + log_streamer + SRCS + HDRS + "${DIR}/log_streamer.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::absl_log + absl::log_severity + absl::optional + absl::strings + absl::strings_internal + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + log_internal_structured + HDRS + "${DIR}/internal/structured.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::log_internal_message + absl::strings +) + +absl_cc_library( + NAME + log_structured + HDRS + "${DIR}/structured.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::log_internal_structured + absl::strings + PUBLIC +) + +absl_cc_library( + NAME + log_internal_fnmatch + SRCS + "${DIR}/internal/fnmatch.cc" + HDRS + "${DIR}/internal/fnmatch.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::strings +) + +set(DIR ${ABSL_ROOT_DIR}/absl/memory) + +absl_cc_library( + NAME + memory + HDRS + "${DIR}/memory.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::meta + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/meta) + +absl_cc_library( + NAME + type_traits + HDRS + "${DIR}/type_traits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + PUBLIC +) + +# component target +absl_cc_library( + NAME + meta + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::type_traits + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/numeric) + +absl_cc_library( + NAME + bits + HDRS + "${DIR}/bits.h" + "${DIR}/internal/bits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + PUBLIC +) + +absl_cc_library( + NAME + int128 + HDRS + "${DIR}/int128.h" + SRCS + "${DIR}/int128.cc" + "${DIR}/int128_have_intrinsic.inc" + "${DIR}/int128_no_intrinsic.inc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::bits + PUBLIC +) + +# component target +absl_cc_library( + NAME + numeric + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::int128 + PUBLIC +) + +absl_cc_library( + NAME + numeric_representation + HDRS + "${DIR}/internal/representation.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +absl_cc_library( + NAME + sample_recorder + HDRS + "${DIR}/internal/sample_recorder.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::synchronization +) + +set(DIR ${ABSL_ROOT_DIR}/absl/profiling) + +absl_cc_library( + NAME + exponential_biased + SRCS + "${DIR}/internal/exponential_biased.cc" + HDRS + "${DIR}/internal/exponential_biased.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers +) + +absl_cc_library( + NAME + periodic_sampler + SRCS + "${DIR}/internal/periodic_sampler.cc" + HDRS + "${DIR}/internal/periodic_sampler.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::exponential_biased +) + +set(DIR ${ABSL_ROOT_DIR}/absl/random) + +absl_cc_library( + NAME + random_random + HDRS + "${DIR}/random.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_distributions + absl::random_internal_nonsecure_base + absl::random_internal_pcg_engine + absl::random_internal_pool_urbg + absl::random_internal_randen_engine + absl::random_seed_sequences +) + +absl_cc_library( + NAME + random_bit_gen_ref + HDRS + "${DIR}/bit_gen_ref.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::core_headers + absl::random_internal_distribution_caller + absl::random_internal_fast_uniform_bits + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_mock_helpers + HDRS + "${DIR}/internal/mock_helpers.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::fast_type_id + absl::optional +) + +absl_cc_library( + NAME + random_distributions + SRCS + "${DIR}/discrete_distribution.cc" + "${DIR}/gaussian_distribution.cc" + HDRS + "${DIR}/bernoulli_distribution.h" + "${DIR}/beta_distribution.h" + "${DIR}/discrete_distribution.h" + "${DIR}/distributions.h" + "${DIR}/exponential_distribution.h" + "${DIR}/gaussian_distribution.h" + "${DIR}/log_uniform_int_distribution.h" + "${DIR}/poisson_distribution.h" + "${DIR}/uniform_int_distribution.h" + "${DIR}/uniform_real_distribution.h" + "${DIR}/zipf_distribution.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base_internal + absl::config + absl::core_headers + absl::random_internal_generate_real + absl::random_internal_distribution_caller + absl::random_internal_fast_uniform_bits + absl::random_internal_fastmath + absl::random_internal_iostream_state_saver + absl::random_internal_traits + absl::random_internal_uniform_helper + absl::random_internal_wide_multiply + absl::strings + absl::type_traits +) + +absl_cc_library( + NAME + random_seed_gen_exception + SRCS + "${DIR}/seed_gen_exception.cc" + HDRS + "${DIR}/seed_gen_exception.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +absl_cc_library( + NAME + random_seed_sequences + SRCS + "${DIR}/seed_sequences.cc" + HDRS + "${DIR}/seed_sequences.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::inlined_vector + absl::random_internal_pool_urbg + absl::random_internal_salted_seed_seq + absl::random_internal_seed_material + absl::random_seed_gen_exception + absl::span +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_traits + HDRS + "${DIR}/internal/traits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_distribution_caller + HDRS + "${DIR}/internal/distribution_caller.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::utility + absl::fast_type_id +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_fast_uniform_bits + HDRS + "${DIR}/internal/fast_uniform_bits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_seed_material + SRCS + "${DIR}/internal/seed_material.cc" + HDRS + "${DIR}/internal/seed_material.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::core_headers + absl::optional + absl::random_internal_fast_uniform_bits + absl::raw_logging_internal + absl::span + absl::strings +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_pool_urbg + SRCS + "${DIR}/internal/pool_urbg.cc" + HDRS + "${DIR}/internal/pool_urbg.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::endian + absl::random_internal_randen + absl::random_internal_seed_material + absl::random_internal_traits + absl::random_seed_gen_exception + absl::raw_logging_internal + absl::span +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_salted_seed_seq + HDRS + "${DIR}/internal/salted_seed_seq.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::inlined_vector + absl::optional + absl::span + absl::random_internal_seed_material + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_iostream_state_saver + HDRS + "${DIR}/internal/iostream_state_saver.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::int128 + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_generate_real + HDRS + "${DIR}/internal/generate_real.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::bits + absl::random_internal_fastmath + absl::random_internal_traits + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_wide_multiply + HDRS + "${DIR}/internal/wide_multiply.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::bits + absl::config + absl::int128 +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_fastmath + HDRS + "${DIR}/internal/fastmath.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::bits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_nonsecure_base + HDRS + "${DIR}/internal/nonsecure_base.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::core_headers + absl::inlined_vector + absl::random_internal_pool_urbg + absl::random_internal_salted_seed_seq + absl::random_internal_seed_material + absl::span + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_pcg_engine + HDRS + "${DIR}/internal/pcg_engine.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::int128 + absl::random_internal_fastmath + absl::random_internal_iostream_state_saver + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen_engine + HDRS + "${DIR}/internal/randen_engine.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::endian + absl::random_internal_iostream_state_saver + absl::random_internal_randen + absl::raw_logging_internal + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_platform + HDRS + "${DIR}/internal/randen_traits.h" + "${DIR}/internal/platform.h" + SRCS + "${DIR}/internal/randen_round_keys.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen + SRCS + "${DIR}/internal/randen.cc" + HDRS + "${DIR}/internal/randen.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_platform + absl::random_internal_randen_hwaes + absl::random_internal_randen_slow +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen_slow + SRCS + "${DIR}/internal/randen_slow.cc" + HDRS + "${DIR}/internal/randen_slow.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_platform + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen_hwaes + SRCS + "${DIR}/internal/randen_detect.cc" + HDRS + "${DIR}/internal/randen_detect.h" + "${DIR}/internal/randen_hwaes.h" + COPTS + ${ABSL_DEFAULT_COPTS} + ${ABSL_RANDOM_RANDEN_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_platform + absl::random_internal_randen_hwaes_impl + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen_hwaes_impl + SRCS + "${DIR}/internal/randen_hwaes.cc" + "${DIR}/internal/randen_hwaes.h" + COPTS + ${ABSL_DEFAULT_COPTS} + ${ABSL_RANDOM_RANDEN_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_platform + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_uniform_helper + HDRS + "${DIR}/internal/uniform_helper.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::random_internal_traits + absl::type_traits +) + +set(DIR ${ABSL_ROOT_DIR}/absl/status) + +absl_cc_library( + NAME + status + HDRS + "${DIR}/status.h" + SRCS + "${DIR}/internal/status_internal.h" + "${DIR}/status.cc" + "${DIR}/status_payload_printer.h" + "${DIR}/status_payload_printer.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEFINES + "$<$:_LINUX_SOURCE_COMPAT>" + DEPS + absl::atomic_hook + absl::config + absl::cord + absl::core_headers + absl::function_ref + absl::inlined_vector + absl::memory + absl::optional + absl::raw_logging_internal + absl::span + absl::stacktrace + absl::strerror + absl::str_format + absl::strings + absl::symbolize + PUBLIC +) + +absl_cc_library( + NAME + statusor + HDRS + "${DIR}/statusor.h" + SRCS + "${DIR}/statusor.cc" + "${DIR}/internal/statusor_internal.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::raw_logging_internal + absl::status + absl::strings + absl::type_traits + absl::utility + absl::variant + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/strings) + +absl_cc_library( + NAME + string_view + HDRS + "${DIR}/string_view.h" + SRCS + "${DIR}/string_view.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::throw_delegate + PUBLIC +) + +absl_cc_library( + NAME + strings + HDRS + "${DIR}/ascii.h" + "${DIR}/charconv.h" + "${DIR}/escaping.h" + "${DIR}/has_absl_stringify.h" + "${DIR}/internal/damerau_levenshtein_distance.h" + "${DIR}/internal/string_constant.h" + "${DIR}/internal/has_absl_stringify.h" + "${DIR}/match.h" + "${DIR}/numbers.h" + "${DIR}/str_cat.h" + "${DIR}/str_join.h" + "${DIR}/str_replace.h" + "${DIR}/str_split.h" + "${DIR}/strip.h" + "${DIR}/substitute.h" + SRCS + "${DIR}/ascii.cc" + "${DIR}/charconv.cc" + "${DIR}/escaping.cc" + "${DIR}/internal/charconv_bigint.cc" + "${DIR}/internal/charconv_bigint.h" + "${DIR}/internal/charconv_parse.cc" + "${DIR}/internal/charconv_parse.h" + "${DIR}/internal/damerau_levenshtein_distance.cc" + "${DIR}/internal/memutil.cc" + "${DIR}/internal/memutil.h" + "${DIR}/internal/stringify_sink.h" + "${DIR}/internal/stringify_sink.cc" + "${DIR}/internal/stl_type_traits.h" + "${DIR}/internal/str_join_internal.h" + "${DIR}/internal/str_split_internal.h" + "${DIR}/match.cc" + "${DIR}/numbers.cc" + "${DIR}/str_cat.cc" + "${DIR}/str_replace.cc" + "${DIR}/str_split.cc" + "${DIR}/substitute.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::string_view + absl::strings_internal + absl::base + absl::bits + absl::charset + absl::config + absl::core_headers + absl::endian + absl::int128 + absl::memory + absl::raw_logging_internal + absl::throw_delegate + absl::type_traits + PUBLIC +) + +absl_cc_library( + NAME + charset + HDRS + charset.h + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::string_view + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + strings_internal + HDRS + "${DIR}/internal/escaping.cc" + "${DIR}/internal/escaping.h" + "${DIR}/internal/ostringstream.h" + "${DIR}/internal/resize_uninitialized.h" + "${DIR}/internal/utf8.h" + SRCS + "${DIR}/internal/ostringstream.cc" + "${DIR}/internal/utf8.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::endian + absl::raw_logging_internal + absl::type_traits +) + +absl_cc_library( + NAME + str_format + HDRS + "${DIR}/str_format.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::str_format_internal + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + str_format_internal + HDRS + "${DIR}/internal/str_format/arg.h" + "${DIR}/internal/str_format/bind.h" + "${DIR}/internal/str_format/checker.h" + "${DIR}/internal/str_format/constexpr_parser.h" + "${DIR}/internal/str_format/extension.h" + "${DIR}/internal/str_format/float_conversion.h" + "${DIR}/internal/str_format/output.h" + "${DIR}/internal/str_format/parser.h" + SRCS + "${DIR}/internal/str_format/arg.cc" + "${DIR}/internal/str_format/bind.cc" + "${DIR}/internal/str_format/extension.cc" + "${DIR}/internal/str_format/float_conversion.cc" + "${DIR}/internal/str_format/output.cc" + "${DIR}/internal/str_format/parser.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bits + absl::strings + absl::config + absl::core_headers + absl::inlined_vector + absl::numeric_representation + absl::type_traits + absl::utility + absl::int128 + absl::span +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cord_internal + HDRS + "${DIR}/internal/cord_data_edge.h" + "${DIR}/internal/cord_internal.h" + "${DIR}/internal/cord_rep_btree.h" + "${DIR}/internal/cord_rep_btree_navigator.h" + "${DIR}/internal/cord_rep_btree_reader.h" + "${DIR}/internal/cord_rep_crc.h" + "${DIR}/internal/cord_rep_consume.h" + "${DIR}/internal/cord_rep_flat.h" + SRCS + "${DIR}/internal/cord_internal.cc" + "${DIR}/internal/cord_rep_btree.cc" + "${DIR}/internal/cord_rep_btree_navigator.cc" + "${DIR}/internal/cord_rep_btree_reader.cc" + "${DIR}/internal/cord_rep_crc.cc" + "${DIR}/internal/cord_rep_consume.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::compressed_tuple + absl::config + absl::container_memory + absl::core_headers + absl::crc_cord_state + absl::endian + absl::inlined_vector + absl::layout + absl::raw_logging_internal + absl::strings + absl::throw_delegate + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_update_tracker + HDRS + "${DIR}/internal/cordz_update_tracker.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_functions + HDRS + "${DIR}/internal/cordz_functions.h" + SRCS + "${DIR}/internal/cordz_functions.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::exponential_biased + absl::raw_logging_internal +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_statistics + HDRS + "${DIR}/internal/cordz_statistics.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::cordz_update_tracker + absl::synchronization +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_handle + HDRS + "${DIR}/internal/cordz_handle.h" + SRCS + "${DIR}/internal/cordz_handle.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::raw_logging_internal + absl::synchronization +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_info + HDRS + "${DIR}/internal/cordz_info.h" + SRCS + "${DIR}/internal/cordz_info.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::cord_internal + absl::cordz_functions + absl::cordz_handle + absl::cordz_statistics + absl::cordz_update_tracker + absl::core_headers + absl::inlined_vector + absl::span + absl::raw_logging_internal + absl::stacktrace + absl::synchronization + absl::time +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_sample_token + HDRS + "${DIR}/internal/cordz_sample_token.h" + SRCS + "${DIR}/internal/cordz_sample_token.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::cordz_handle + absl::cordz_info +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_update_scope + HDRS + "${DIR}/internal/cordz_update_scope.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::cord_internal + absl::cordz_info + absl::cordz_update_tracker + absl::core_headers +) + +absl_cc_library( + NAME + cord + HDRS + "${DIR}/cord.h" + "${DIR}/cord_buffer.h" + SRCS + "${DIR}/cord.cc" + "${DIR}/cord_analysis.cc" + "${DIR}/cord_analysis.h" + "${DIR}/cord_buffer.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::cord_internal + absl::cordz_functions + absl::cordz_info + absl::cordz_update_scope + absl::cordz_update_tracker + absl::core_headers + absl::crc32c + absl::crc_cord_state + absl::endian + absl::function_ref + absl::inlined_vector + absl::optional + absl::raw_logging_internal + absl::span + absl::strings + absl::type_traits + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/synchronization) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + graphcycles_internal + HDRS + "${DIR}/internal/graphcycles.h" + SRCS + "${DIR}/internal/graphcycles.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::base_internal + absl::config + absl::core_headers + absl::malloc_internal + absl::raw_logging_internal +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + kernel_timeout_internal + HDRS + "${DIR}/internal/kernel_timeout.h" + SRCS + "${DIR}/internal/kernel_timeout.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::raw_logging_internal + absl::time +) + +absl_cc_library( + NAME + synchronization + HDRS + "${DIR}/barrier.h" + "${DIR}/blocking_counter.h" + "${DIR}/internal/create_thread_identity.h" + "${DIR}/internal/futex.h" + "${DIR}/internal/futex_waiter.h" + "${DIR}/internal/per_thread_sem.h" + "${DIR}/internal/pthread_waiter.h" + "${DIR}/internal/sem_waiter.h" + "${DIR}/internal/stdcpp_waiter.h" + "${DIR}/internal/waiter.h" + "${DIR}/internal/waiter_base.h" + "${DIR}/internal/win32_waiter.h" + "${DIR}/mutex.h" + "${DIR}/notification.h" + SRCS + "${DIR}/barrier.cc" + "${DIR}/blocking_counter.cc" + "${DIR}/internal/create_thread_identity.cc" + "${DIR}/internal/futex_waiter.cc" + "${DIR}/internal/per_thread_sem.cc" + "${DIR}/internal/pthread_waiter.cc" + "${DIR}/internal/sem_waiter.cc" + "${DIR}/internal/stdcpp_waiter.cc" + "${DIR}/internal/waiter_base.cc" + "${DIR}/internal/win32_waiter.cc" + "${DIR}/notification.cc" + "${DIR}/mutex.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::graphcycles_internal + absl::kernel_timeout_internal + absl::atomic_hook + absl::base + absl::base_internal + absl::config + absl::core_headers + absl::dynamic_annotations + absl::malloc_internal + absl::raw_logging_internal + absl::stacktrace + absl::symbolize + absl::time + Threads::Threads + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/time) + +absl_cc_library( + NAME + time + HDRS + "${DIR}/civil_time.h" + "${DIR}/clock.h" + "${DIR}/time.h" + SRCS + "${DIR}/civil_time.cc" + "${DIR}/clock.cc" + "${DIR}/duration.cc" + "${DIR}/format.cc" + "${DIR}/internal/get_current_time_chrono.inc" + "${DIR}/internal/get_current_time_posix.inc" + "${DIR}/time.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::civil_time + absl::core_headers + absl::int128 + absl::raw_logging_internal + absl::strings + absl::time_zone + PUBLIC +) + +absl_cc_library( + NAME + civil_time + HDRS + "${DIR}/internal/cctz/include/cctz/civil_time.h" + "${DIR}/internal/cctz/include/cctz/civil_time_detail.h" + SRCS + "${DIR}/internal/cctz/src/civil_time_detail.cc" + COPTS + ${ABSL_DEFAULT_COPTS} +) + +absl_cc_library( + NAME + time_zone + HDRS + "${DIR}/internal/cctz/include/cctz/time_zone.h" + "${DIR}/internal/cctz/include/cctz/zone_info_source.h" + SRCS + "${DIR}/internal/cctz/src/time_zone_fixed.cc" + "${DIR}/internal/cctz/src/time_zone_fixed.h" + "${DIR}/internal/cctz/src/time_zone_format.cc" + "${DIR}/internal/cctz/src/time_zone_if.cc" + "${DIR}/internal/cctz/src/time_zone_if.h" + "${DIR}/internal/cctz/src/time_zone_impl.cc" + "${DIR}/internal/cctz/src/time_zone_impl.h" + "${DIR}/internal/cctz/src/time_zone_info.cc" + "${DIR}/internal/cctz/src/time_zone_info.h" + "${DIR}/internal/cctz/src/time_zone_libc.cc" + "${DIR}/internal/cctz/src/time_zone_libc.h" + "${DIR}/internal/cctz/src/time_zone_lookup.cc" + "${DIR}/internal/cctz/src/time_zone_posix.cc" + "${DIR}/internal/cctz/src/time_zone_posix.h" + "${DIR}/internal/cctz/src/tzfile.h" + "${DIR}/internal/cctz/src/zone_info_source.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + Threads::Threads + $<$:-Wl,-framework,CoreFoundation> +) + +set(DIR ${ABSL_ROOT_DIR}/absl/types) + +absl_cc_library( + NAME + any + HDRS + "${DIR}/any.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bad_any_cast + absl::config + absl::core_headers + absl::fast_type_id + absl::type_traits + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + bad_any_cast + HDRS + "${DIR}/bad_any_cast.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bad_any_cast_impl + absl::config + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + bad_any_cast_impl + SRCS + "${DIR}/bad_any_cast.h" + "${DIR}/bad_any_cast.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal +) + +absl_cc_library( + NAME + span + HDRS + "${DIR}/span.h" + SRCS + "${DIR}/internal/span.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::algorithm + absl::core_headers + absl::throw_delegate + absl::type_traits + PUBLIC +) + +absl_cc_library( + NAME + optional + HDRS + "${DIR}/optional.h" + SRCS + "${DIR}/internal/optional.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bad_optional_access + absl::base_internal + absl::config + absl::core_headers + absl::memory + absl::type_traits + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + bad_optional_access + HDRS + "${DIR}/bad_optional_access.h" + SRCS + "${DIR}/bad_optional_access.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal + PUBLIC +) + +absl_cc_library( + NAME + bad_variant_access + HDRS + "${DIR}/bad_variant_access.h" + SRCS + "${DIR}/bad_variant_access.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal + PUBLIC +) + +absl_cc_library( + NAME + variant + HDRS + "${DIR}/variant.h" + SRCS + "${DIR}/internal/variant.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bad_variant_access + absl::base_internal + absl::config + absl::core_headers + absl::type_traits + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + compare + HDRS + "${DIR}/compare.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::type_traits + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/utility) + +absl_cc_library( + NAME + utility + HDRS + "${DIR}/utility.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::config + absl::type_traits + PUBLIC +) + +absl_cc_library( + NAME + if_constexpr + HDRS + "${DIR}/internal/if_constexpr.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) -set(ABSL_PROPAGATE_CXX_STD ON) -add_subdirectory("${ABSL_ROOT_DIR}" "${ClickHouse_BINARY_DIR}/contrib/abseil-cpp") add_library(_abseil_swiss_tables INTERFACE) - -target_link_libraries(_abseil_swiss_tables INTERFACE - absl::flat_hash_map - absl::flat_hash_set -) - -get_target_property(FLAT_HASH_MAP_INCLUDE_DIR absl::flat_hash_map INTERFACE_INCLUDE_DIRECTORIES) -target_include_directories (_abseil_swiss_tables SYSTEM BEFORE INTERFACE ${FLAT_HASH_MAP_INCLUDE_DIR}) - -get_target_property(FLAT_HASH_SET_INCLUDE_DIR absl::flat_hash_set INTERFACE_INCLUDE_DIRECTORIES) -target_include_directories (_abseil_swiss_tables SYSTEM BEFORE INTERFACE ${FLAT_HASH_SET_INCLUDE_DIR}) - +target_include_directories (_abseil_swiss_tables SYSTEM BEFORE INTERFACE ${ABSL_ROOT_DIR}) add_library(ch_contrib::abseil_swiss_tables ALIAS _abseil_swiss_tables) - -set(ABSL_FORMAT_SRC - ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/arg.cc - ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/bind.cc - ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/extension.cc - ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/float_conversion.cc - ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/output.cc - ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/parser.cc -) - -add_library(_abseil_str_format ${ABSL_FORMAT_SRC}) -target_include_directories(_abseil_str_format PUBLIC ${ABSL_ROOT_DIR}) - -add_library(ch_contrib::abseil_str_format ALIAS _abseil_str_format) diff --git a/contrib/re2-cmake/CMakeLists.txt b/contrib/re2-cmake/CMakeLists.txt index e72b5e1fca8..f773bc65a69 100644 --- a/contrib/re2-cmake/CMakeLists.txt +++ b/contrib/re2-cmake/CMakeLists.txt @@ -27,6 +27,17 @@ set(RE2_SOURCES add_library(_re2 ${RE2_SOURCES}) target_include_directories(_re2 PUBLIC "${SRC_DIR}") -target_link_libraries(_re2 ch_contrib::abseil_str_format) +target_link_libraries(_re2 PRIVATE + absl::base + absl::core_headers + absl::fixed_array + absl::flat_hash_map + absl::flat_hash_set + absl::inlined_vector + absl::strings + absl::str_format + absl::synchronization + absl::optional + absl::span) add_library(ch_contrib::re2 ALIAS _re2) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 42bfb48db70..c764a4dd8c1 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -38,7 +38,7 @@ rm -f CMakeCache.txt # To check it, find and delete them. grep -o -P '"contrib/[^"]+"' ../.gitmodules | - grep -v -P 'llvm-project|abseil-cpp|grpc|corrosion' | + grep -v -P 'llvm-project|grpc|corrosion' | xargs -I@ find ../@ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | xargs rm From 0548eefbb784de32ec7e409ca9150482f3a171a9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 11:35:48 +0100 Subject: [PATCH 130/192] Simpler CMake --- docker/packager/binary/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index c764a4dd8c1..37440fe8202 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -38,7 +38,7 @@ rm -f CMakeCache.txt # To check it, find and delete them. grep -o -P '"contrib/[^"]+"' ../.gitmodules | - grep -v -P 'llvm-project|grpc|corrosion' | + grep -v -P 'llvm-project|google-protobuf|grpc|corrosion' | xargs -I@ find ../@ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | xargs rm From 74a8f3191dc96f3ac46187b556b5d127b7ae6030 Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:38:56 +0100 Subject: [PATCH 131/192] Update HTTPSession.cpp --- base/poco/Net/src/HTTPSession.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index d303a4c654b..8f951b3102c 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -113,13 +113,12 @@ void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco } catch (NetException &) { - #ifndef NDEBUG + throw; +#else // mute exceptions in release // just in case when changing settings on socket is not allowed // however it should be OK for timeouts -#else - throw; #endif } } From d035e5f44fc8820a0aef96b30106ed9e78bb6408 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Thu, 16 Nov 2023 17:41:05 +0100 Subject: [PATCH 132/192] Improve diagnostics in test 02908_many_requests_to_system_replicas --- .../02908_many_requests_to_system_replicas.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh index 70dc5f4d8c4..c620fcf4bea 100755 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh @@ -14,12 +14,13 @@ echo "Creating $NUM_TABLES tables" function init_table() { + set -e i=$1 - curl $CLICKHOUSE_URL --silent --fail --data "CREATE TABLE test_02908_r1_$i (a UInt64) ENGINE=ReplicatedMergeTree('/02908/{database}/test_$i', 'r1') ORDER BY tuple()" - curl $CLICKHOUSE_URL --silent --fail --data "CREATE TABLE test_02908_r2_$i (a UInt64) ENGINE=ReplicatedMergeTree('/02908/{database}/test_$i', 'r2') ORDER BY tuple()" - curl $CLICKHOUSE_URL --silent --fail --data "CREATE TABLE test_02908_r3_$i (a UInt64) ENGINE=ReplicatedMergeTree('/02908/{database}/test_$i', 'r3') ORDER BY tuple()" + curl $CLICKHOUSE_URL --silent --fail --show-error --data "CREATE TABLE test_02908_r1_$i (a UInt64) ENGINE=ReplicatedMergeTree('/02908/{database}/test_$i', 'r1') ORDER BY tuple()" 2>&1 + curl $CLICKHOUSE_URL --silent --fail --show-error --data "CREATE TABLE test_02908_r2_$i (a UInt64) ENGINE=ReplicatedMergeTree('/02908/{database}/test_$i', 'r2') ORDER BY tuple()" 2>&1 + curl $CLICKHOUSE_URL --silent --fail --show-error --data "CREATE TABLE test_02908_r3_$i (a UInt64) ENGINE=ReplicatedMergeTree('/02908/{database}/test_$i', 'r3') ORDER BY tuple()" 2>&1 - curl $CLICKHOUSE_URL --silent --fail --data "INSERT INTO test_02908_r1_$i SELECT rand64() FROM numbers(5);" + curl $CLICKHOUSE_URL --silent --fail --show-error --data "INSERT INTO test_02908_r1_$i SELECT rand64() FROM numbers(5);" 2>&1 } export init_table; @@ -36,13 +37,13 @@ echo "Making making $CONCURRENCY requests to system.replicas" for i in `seq 1 $CONCURRENCY`; do - curl $CLICKHOUSE_URL --silent --fail --data "SELECT * FROM system.replicas WHERE database=currentDatabase() FORMAT Null;" & + curl $CLICKHOUSE_URL --silent --fail --show-error --data "SELECT * FROM system.replicas WHERE database=currentDatabase() FORMAT Null;" 2>&1 || echo "query $i failed" & done echo "Query system.replicas while waiting for other concurrent requests to finish" # lost_part_count column is read from ZooKeeper -curl $CLICKHOUSE_URL --silent --fail --data "SELECT sum(lost_part_count) FROM system.replicas WHERE database=currentDatabase();"; +curl $CLICKHOUSE_URL --silent --fail --show-error --data "SELECT sum(lost_part_count) FROM system.replicas WHERE database=currentDatabase();" 2>&1; # is_leader column is filled without ZooKeeper -curl $CLICKHOUSE_URL --silent --fail --data "SELECT sum(is_leader) FROM system.replicas WHERE database=currentDatabase();"; +curl $CLICKHOUSE_URL --silent --fail --show-error --data "SELECT sum(is_leader) FROM system.replicas WHERE database=currentDatabase();" 2>&1; wait; From cdfc0e07e5161dc2d381c0ad74205c0514e94850 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 17 Nov 2023 11:06:03 +0000 Subject: [PATCH 133/192] Update version_date.tsv and changelogs after v23.10.4.25-stable --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v23.10.4.25-stable.md | 28 +++++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 2 ++ 5 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v23.10.4.25-stable.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 1f4fd39bc26..63de9f6c462 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 41be7e611a3..d26bb344fef 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 0ff6ae2e227..53a36818121 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -30,7 +30,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v23.10.4.25-stable.md b/docs/changelogs/v23.10.4.25-stable.md new file mode 100644 index 00000000000..2d7d2a38e04 --- /dev/null +++ b/docs/changelogs/v23.10.4.25-stable.md @@ -0,0 +1,28 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.10.4.25-stable (330fd687d41) FIXME as compared to v23.10.3.5-stable (b2ba7637a41) + +#### Build/Testing/Packaging Improvement +* Backported in [#56633](https://github.com/ClickHouse/ClickHouse/issues/56633): In [#54043](https://github.com/ClickHouse/ClickHouse/issues/54043) the setup plan started to appear in the logs. It should be only in the `runner_get_all_tests.log` only. As well, send the failed infrastructure event to CI db. [#56214](https://github.com/ClickHouse/ClickHouse/pull/56214) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#56737](https://github.com/ClickHouse/ClickHouse/issues/56737): Do not fetch changed submodules in the builder container. [#56689](https://github.com/ClickHouse/ClickHouse/pull/56689) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Select from system tables when table based on table function. [#55540](https://github.com/ClickHouse/ClickHouse/pull/55540) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Fix restore from backup with `flatten_nested` and `data_type_default_nullable` [#56306](https://github.com/ClickHouse/ClickHouse/pull/56306) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix segfault during Kerberos initialization [#56401](https://github.com/ClickHouse/ClickHouse/pull/56401) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix: RabbitMQ OpenSSL dynamic loading issue [#56703](https://github.com/ClickHouse/ClickHouse/pull/56703) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix crash in GCD codec in case when zeros present in data [#56704](https://github.com/ClickHouse/ClickHouse/pull/56704) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix crash in FPC codec [#56795](https://github.com/ClickHouse/ClickHouse/pull/56795) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Rewrite jobs to use callable workflow [#56385](https://github.com/ClickHouse/ClickHouse/pull/56385) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Continue rewriting workflows to reusable tests [#56501](https://github.com/ClickHouse/ClickHouse/pull/56501) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Better exception messages [#56854](https://github.com/ClickHouse/ClickHouse/pull/56854) ([Antonio Andelic](https://github.com/antonio2368)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 0f2684cd91d..ace5546aadb 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v23.10.4.25-stable 2023-11-17 v23.10.3.5-stable 2023-11-10 v23.10.2.13-stable 2023-11-08 v23.10.1.1976-stable 2023-11-02 @@ -31,6 +32,7 @@ v23.4.4.16-stable 2023-06-17 v23.4.3.48-stable 2023-06-12 v23.4.2.11-stable 2023-05-02 v23.4.1.1943-stable 2023-04-27 +v23.3.17.13-lts 2023-11-17 v23.3.16.7-lts 2023-11-08 v23.3.15.29-lts 2023-10-31 v23.3.14.78-lts 2023-10-18 From c695405c85454e5f0266342fdbe02ab000a670cb Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 17 Nov 2023 11:08:05 +0000 Subject: [PATCH 134/192] Update version_date.tsv and changelogs after v23.3.17.13-lts --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v23.3.17.13-lts.md | 23 +++++++++++++++++++++++ utils/list-versions/version_date.tsv | 2 ++ 5 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v23.3.17.13-lts.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 1f4fd39bc26..63de9f6c462 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 41be7e611a3..d26bb344fef 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 0ff6ae2e227..53a36818121 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -30,7 +30,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v23.3.17.13-lts.md b/docs/changelogs/v23.3.17.13-lts.md new file mode 100644 index 00000000000..a18ced70d46 --- /dev/null +++ b/docs/changelogs/v23.3.17.13-lts.md @@ -0,0 +1,23 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.3.17.13-lts (e867d59020f) FIXME as compared to v23.3.16.7-lts (fb4125cc92a) + +#### Build/Testing/Packaging Improvement +* Backported in [#56731](https://github.com/ClickHouse/ClickHouse/issues/56731): Do not fetch changed submodules in the builder container. [#56689](https://github.com/ClickHouse/ClickHouse/pull/56689) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix segfault during Kerberos initialization [#56401](https://github.com/ClickHouse/ClickHouse/pull/56401) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix crash in FPC codec [#56795](https://github.com/ClickHouse/ClickHouse/pull/56795) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Rewrite jobs to use callable workflow [#56385](https://github.com/ClickHouse/ClickHouse/pull/56385) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Continue rewriting workflows to reusable tests [#56501](https://github.com/ClickHouse/ClickHouse/pull/56501) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Better exception messages [#56854](https://github.com/ClickHouse/ClickHouse/pull/56854) ([Antonio Andelic](https://github.com/antonio2368)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 0f2684cd91d..ace5546aadb 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v23.10.4.25-stable 2023-11-17 v23.10.3.5-stable 2023-11-10 v23.10.2.13-stable 2023-11-08 v23.10.1.1976-stable 2023-11-02 @@ -31,6 +32,7 @@ v23.4.4.16-stable 2023-06-17 v23.4.3.48-stable 2023-06-12 v23.4.2.11-stable 2023-05-02 v23.4.1.1943-stable 2023-04-27 +v23.3.17.13-lts 2023-11-17 v23.3.16.7-lts 2023-11-08 v23.3.15.29-lts 2023-10-31 v23.3.14.78-lts 2023-10-18 From 09b3f5e541935a1dd15ae1cab056ae680cec46ed Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 17 Nov 2023 11:10:10 +0000 Subject: [PATCH 135/192] Update version_date.tsv and changelogs after v23.8.7.24-lts --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v23.8.7.24-lts.md | 31 ++++++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 3 +++ 5 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v23.8.7.24-lts.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 1f4fd39bc26..63de9f6c462 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 41be7e611a3..d26bb344fef 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 0ff6ae2e227..53a36818121 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -30,7 +30,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v23.8.7.24-lts.md b/docs/changelogs/v23.8.7.24-lts.md new file mode 100644 index 00000000000..37862c17315 --- /dev/null +++ b/docs/changelogs/v23.8.7.24-lts.md @@ -0,0 +1,31 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.8.7.24-lts (812b95e14ba) FIXME as compared to v23.8.6.16-lts (077df679bed) + +#### Build/Testing/Packaging Improvement +* Backported in [#56733](https://github.com/ClickHouse/ClickHouse/issues/56733): Do not fetch changed submodules in the builder container. [#56689](https://github.com/ClickHouse/ClickHouse/pull/56689) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Select from system tables when table based on table function. [#55540](https://github.com/ClickHouse/ClickHouse/pull/55540) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Fix incomplete query result for UNION in view() function. [#56274](https://github.com/ClickHouse/ClickHouse/pull/56274) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in case of adding a column with type Object(JSON) [#56307](https://github.com/ClickHouse/ClickHouse/pull/56307) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix segfault during Kerberos initialization [#56401](https://github.com/ClickHouse/ClickHouse/pull/56401) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix: RabbitMQ OpenSSL dynamic loading issue [#56703](https://github.com/ClickHouse/ClickHouse/pull/56703) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix crash in FPC codec [#56795](https://github.com/ClickHouse/ClickHouse/pull/56795) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NO CL CATEGORY + +* Backported in [#56601](https://github.com/ClickHouse/ClickHouse/issues/56601):. [#56598](https://github.com/ClickHouse/ClickHouse/pull/56598) ([Maksim Kita](https://github.com/kitaisreal)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Rewrite jobs to use callable workflow [#56385](https://github.com/ClickHouse/ClickHouse/pull/56385) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Continue rewriting workflows to reusable tests [#56501](https://github.com/ClickHouse/ClickHouse/pull/56501) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Better exception messages [#56854](https://github.com/ClickHouse/ClickHouse/pull/56854) ([Antonio Andelic](https://github.com/antonio2368)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 0f2684cd91d..c0b9fd22964 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v23.10.4.25-stable 2023-11-17 v23.10.3.5-stable 2023-11-10 v23.10.2.13-stable 2023-11-08 v23.10.1.1976-stable 2023-11-02 @@ -5,6 +6,7 @@ v23.9.4.11-stable 2023-11-08 v23.9.3.12-stable 2023-10-31 v23.9.2.56-stable 2023-10-19 v23.9.1.1854-stable 2023-09-29 +v23.8.7.24-lts 2023-11-17 v23.8.6.16-lts 2023-11-08 v23.8.5.16-lts 2023-10-31 v23.8.4.69-lts 2023-10-19 @@ -31,6 +33,7 @@ v23.4.4.16-stable 2023-06-17 v23.4.3.48-stable 2023-06-12 v23.4.2.11-stable 2023-05-02 v23.4.1.1943-stable 2023-04-27 +v23.3.17.13-lts 2023-11-17 v23.3.16.7-lts 2023-11-08 v23.3.15.29-lts 2023-10-31 v23.3.14.78-lts 2023-10-18 From af2f06db5282fa801b82d1b9def56284f49ce77c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 17 Nov 2023 11:12:16 +0000 Subject: [PATCH 136/192] Update version_date.tsv and changelogs after v23.9.5.29-stable --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v23.9.5.29-stable.md | 34 ++++++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 4 ++++ 5 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v23.9.5.29-stable.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 1f4fd39bc26..63de9f6c462 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 41be7e611a3..d26bb344fef 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 0ff6ae2e227..53a36818121 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -30,7 +30,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v23.9.5.29-stable.md b/docs/changelogs/v23.9.5.29-stable.md new file mode 100644 index 00000000000..02572d0e562 --- /dev/null +++ b/docs/changelogs/v23.9.5.29-stable.md @@ -0,0 +1,34 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.9.5.29-stable (f8554c1a1ff) FIXME as compared to v23.9.4.11-stable (74c1f49dd6a) + +#### Build/Testing/Packaging Improvement +* Backported in [#56631](https://github.com/ClickHouse/ClickHouse/issues/56631): In [#54043](https://github.com/ClickHouse/ClickHouse/issues/54043) the setup plan started to appear in the logs. It should be only in the `runner_get_all_tests.log` only. As well, send the failed infrastructure event to CI db. [#56214](https://github.com/ClickHouse/ClickHouse/pull/56214) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#56735](https://github.com/ClickHouse/ClickHouse/issues/56735): Do not fetch changed submodules in the builder container. [#56689](https://github.com/ClickHouse/ClickHouse/pull/56689) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Select from system tables when table based on table function. [#55540](https://github.com/ClickHouse/ClickHouse/pull/55540) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Fix incomplete query result for UNION in view() function. [#56274](https://github.com/ClickHouse/ClickHouse/pull/56274) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in case of adding a column with type Object(JSON) [#56307](https://github.com/ClickHouse/ClickHouse/pull/56307) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix segfault during Kerberos initialization [#56401](https://github.com/ClickHouse/ClickHouse/pull/56401) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix: RabbitMQ OpenSSL dynamic loading issue [#56703](https://github.com/ClickHouse/ClickHouse/pull/56703) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix crash in GCD codec in case when zeros present in data [#56704](https://github.com/ClickHouse/ClickHouse/pull/56704) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix crash in FPC codec [#56795](https://github.com/ClickHouse/ClickHouse/pull/56795) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NO CL CATEGORY + +* Backported in [#56603](https://github.com/ClickHouse/ClickHouse/issues/56603):. [#56598](https://github.com/ClickHouse/ClickHouse/pull/56598) ([Maksim Kita](https://github.com/kitaisreal)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Improve enrich image [#55793](https://github.com/ClickHouse/ClickHouse/pull/55793) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Rewrite jobs to use callable workflow [#56385](https://github.com/ClickHouse/ClickHouse/pull/56385) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Continue rewriting workflows to reusable tests [#56501](https://github.com/ClickHouse/ClickHouse/pull/56501) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Better exception messages [#56854](https://github.com/ClickHouse/ClickHouse/pull/56854) ([Antonio Andelic](https://github.com/antonio2368)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 0f2684cd91d..014ee5e9a17 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,10 +1,13 @@ +v23.10.4.25-stable 2023-11-17 v23.10.3.5-stable 2023-11-10 v23.10.2.13-stable 2023-11-08 v23.10.1.1976-stable 2023-11-02 +v23.9.5.29-stable 2023-11-17 v23.9.4.11-stable 2023-11-08 v23.9.3.12-stable 2023-10-31 v23.9.2.56-stable 2023-10-19 v23.9.1.1854-stable 2023-09-29 +v23.8.7.24-lts 2023-11-17 v23.8.6.16-lts 2023-11-08 v23.8.5.16-lts 2023-10-31 v23.8.4.69-lts 2023-10-19 @@ -31,6 +34,7 @@ v23.4.4.16-stable 2023-06-17 v23.4.3.48-stable 2023-06-12 v23.4.2.11-stable 2023-05-02 v23.4.1.1943-stable 2023-04-27 +v23.3.17.13-lts 2023-11-17 v23.3.16.7-lts 2023-11-08 v23.3.15.29-lts 2023-10-31 v23.3.14.78-lts 2023-10-18 From d6fdfdd45f590860a12023c25b2af735f2853796 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 12:16:54 +0100 Subject: [PATCH 137/192] Remove outdated instructions --- tests/instructions/coverity.txt | 28 ----------- tests/instructions/cppcheck.txt | 22 --------- tests/instructions/heap-profiler.txt | 14 ------ tests/instructions/kafka.txt | 45 ----------------- tests/instructions/sanitizers.md | 72 ---------------------------- tests/instructions/syntax.txt | 5 -- tests/instructions/tscancode.txt | 26 ---------- 7 files changed, 212 deletions(-) delete mode 100644 tests/instructions/coverity.txt delete mode 100644 tests/instructions/cppcheck.txt delete mode 100644 tests/instructions/heap-profiler.txt delete mode 100644 tests/instructions/kafka.txt delete mode 100644 tests/instructions/sanitizers.md delete mode 100644 tests/instructions/syntax.txt delete mode 100644 tests/instructions/tscancode.txt diff --git a/tests/instructions/coverity.txt b/tests/instructions/coverity.txt deleted file mode 100644 index f8d6d68d326..00000000000 --- a/tests/instructions/coverity.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Download tool at https://scan.coverity.com/download?tab=cxx - -tar xf cov-analysis-linux64-2017.07.tar.gz -export PATH=$PATH:/home/milovidov/cov-analysis-linux64-2017.07/bin - -mkdir ClickHouse_coverity -cd ClickHouse_coverity -git clone --recursive git@github.com:yandex/ClickHouse.git . - -mkdir build -cd build - -# "Debug" is for faster build -CC=gcc-7 CXX=g++-7 cmake -D CMAKE_BUILD_TYPE=Debug -D CCACHE_FOUND=0 .. - -# Build all targets that we don't want to analyze. -cd contrib && make -j24 && cd .. - -cov-configure --comptype gcc --compiler gcc-7 --template - -cov-build --dir cov-int make -j24 - -# Build is painful slow. Some targets compile in about one hour. Total time is about 4..5 hours. - -tar czvf clickhouse.tgz cov-int - -# tarball is 1.2 GB. -# Upload result at https://scan.coverity.com/projects/yandex-clickhouse/builds/new diff --git a/tests/instructions/cppcheck.txt b/tests/instructions/cppcheck.txt deleted file mode 100644 index 1bc6d1f6c09..00000000000 --- a/tests/instructions/cppcheck.txt +++ /dev/null @@ -1,22 +0,0 @@ -# Install cppcheck - -mkdir cppcheck && cd cppcheck -git clone git@github.com:danmar/cppcheck.git . -mkdir build && cd build -CC=gcc-7 CXX=g++-7 cmake -D CMAKE_BUILD_TYPE=Release .. -make -j24 -sudo make install - -# Perform analysis - -cd ClickHouse_clean/build -cppcheck -j24 --project=compile_commands.json --enable=all 2> cppcheck-errors.txt - -# or (from directory with sources) -# cppcheck -i contrib -i build --enable=all . 2> cppcheck-errors.txt - -# Check is pretty fast. -# It gives many false positives. -# But the result is worth looking and at least few real errors found. - -grep -v -F 'contrib/' cppcheck-errors.txt diff --git a/tests/instructions/heap-profiler.txt b/tests/instructions/heap-profiler.txt deleted file mode 100644 index 3c35e9cf518..00000000000 --- a/tests/instructions/heap-profiler.txt +++ /dev/null @@ -1,14 +0,0 @@ -Build clickhouse without tcmalloc. cmake -D ENABLE_TCMALLOC=0 - -Copy clickhouse binary to your server. -scp programs/clickhouse server:~ - -ssh to your server - -Stop clickhouse: -sudo service clickhouse-server stop - -Run clickhouse with heap profiler from the terminal: -sudo -u clickhouse LD_PRELOAD=/usr/lib/libtcmalloc.so HEAPPROFILE=/var/log/clickhouse-server/heap.hprof ./clickhouse server --config /etc/clickhouse-server/config.xml - -Profiles will appear in /var/log/clickhouse-server/ diff --git a/tests/instructions/kafka.txt b/tests/instructions/kafka.txt deleted file mode 100644 index 69e87f38b24..00000000000 --- a/tests/instructions/kafka.txt +++ /dev/null @@ -1,45 +0,0 @@ -Use this config for docker-compose: - - version: '3' - - services: - - kafka: - depends_on: - - zookeeper - hostname: kafka - image: wurstmeister/kafka - environment: - KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - ports: - - "9092:9092" - - "9094:9094" - - security_opt: - - label:disable - - zookeeper: - hostname: zookeeper - image: zookeeper - - security_opt: - - label:disable - -Start containers with `docker-compose up`. - -In clickhouse-client create table like: - - CREATE TABLE kafka ( a UInt8, b String) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'CSV') SETTINGS kafka_row_delimiter = '\n'; - -Login inside Kafka container and stream some data: - - docker exec -it bash --login - vi data.csv - cat data.csv | /opt/kafka/bin/kafka-console-producer.sh --topic topic --broker-list localhost:9092 - -Read data in clickhouse: - - SELECT * FROM kafka; diff --git a/tests/instructions/sanitizers.md b/tests/instructions/sanitizers.md deleted file mode 100644 index 3c50f6cbab7..00000000000 --- a/tests/instructions/sanitizers.md +++ /dev/null @@ -1,72 +0,0 @@ -# How to use Address Sanitizer - -Note: We use Address Sanitizer to run functional tests for every commit automatically. - -``` -mkdir build_asan && cd build_asan -``` - -Note: using clang instead of gcc is strongly recommended. Make sure you have installed required packages (`clang`, `lld`). It may be required to specify non-standard `lld` binary using `LINKER_NAME` option (e.g. `-D LINKER_NAME=lld-8`). - -``` -CC=clang CXX=clang++ cmake -D SANITIZE=address .. -ninja -``` - -## Copy binary to your server - -``` -scp ./programs/clickhouse yourserver:~/clickhouse-asan -``` - -## Start ClickHouse and run tests - -``` -sudo -u clickhouse ./clickhouse-asan server --config /etc/clickhouse-server/config.xml -``` - - -# How to use Thread Sanitizer - -``` -mkdir build_tsan && cd build_tsan -``` - -``` -CC=clang CXX=clang++ cmake -D SANITIZE=thread .. -ninja -``` - -## Start ClickHouse and run tests - -``` -sudo -u clickhouse TSAN_OPTIONS='halt_on_error=1' ./clickhouse-tsan server --config /etc/clickhouse-server/config.xml -``` - - -# How to use Undefined Behaviour Sanitizer - -``` -mkdir build_ubsan && cd build_ubsan -``` - -Note: clang is mandatory, because gcc (in version 8) has false positives due to devirtualization and it has less amount of checks. - -``` -CC=clang CXX=clang++ cmake -D SANITIZE=undefined .. -ninja -``` - -## Start ClickHouse and run tests - -``` -sudo -u clickhouse UBSAN_OPTIONS='print_stacktrace=1' ./clickhouse-ubsan server --config /etc/clickhouse-server/config.xml -``` - - -# How to use Memory Sanitizer - -``` -CC=clang CXX=clang++ cmake -D SANITIZE=memory .. -ninja -``` diff --git a/tests/instructions/syntax.txt b/tests/instructions/syntax.txt deleted file mode 100644 index 228b0eb6045..00000000000 --- a/tests/instructions/syntax.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Relatively quick syntax check (20 minutes on 16-core server) - -mkdir build && cd build -cmake -D CMAKE_BUILD_TYPE=Debug .. -time jq --raw-output '.[] | .command' compile_commands.json | grep -P -- ' -o [^ ]+\.o' | grep -v -P -- '-c .+/contrib/' | grep -vP '\.(s|asm)$' | sed -r -e 's/ -o [^ ]+\.o/ -fsyntax-only/' | sort -R | xargs -I{} -P$(nproc) sh -c '{}' diff --git a/tests/instructions/tscancode.txt b/tests/instructions/tscancode.txt deleted file mode 100644 index 33a4eb34f35..00000000000 --- a/tests/instructions/tscancode.txt +++ /dev/null @@ -1,26 +0,0 @@ -# TScanCode is a static analyzer from Tencent -# It looks like to be based on CppCheck - -git clone git@github.com:Tencent/TscanCode.git -cd TscanCode/trunk -make -j4 - -# It looks weird that TScanCode itself compiles with multiple warnings like 'unused-but-set-variable' and 'misleading-indentation' - -# Run analysis: - -./tscancode -j4 --enable=all ~/work/ClickHouse 2> result.txt - -# It has no way to remove specific directories. We have to checkout ClickHouse to separate directory and manually remove "contrib". -# Otherwise it segfaults when analysing llvm submodule. - -# It works quite fast: - -real 0m17.174s -user 0m45.498s -sys 0m0.496s - -wc -l result.txt -61 result.txt - -# It gives almost all false positives. From 216450e789f62b509bbef9f451f852e8527ac034 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 12:23:06 +0100 Subject: [PATCH 138/192] Some tasks are done --- tests/instructions/easy_tasks_sorted_ru.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/instructions/easy_tasks_sorted_ru.md b/tests/instructions/easy_tasks_sorted_ru.md index 17e9708eef5..bc95e6b1c37 100644 --- a/tests/instructions/easy_tasks_sorted_ru.md +++ b/tests/instructions/easy_tasks_sorted_ru.md @@ -201,9 +201,9 @@ https://clickhouse.com/docs/en/operations/table_engines/external_data/ ## ВозможноŃŃ‚ŃŚ ATTACH партиции Ń ĐĽĐµĐ˝ŃŚŃим или больŃим количеŃтвом Ńтолбцов. -## Поддержка неконŃтантного аргŃмента Ń Ń‚Đ°ĐąĐĽ-зоной Ń Đ˝ĐµĐşĐľŃ‚ĐľŃ€Ń‹Ń… Ń„Ńнкций для работы Ń Đ´Đ°Ń‚ĐľĐą и временем. +## + Поддержка неконŃтантного аргŃмента Ń Ń‚Đ°ĐąĐĽ-зоной Ń Đ˝ĐµĐşĐľŃ‚ĐľŃ€Ń‹Ń… Ń„Ńнкций для работы Ń Đ´Đ°Ń‚ĐľĐą и временем. -## ВозможноŃŃ‚ŃŚ задавать параметры Ńоединений для табличных Ń„Ńнкций, движков таблиц и для реплик из отдельных разделов конфигŃрации. +## + ВозможноŃŃ‚ŃŚ задавать параметры Ńоединений для табличных Ń„Ńнкций, движков таблиц и для реплик из отдельных разделов конфигŃрации. ## + НаŃтройка rollup_use_nulls. From 86119dbc3f56a78234eaa67ea40760bb0e30e7fa Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Fri, 17 Nov 2023 12:26:50 +0100 Subject: [PATCH 139/192] fix data race --- src/Storages/MergeTree/DataPartsExchange.cpp | 10 ++++++++-- .../02916_replication_protocol_wait_for_part.sql | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index c39263a0b73..0192fb1868b 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -349,7 +349,7 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( return data_checksums; } -bool wait_loop(UInt32 wait_timeout_ms, std::function pred) +bool wait_loop(UInt32 wait_timeout_ms, const std::function & pred) { static const UInt32 loop_delay_ms = 5; @@ -360,6 +360,7 @@ bool wait_loop(UInt32 wait_timeout_ms, std::function pred) return true; Stopwatch timer; + sleepForMilliseconds(loop_delay_ms); while (!pred() && timer.elapsedMilliseconds() < wait_timeout_ms) { sleepForMilliseconds(loop_delay_ms); @@ -387,8 +388,13 @@ MergeTreeData::DataPartPtr Service::findPart(const String & name) /// do not expose PreActive parts for zero-copy static const UInt32 wait_timeout_ms = 1000; - bool pred_result = wait_loop(wait_timeout_ms, [&] () { return part->getState() != MergeTreeDataPartState::PreActive; }); + auto pred = [&] () + { + auto lock = data.lockParts(); + return part->getState() != MergeTreeDataPartState::PreActive; + }; + bool pred_result = wait_loop(wait_timeout_ms, pred); if (!pred_result) throw Exception( ErrorCodes::ABORTED, diff --git a/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql index 97ef33f96e8..010e29a34e8 100644 --- a/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql +++ b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql @@ -22,5 +22,5 @@ insert into tableIn values(2); system sync replica tableOut; select count() from tableOut; -drop table tableIn -drop table tableOut +drop table tableIn; +drop table tableOut; From b2dc5ada6e1702332d15fbd515c728e5d06cb7d2 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 17 Nov 2023 11:31:52 +0000 Subject: [PATCH 140/192] Fix tryDecodeBase64() with invalid input --- src/Functions/FunctionBase64Conversion.h | 10 ++++----- .../00732_base64_functions.reference | 6 ++--- .../0_stateless/00732_base64_functions.sql | 22 +++++++++++++------ 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/Functions/FunctionBase64Conversion.h b/src/Functions/FunctionBase64Conversion.h index f52dec0eaf7..de922747ccd 100644 --- a/src/Functions/FunctionBase64Conversion.h +++ b/src/Functions/FunctionBase64Conversion.h @@ -76,12 +76,10 @@ struct TryBase64Decode static size_t perform(const std::span src, UInt8 * dst) { size_t outlen = 0; - base64_decode(reinterpret_cast(src.data()), src.size(), reinterpret_cast(dst), &outlen, 0); + int rc = base64_decode(reinterpret_cast(src.data()), src.size(), reinterpret_cast(dst), &outlen, 0); - // during decoding character array can be partially polluted - // if fail, revert back and clean - if (!outlen) - *dst = 0; + if (rc != 1) + outlen = 0; return outlen; } @@ -147,7 +145,7 @@ private: for (size_t row = 0; row < src_row_count; ++row) { const size_t src_length = src_offsets[row] - src_offset_prev - 1; - const auto outlen = Func::perform({src, src_length}, dst_pos); + const size_t outlen = Func::perform({src, src_length}, dst_pos); /// Base64 library is using AVX-512 with some shuffle operations. /// Memory sanitizer don't understand if there was uninitialized memory in SIMD register but it was not used in the result of shuffle. diff --git a/tests/queries/0_stateless/00732_base64_functions.reference b/tests/queries/0_stateless/00732_base64_functions.reference index f97c19427e7..8f91ffa74ab 100644 --- a/tests/queries/0_stateless/00732_base64_functions.reference +++ b/tests/queries/0_stateless/00732_base64_functions.reference @@ -21,9 +21,9 @@ fooba foobar 1 1 1 1 -fooba -~Š + + + Zm9v foo foo -TEcgT3B0aW11cw== diff --git a/tests/queries/0_stateless/00732_base64_functions.sql b/tests/queries/0_stateless/00732_base64_functions.sql index 99268004003..3c60bf939fe 100644 --- a/tests/queries/0_stateless/00732_base64_functions.sql +++ b/tests/queries/0_stateless/00732_base64_functions.sql @@ -2,17 +2,23 @@ SET send_logs_level = 'fatal'; -SELECT base64Encode(val) FROM (select arrayJoin(['', 'f', 'fo', 'foo', 'foob', 'fooba', 'foobar']) val); +SELECT base64Encode(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT base64Decode(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT tryBase64Decode(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT base64Encode('foo', 'excess argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT base64Decode('foo', 'excess argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT tryBase64Decode('foo', 'excess argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +-- test with valid inputs + +SELECT base64Encode(val) FROM (select arrayJoin(['', 'f', 'fo', 'foo', 'foob', 'fooba', 'foobar']) val); SELECT base64Decode(val) FROM (select arrayJoin(['', 'Zg==', 'Zm8=', 'Zm9v', 'Zm9vYg==', 'Zm9vYmE=', 'Zm9vYmFy']) val); SELECT tryBase64Decode(val) FROM (select arrayJoin(['', 'Zg==', 'Zm8=', 'Zm9v', 'Zm9vYg==', 'Zm9vYmE=', 'Zm9vYmFy']) val); SELECT base64Decode(base64Encode('foo')) = 'foo', base64Encode(base64Decode('Zm9v')) == 'Zm9v'; SELECT tryBase64Decode(base64Encode('foo')) = 'foo', base64Encode(tryBase64Decode('Zm9v')) == 'Zm9v'; -SELECT base64Encode('foo', 'excess argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } -SELECT base64Decode('foo', 'excess argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } -SELECT tryBase64Decode('foo', 'excess argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +-- test with invalid inputs SELECT base64Decode('Zm9vYmF=Zm9v'); -- { serverError INCORRECT_DATA } SELECT tryBase64Decode('Zm9vYmF=Zm9v'); @@ -20,9 +26,11 @@ SELECT tryBase64Decode('Zm9vYmF=Zm9v'); SELECT base64Decode('foo'); -- { serverError INCORRECT_DATA } SELECT tryBase64Decode('foo'); +SELECT base64Decode('aoeo054640eu='); -- { serverError INCORRECT_DATA } +SELECT tryBase64Decode('aoeo054640eu='); + +-- test FixedString arguments + select base64Encode(toFixedString('foo', 3)); select base64Decode(toFixedString('Zm9v', 4)); select tryBase64Decode(toFixedString('Zm9v', 4)); - --- This query reproduces a bug in TurboBase64 library (which we no longer use) -select distinct base64Encode(materialize('LG Optimus')) from numbers(100); From 318c7a06f92db53d1cf7389a37387193da187ff6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 12:39:00 +0100 Subject: [PATCH 141/192] Avoid dependencies with no fixed versions --- docker/packager/binary/Dockerfile | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index fb033e28959..65f79da44a0 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -6,29 +6,27 @@ FROM clickhouse/test-util:latest AS cctools ENV CC=clang-${LLVM_VERSION} ENV CXX=clang++-${LLVM_VERSION} # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -# DO NOT PUT ANYTHING BEFORE THREE NEXT `RUN` DIRECTIVES +# DO NOT PUT ANYTHING BEFORE THE NEXT TWO `RUN` DIRECTIVES # THE MOST HEAVY OPERATION MUST BE THE FIRST IN THE CACHE # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # libtapi is required to support .tbh format from recent MacOS SDKs RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \ && cd apple-libtapi \ + && git checkout 15dfc2a8c9a2a89d06ff227560a69f5265b692f9 \ && INSTALLPREFIX=/cctools ./build.sh \ && ./install.sh \ && cd .. \ && rm -rf apple-libtapi # Build and install tools for cross-linking to Darwin (x86-64) -RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ - && cd cctools-port/cctools \ - && ./configure --prefix=/cctools --with-libtapi=/cctools \ - --target=x86_64-apple-darwin \ - && make install -j$(nproc) \ - && cd ../.. \ - && rm -rf cctools-port - # Build and install tools for cross-linking to Darwin (aarch64) RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ && cd cctools-port/cctools \ + && git checkout 59f5fb87a3d2c6fd2ba3df6533015cd6172001c6 \ + && ./configure --prefix=/cctools --with-libtapi=/cctools \ + --target=x86_64-apple-darwin \ + && make install -j$(nproc) \ + && make clean \ && ./configure --prefix=/cctools --with-libtapi=/cctools \ --target=aarch64-apple-darwin \ && make install -j$(nproc) \ From 45b69566012ece56f9e284f2521c4bb13fb43c7f Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Fri, 17 Nov 2023 12:42:06 +0100 Subject: [PATCH 142/192] Fix race on zk_log --- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index fd845016f8a..4335ea4655f 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1147,7 +1147,8 @@ void ZooKeeper::pushRequest(RequestInfo && info) { checkSessionDeadline(); info.time = clock::now(); - if (zk_log) + auto maybe_zk_log = std::atomic_load(&zk_log); + if (maybe_zk_log) { info.request->thread_id = getThreadId(); info.request->query_id = String(CurrentThread::getQueryId()); From 1d5bc13e2aa18dd30553eede612df350973425c8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 12:50:51 +0100 Subject: [PATCH 143/192] Check what will happen if I remove some lines --- docker/packager/binary/Dockerfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index fb033e28959..2d3b83814b8 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -62,19 +62,12 @@ RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \ rustup target add aarch64-unknown-linux-musl && \ rustup target add riscv64gc-unknown-linux-gnu -# NOTE: Seems like gcc-11 is too new for ubuntu20 repository # A cross-linker for RISC-V 64 (we need it, because LLVM's LLD does not work): RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \ && apt-get update \ && apt-get install --yes \ binutils-riscv64-linux-gnu \ build-essential \ - g++-11 \ - gcc-11 \ - gcc-aarch64-linux-gnu \ - libc6 \ - libc6-dev \ - libc6-dev-arm64-cross \ python3-boto3 \ yasm \ zstd \ From 2a9d05e24541d098dfe375567d5d369698a859bf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 12:59:04 +0100 Subject: [PATCH 144/192] Remove more lines --- docker/packager/packager | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index e63a4912e7c..b5bcbada1da 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -236,16 +236,14 @@ def parse_env_variables( cc = compiler result.append("DEB_ARCH=amd64") - cxx = cc.replace("gcc", "g++").replace("clang", "clang++") + cxx = cc.replace("clang", "clang++") if package_type == "deb": - # NOTE: This are the env for packages/build script + # NOTE: This is the env for packages/build script result.append("MAKE_DEB=true") cmake_flags.append("-DENABLE_TESTS=0") cmake_flags.append("-DENABLE_UTILS=0") - cmake_flags.append("-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON") cmake_flags.append("-DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON") - cmake_flags.append("-DCMAKE_AUTOGEN_VERBOSE=ON") cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr") cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc") cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var") @@ -265,12 +263,7 @@ def parse_env_variables( elif package_type == "fuzzers": cmake_flags.append("-DENABLE_FUZZING=1") cmake_flags.append("-DENABLE_PROTOBUF=1") - cmake_flags.append("-DUSE_INTERNAL_PROTOBUF_LIBRARY=1") cmake_flags.append("-DWITH_COVERAGE=1") - cmake_flags.append("-DCMAKE_AUTOGEN_VERBOSE=ON") - # cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr") - # cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc") - # cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var") # Reduce linking and building time by avoid *install/all dependencies cmake_flags.append("-DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON") From 354fb5182bfc15c37687da9918680430e6478eb8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 13:01:33 +0100 Subject: [PATCH 145/192] Remove some code that I don't understand --- cmake/darwin/toolchain-aarch64.cmake | 6 ------ cmake/darwin/toolchain-x86_64.cmake | 6 ------ cmake/freebsd/toolchain-aarch64.cmake | 6 ------ cmake/freebsd/toolchain-ppc64le.cmake | 6 ------ cmake/freebsd/toolchain-x86_64.cmake | 6 ------ cmake/linux/toolchain-aarch64.cmake | 6 ------ cmake/linux/toolchain-ppc64le.cmake | 6 ------ cmake/linux/toolchain-riscv64.cmake | 6 ------ cmake/linux/toolchain-s390x.cmake | 6 ------ cmake/linux/toolchain-x86_64-musl.cmake | 6 ------ cmake/linux/toolchain-x86_64.cmake | 6 ------ 11 files changed, 66 deletions(-) diff --git a/cmake/darwin/toolchain-aarch64.cmake b/cmake/darwin/toolchain-aarch64.cmake index 569b02bb642..178153c1098 100644 --- a/cmake/darwin/toolchain-aarch64.cmake +++ b/cmake/darwin/toolchain-aarch64.cmake @@ -9,9 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "aarch64-apple-darwin") set (CMAKE_OSX_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/darwin-aarch64") set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/darwin/toolchain-x86_64.cmake b/cmake/darwin/toolchain-x86_64.cmake index c4527d2fc0d..b9cbe72a2b6 100644 --- a/cmake/darwin/toolchain-x86_64.cmake +++ b/cmake/darwin/toolchain-x86_64.cmake @@ -9,9 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "x86_64-apple-darwin") set (CMAKE_OSX_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/darwin-x86_64") set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/freebsd/toolchain-aarch64.cmake b/cmake/freebsd/toolchain-aarch64.cmake index 8a8da00f3be..0d7eba7c198 100644 --- a/cmake/freebsd/toolchain-aarch64.cmake +++ b/cmake/freebsd/toolchain-aarch64.cmake @@ -13,9 +13,3 @@ set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it # Will be changed later, but somehow needed to be set here. set (CMAKE_AR "ar") set (CMAKE_RANLIB "ranlib") - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/freebsd/toolchain-ppc64le.cmake b/cmake/freebsd/toolchain-ppc64le.cmake index c3f6594204d..f9878bb47be 100644 --- a/cmake/freebsd/toolchain-ppc64le.cmake +++ b/cmake/freebsd/toolchain-ppc64le.cmake @@ -13,9 +13,3 @@ set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it # Will be changed later, but somehow needed to be set here. set (CMAKE_AR "ar") set (CMAKE_RANLIB "ranlib") - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/freebsd/toolchain-x86_64.cmake b/cmake/freebsd/toolchain-x86_64.cmake index 460de6a7d39..60489da1d65 100644 --- a/cmake/freebsd/toolchain-x86_64.cmake +++ b/cmake/freebsd/toolchain-x86_64.cmake @@ -13,9 +13,3 @@ set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it # Will be changed later, but somehow needed to be set here. set (CMAKE_AR "ar") set (CMAKE_RANLIB "ranlib") - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/linux/toolchain-aarch64.cmake b/cmake/linux/toolchain-aarch64.cmake index 2dedef8859f..954f3da4331 100644 --- a/cmake/linux/toolchain-aarch64.cmake +++ b/cmake/linux/toolchain-aarch64.cmake @@ -20,9 +20,3 @@ set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/aarch64-linux-gnu/libc") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/linux/toolchain-ppc64le.cmake b/cmake/linux/toolchain-ppc64le.cmake index c46ea954b71..ae10cac9a55 100644 --- a/cmake/linux/toolchain-ppc64le.cmake +++ b/cmake/linux/toolchain-ppc64le.cmake @@ -20,9 +20,3 @@ set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/powerpc64le-linux-gnu/libc") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/linux/toolchain-riscv64.cmake b/cmake/linux/toolchain-riscv64.cmake index 7f876f88d72..7f0e30869fc 100644 --- a/cmake/linux/toolchain-riscv64.cmake +++ b/cmake/linux/toolchain-riscv64.cmake @@ -27,9 +27,3 @@ set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=bfd") # ld.lld: error: section size decrease is too large # But GNU BinUtils work. set (LINKER_NAME "riscv64-linux-gnu-ld.bfd" CACHE STRING "Linker name" FORCE) - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/linux/toolchain-s390x.cmake b/cmake/linux/toolchain-s390x.cmake index 945eb9affa4..b89275d5812 100644 --- a/cmake/linux/toolchain-s390x.cmake +++ b/cmake/linux/toolchain-s390x.cmake @@ -23,9 +23,3 @@ set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=mold -Wl,-L${CMAKE_SYSROOT}/usr/lib64") set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=mold -Wl,-L${CMAKE_SYSROOT}/usr/lib64") set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=mold -Wl,-L${CMAKE_SYSROOT}/usr/lib64") - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/linux/toolchain-x86_64-musl.cmake b/cmake/linux/toolchain-x86_64-musl.cmake index bc327e5ac25..250e52d5f58 100644 --- a/cmake/linux/toolchain-x86_64-musl.cmake +++ b/cmake/linux/toolchain-x86_64-musl.cmake @@ -21,11 +21,5 @@ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - set (USE_MUSL 1) add_definitions(-DUSE_MUSL=1) diff --git a/cmake/linux/toolchain-x86_64.cmake b/cmake/linux/toolchain-x86_64.cmake index 55b9df79f70..8f54cbb0b48 100644 --- a/cmake/linux/toolchain-x86_64.cmake +++ b/cmake/linux/toolchain-x86_64.cmake @@ -32,9 +32,3 @@ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) From a3083f305ba50de44e33bcd97a089cb41e24ce22 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 13:06:38 +0100 Subject: [PATCH 146/192] Remove more code that I don't understand --- cmake/freebsd/toolchain-aarch64.cmake | 4 ---- cmake/freebsd/toolchain-ppc64le.cmake | 4 ---- cmake/freebsd/toolchain-x86_64.cmake | 4 ---- cmake/linux/toolchain-aarch64.cmake | 4 ---- cmake/linux/toolchain-ppc64le.cmake | 4 ---- cmake/linux/toolchain-riscv64.cmake | 4 ---- cmake/linux/toolchain-s390x.cmake | 4 ---- cmake/linux/toolchain-x86_64-musl.cmake | 4 ---- cmake/linux/toolchain-x86_64.cmake | 4 ---- 9 files changed, 36 deletions(-) diff --git a/cmake/freebsd/toolchain-aarch64.cmake b/cmake/freebsd/toolchain-aarch64.cmake index 0d7eba7c198..53b7856ed03 100644 --- a/cmake/freebsd/toolchain-aarch64.cmake +++ b/cmake/freebsd/toolchain-aarch64.cmake @@ -9,7 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "aarch64-unknown-freebsd12") set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-aarch64") set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake - -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") diff --git a/cmake/freebsd/toolchain-ppc64le.cmake b/cmake/freebsd/toolchain-ppc64le.cmake index f9878bb47be..bb23f0fbafc 100644 --- a/cmake/freebsd/toolchain-ppc64le.cmake +++ b/cmake/freebsd/toolchain-ppc64le.cmake @@ -9,7 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "powerpc64le-unknown-freebsd13") set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-ppc64le") set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake - -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") diff --git a/cmake/freebsd/toolchain-x86_64.cmake b/cmake/freebsd/toolchain-x86_64.cmake index 60489da1d65..4635880b4a6 100644 --- a/cmake/freebsd/toolchain-x86_64.cmake +++ b/cmake/freebsd/toolchain-x86_64.cmake @@ -9,7 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "x86_64-pc-freebsd11") set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-x86_64") set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake - -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") diff --git a/cmake/linux/toolchain-aarch64.cmake b/cmake/linux/toolchain-aarch64.cmake index 954f3da4331..b80cc01296d 100644 --- a/cmake/linux/toolchain-aarch64.cmake +++ b/cmake/linux/toolchain-aarch64.cmake @@ -9,10 +9,6 @@ set (CMAKE_C_COMPILER_TARGET "aarch64-linux-gnu") set (CMAKE_CXX_COMPILER_TARGET "aarch64-linux-gnu") set (CMAKE_ASM_COMPILER_TARGET "aarch64-linux-gnu") -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") - set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-aarch64") set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/aarch64-linux-gnu/libc") diff --git a/cmake/linux/toolchain-ppc64le.cmake b/cmake/linux/toolchain-ppc64le.cmake index ae10cac9a55..98e8f7e8489 100644 --- a/cmake/linux/toolchain-ppc64le.cmake +++ b/cmake/linux/toolchain-ppc64le.cmake @@ -9,10 +9,6 @@ set (CMAKE_C_COMPILER_TARGET "powerpc64le-linux-gnu") set (CMAKE_CXX_COMPILER_TARGET "powerpc64le-linux-gnu") set (CMAKE_ASM_COMPILER_TARGET "powerpc64le-linux-gnu") -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") - set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-powerpc64le") set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/powerpc64le-linux-gnu/libc") diff --git a/cmake/linux/toolchain-riscv64.cmake b/cmake/linux/toolchain-riscv64.cmake index 7f0e30869fc..ae5a38f08eb 100644 --- a/cmake/linux/toolchain-riscv64.cmake +++ b/cmake/linux/toolchain-riscv64.cmake @@ -9,10 +9,6 @@ set (CMAKE_C_COMPILER_TARGET "riscv64-linux-gnu") set (CMAKE_CXX_COMPILER_TARGET "riscv64-linux-gnu") set (CMAKE_ASM_COMPILER_TARGET "riscv64-linux-gnu") -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") - set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-riscv64") set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}") diff --git a/cmake/linux/toolchain-s390x.cmake b/cmake/linux/toolchain-s390x.cmake index b89275d5812..d34329fb3bb 100644 --- a/cmake/linux/toolchain-s390x.cmake +++ b/cmake/linux/toolchain-s390x.cmake @@ -9,10 +9,6 @@ set (CMAKE_C_COMPILER_TARGET "s390x-linux-gnu") set (CMAKE_CXX_COMPILER_TARGET "s390x-linux-gnu") set (CMAKE_ASM_COMPILER_TARGET "s390x-linux-gnu") -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") - set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-s390x") set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/s390x-linux-gnu/libc") diff --git a/cmake/linux/toolchain-x86_64-musl.cmake b/cmake/linux/toolchain-x86_64-musl.cmake index 250e52d5f58..fa7b3eaf0d1 100644 --- a/cmake/linux/toolchain-x86_64-musl.cmake +++ b/cmake/linux/toolchain-x86_64-musl.cmake @@ -9,10 +9,6 @@ set (CMAKE_C_COMPILER_TARGET "x86_64-linux-musl") set (CMAKE_CXX_COMPILER_TARGET "x86_64-linux-musl") set (CMAKE_ASM_COMPILER_TARGET "x86_64-linux-musl") -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") - set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-x86_64-musl") set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}") diff --git a/cmake/linux/toolchain-x86_64.cmake b/cmake/linux/toolchain-x86_64.cmake index 8f54cbb0b48..e341219a7e5 100644 --- a/cmake/linux/toolchain-x86_64.cmake +++ b/cmake/linux/toolchain-x86_64.cmake @@ -19,10 +19,6 @@ set (CMAKE_C_COMPILER_TARGET "x86_64-linux-gnu") set (CMAKE_CXX_COMPILER_TARGET "x86_64-linux-gnu") set (CMAKE_ASM_COMPILER_TARGET "x86_64-linux-gnu") -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") - set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-x86_64") set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/x86_64-linux-gnu/libc") From 215cd7b9f285bbfa5d07882dfd64820e3428c402 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 15:13:16 +0300 Subject: [PATCH 147/192] Update build.sh --- docker/packager/binary/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 37440fe8202..f943011df9d 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -38,7 +38,7 @@ rm -f CMakeCache.txt # To check it, find and delete them. grep -o -P '"contrib/[^"]+"' ../.gitmodules | - grep -v -P 'llvm-project|google-protobuf|grpc|corrosion' | + grep -v -P 'llvm-project|google-protobuf|grpc|abseil-cpp|corrosion' | xargs -I@ find ../@ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | xargs rm From 19dd29e8af5f79323a8f86c6a72f68ae2b45fa6e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 15:19:00 +0300 Subject: [PATCH 148/192] Update Dockerfile --- docker/packager/binary/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 65f79da44a0..d7864a11672 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -22,7 +22,7 @@ RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \ # Build and install tools for cross-linking to Darwin (aarch64) RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ && cd cctools-port/cctools \ - && git checkout 59f5fb87a3d2c6fd2ba3df6533015cd6172001c6 \ + && git checkout 2ea20c36c10fa1ec70ada3d5aeb2c205d4aa591e \ && ./configure --prefix=/cctools --with-libtapi=/cctools \ --target=x86_64-apple-darwin \ && make install -j$(nproc) \ From ea3cd71225794724ed63213d3567a9167133eade Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 15:42:10 +0300 Subject: [PATCH 149/192] Update Dockerfile --- docker/packager/binary/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index d7864a11672..8b5049c0fc7 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -10,7 +10,7 @@ ENV CXX=clang++-${LLVM_VERSION} # THE MOST HEAVY OPERATION MUST BE THE FIRST IN THE CACHE # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # libtapi is required to support .tbh format from recent MacOS SDKs -RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \ +RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \ && cd apple-libtapi \ && git checkout 15dfc2a8c9a2a89d06ff227560a69f5265b692f9 \ && INSTALLPREFIX=/cctools ./build.sh \ @@ -20,9 +20,9 @@ RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \ # Build and install tools for cross-linking to Darwin (x86-64) # Build and install tools for cross-linking to Darwin (aarch64) -RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ +RUN git clone https://github.com/tpoechtrager/cctools-port.git \ && cd cctools-port/cctools \ - && git checkout 2ea20c36c10fa1ec70ada3d5aeb2c205d4aa591e \ + && git checkout 319ef50ea51a73acfc3d691396c05005e48647da \ && ./configure --prefix=/cctools --with-libtapi=/cctools \ --target=x86_64-apple-darwin \ && make install -j$(nproc) \ From b42db2ec298d67ad0c4cb5f190fc7c1f1815e781 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 13:50:13 +0100 Subject: [PATCH 150/192] Update fasttest --- docker/test/fasttest/run.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 1b72dab5e3c..d3695ba2613 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -206,7 +206,7 @@ function build ( cd "$FASTTEST_BUILD" TIMEFORMAT=$'\nreal\t%3R\nuser\t%3U\nsys\t%3S' - ( time ninja clickhouse-bundle) |& ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/build_log.txt" + ( time ninja clickhouse-bundle clickhouse-stripped) |& ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/build_log.txt" BUILD_SECONDS_ELAPSED=$(awk '/^....-..-.. ..:..:.. real\t[0-9]/ {print $4}' < "$FASTTEST_OUTPUT/build_log.txt") echo "build_clickhouse_fasttest_binary: [ OK ] $BUILD_SECONDS_ELAPSED sec." \ | ts '%Y-%m-%d %H:%M:%S' \ @@ -215,7 +215,6 @@ function build mkdir -p "$FASTTEST_OUTPUT/binaries/" cp programs/clickhouse "$FASTTEST_OUTPUT/binaries/clickhouse" - strip programs/clickhouse -o programs/clickhouse-stripped zstd --threads=0 programs/clickhouse-stripped -o "$FASTTEST_OUTPUT/binaries/clickhouse-stripped.zst" fi ccache_status From 9eb0b74167b6b72343fba25bbc31fd42f919ef1b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 15:54:32 +0300 Subject: [PATCH 151/192] Update Dockerfile --- docker/packager/binary/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 8b5049c0fc7..5e630ea5a3f 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -22,7 +22,7 @@ RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \ # Build and install tools for cross-linking to Darwin (aarch64) RUN git clone https://github.com/tpoechtrager/cctools-port.git \ && cd cctools-port/cctools \ - && git checkout 319ef50ea51a73acfc3d691396c05005e48647da \ + && git checkout 2a3e1c2a6ff54a30f898b70cfb9ba1692a55fad7 \ && ./configure --prefix=/cctools --with-libtapi=/cctools \ --target=x86_64-apple-darwin \ && make install -j$(nproc) \ From 332e7f565e613ab3a519749534472d14b30845dd Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Fri, 17 Nov 2023 14:41:26 +0100 Subject: [PATCH 152/192] Fix concat tests --- tests/queries/0_stateless/00727_concat.reference | 2 +- tests/queries/0_stateless/00727_concat.sql | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/00727_concat.reference b/tests/queries/0_stateless/00727_concat.reference index 7c48ba97c2b..1e102051fd0 100644 --- a/tests/queries/0_stateless/00727_concat.reference +++ b/tests/queries/0_stateless/00727_concat.reference @@ -46,7 +46,7 @@ With [[[(0,0),(10,0),(10,10),(0,10)]],[[(20,20),(50,20),(50,50),(20,50)],[(30,30 With 42 With 4 -- Nested -With [(\'foo\',\'qaz\'),(\'bar\',\'qux\')] +With [\'foo\',\'bar\'][\'qaz\',\'qux\'] -- NULL arguments \N \N diff --git a/tests/queries/0_stateless/00727_concat.sql b/tests/queries/0_stateless/00727_concat.sql index 7d901514aea..edeaf9340dd 100644 --- a/tests/queries/0_stateless/00727_concat.sql +++ b/tests/queries/0_stateless/00727_concat.sql @@ -50,15 +50,19 @@ SELECT concat('With ', materialize([[(20, 20), (50, 20), (50, 50), (20, 50)], [( SELECT concat('With ', materialize([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]] :: MultiPolygon)); SELECT '-- SimpleAggregateFunction'; -CREATE OR REPLACE TABLE concat_saf_test(x SimpleAggregateFunction(max, Int32)) ENGINE=MergeTree ORDER BY tuple(); +DROP TABLE IF EXISTS concat_saf_test; +CREATE TABLE concat_saf_test(x SimpleAggregateFunction(max, Int32)) ENGINE=MergeTree ORDER BY tuple(); INSERT INTO concat_saf_test VALUES (42); INSERT INTO concat_saf_test SELECT max(number) FROM numbers(5); SELECT concat('With ', x) FROM concat_saf_test ORDER BY x DESC; +DROP TABLE concat_saf_test; SELECT '-- Nested'; -CREATE OR REPLACE TABLE concat_nested_test(kv Nested(k String, v String)) ENGINE = MergeTree ORDER BY tuple(); +DROP TABLE IF EXISTS concat_nested_test; +CREATE TABLE concat_nested_test(attrs Nested(k String, v String)) ENGINE = MergeTree ORDER BY tuple(); INSERT INTO concat_nested_test VALUES (['foo', 'bar'], ['qaz', 'qux']); -SELECT concat('With ', kv) FROM concat_nested_test; +SELECT concat('With ', attrs.k, attrs.v) FROM concat_nested_test; +DROP TABLE concat_nested_test; SELECT '-- NULL arguments'; SELECT concat(NULL, NULL); From 571a35c84d70fc54ce6852b1fc2084ab2e5adf47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 17 Nov 2023 14:45:19 +0100 Subject: [PATCH 153/192] Make some tests independent on macro settings --- ...licated_minimalistic_part_header_zookeeper.sh | 11 +++++++---- .../00953_zookeeper_suetin_deduplication_bug.sh | 14 +++++++------- ...6_inactive_replica_cleanup_nodes_zookeeper.sh | 11 +++++++---- ...1586_replicated_mutations_empty_partition.sql | 4 ++-- ...nt_alter_mutations_kill_many_replicas_long.sh | 7 +++++-- .../01700_system_zookeeper_path_in.reference | 2 -- .../01700_system_zookeeper_path_in.sql | 16 ++++++++-------- .../02439_merge_selecting_partitions.sql | 2 +- 8 files changed, 37 insertions(+), 30 deletions(-) diff --git a/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sh b/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sh index bab2304cec2..12d889a7137 100755 --- a/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sh +++ b/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sh @@ -7,6 +7,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +SHARD=$($CLICKHOUSE_CLIENT --query "Select getMacro('shard')") +REPLICA=$($CLICKHOUSE_CLIENT --query "Select getMacro('replica')") + $CLICKHOUSE_CLIENT -nm -q " DROP TABLE IF EXISTS part_header_r1; @@ -54,8 +57,8 @@ elapsed=1 until [ $elapsed -eq 5 ]; do sleep $(( elapsed++ )) - count1=$($CLICKHOUSE_CLIENT --query="SELECT count(name) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/s1/replicas/1r1/parts'") - count2=$($CLICKHOUSE_CLIENT --query="SELECT count(name) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/s1/replicas/2r1/parts'") + count1=$($CLICKHOUSE_CLIENT --query="SELECT count(name) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/$SHARD/replicas/1$REPLICA/parts'") + count2=$($CLICKHOUSE_CLIENT --query="SELECT count(name) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/$SHARD/replicas/2$REPLICA/parts'") [[ $count1 == 1 && $count2 == 1 ]] && break done @@ -64,10 +67,10 @@ $CLICKHOUSE_CLIENT -nm -q " SELECT '*** Test part removal ***'; SELECT '*** replica 1 ***'; SELECT name FROM system.parts WHERE active AND database = currentDatabase() AND table = 'part_header_r1'; -SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/s1/replicas/1r1/parts'; +SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/$SHARD/replicas/1$REPLICA/parts'; SELECT '*** replica 2 ***'; SELECT name FROM system.parts WHERE active AND database = currentDatabase() AND table = 'part_header_r2'; -SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/s1/replicas/2r1/parts'; +SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/$SHARD/replicas/2$REPLICA/parts'; SELECT '*** Test ALTER ***'; ALTER TABLE part_header_r1 MODIFY COLUMN y String; diff --git a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh index ad0146b9d99..57a41526900 100755 --- a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh +++ b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh @@ -9,7 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh CLICKHOUSE_TEST_ZOOKEEPER_PREFIX="${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_DATABASE}" - +SHARD=$($CLICKHOUSE_CLIENT --query "Select getMacro('shard')") $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS elog;" @@ -30,33 +30,33 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 3, 'h $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 3 rows -count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") +count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/$SHARD/blocks'") while [[ $count != 2 ]] do sleep 1 - count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") + count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/$SHARD/blocks'") done $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 1, 'hello')" $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 4 rows -count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") +count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/$SHARD/blocks'") while [[ $count != 2 ]] do sleep 1 - count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") + count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/$SHARD/blocks'") done $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')" $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 5 rows -count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") +count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/$SHARD/blocks'") while [[ $count != 2 ]] do sleep 1 - count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") + count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/$SHARD/blocks'") done $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')" diff --git a/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh b/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh index 2d761df998e..67a2a70b509 100755 --- a/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh +++ b/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh @@ -5,6 +5,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +SHARD=$($CLICKHOUSE_CLIENT --query "Select getMacro('shard')") +REPLICA=$($CLICKHOUSE_CLIENT --query "Select getMacro('replica')") + # Check that if we have one inactive replica and a huge number of INSERTs to active replicas, # the number of nodes in ZooKeeper does not grow unbounded. @@ -32,16 +35,16 @@ for _ in {1..60}; do done -$CLICKHOUSE_CLIENT --query "SELECT numChildren < $((SCALE / 4)) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1' AND name = 'log'"; +$CLICKHOUSE_CLIENT --query "SELECT numChildren < $((SCALE / 4)) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/$SHARD' AND name = 'log'"; echo -e '\n---\n'; -$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/1r1' AND name = 'is_lost'"; -$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/2r1' AND name = 'is_lost'"; +$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/$SHARD/replicas/1$REPLICA' AND name = 'is_lost'"; +$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/$SHARD/replicas/2$REPLICA' AND name = 'is_lost'"; echo -e '\n---\n'; $CLICKHOUSE_CLIENT --query "ATTACH TABLE r2" $CLICKHOUSE_CLIENT --receive_timeout 600 --query "SYSTEM SYNC REPLICA r2" # Need to increase timeout, otherwise it timed out in debug build -$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/2r1' AND name = 'is_lost'"; +$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/$SHARD/replicas/2$REPLICA' AND name = 'is_lost'"; $CLICKHOUSE_CLIENT -n --query " DROP TABLE IF EXISTS r1; diff --git a/tests/queries/0_stateless/01586_replicated_mutations_empty_partition.sql b/tests/queries/0_stateless/01586_replicated_mutations_empty_partition.sql index b5ad6c06e96..c4a3c939c26 100644 --- a/tests/queries/0_stateless/01586_replicated_mutations_empty_partition.sql +++ b/tests/queries/0_stateless/01586_replicated_mutations_empty_partition.sql @@ -16,7 +16,7 @@ INSERT INTO replicated_mutations_empty_partitions SETTINGS insert_keeper_fault_i SELECT count(distinct value) FROM replicated_mutations_empty_partitions; -SELECT count() FROM system.zookeeper WHERE path = '/clickhouse/test/'||currentDatabase()||'/01586_replicated_mutations_empty_partitions/s1/block_numbers'; +SELECT count() FROM system.zookeeper WHERE path = '/clickhouse/test/'||currentDatabase()||'/01586_replicated_mutations_empty_partitions/'||getMacro('shard')||'/block_numbers'; ALTER TABLE replicated_mutations_empty_partitions DROP PARTITION '3'; ALTER TABLE replicated_mutations_empty_partitions DROP PARTITION '4'; @@ -24,7 +24,7 @@ ALTER TABLE replicated_mutations_empty_partitions DROP PARTITION '5'; ALTER TABLE replicated_mutations_empty_partitions DROP PARTITION '9'; -- still ten records -SELECT count() FROM system.zookeeper WHERE path = '/clickhouse/test/'||currentDatabase()||'/01586_replicated_mutations_empty_partitions/s1/block_numbers'; +SELECT count() FROM system.zookeeper WHERE path = '/clickhouse/test/'||currentDatabase()||'/01586_replicated_mutations_empty_partitions/'||getMacro('shard')||'/block_numbers'; ALTER TABLE replicated_mutations_empty_partitions MODIFY COLUMN value UInt64 SETTINGS replication_alter_partitions_sync=2; diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh index f8f3ccd6dd6..2762f918d72 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh @@ -7,6 +7,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=./replication.lib . "$CURDIR"/replication.lib +SHARD=$($CLICKHOUSE_CLIENT --query "Select getMacro('shard')") +REPLICA=$($CLICKHOUSE_CLIENT --query "Select getMacro('replica')") + REPLICAS=5 for i in $(seq $REPLICAS); do @@ -79,9 +82,9 @@ while true; do done -metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/r11/' and name = 'metadata_version'") +metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/$SHARD/replicas/${REPLICA}1/' and name = 'metadata_version'") for i in $(seq $REPLICAS); do - replica_metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/r1$i/' and name = 'metadata_version'") + replica_metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/$SHARD/replicas/${REPLICA}$i/' and name = 'metadata_version'") if [ "$metadata_version" != "$replica_metadata_version" ]; then echo "Metadata version on replica $i differs from the first replica, FAIL" diff --git a/tests/queries/0_stateless/01700_system_zookeeper_path_in.reference b/tests/queries/0_stateless/01700_system_zookeeper_path_in.reference index 664d8e84f27..b4eaf226106 100644 --- a/tests/queries/0_stateless/01700_system_zookeeper_path_in.reference +++ b/tests/queries/0_stateless/01700_system_zookeeper_path_in.reference @@ -14,5 +14,3 @@ abandonable_lock-other failed_parts last_part parallel -shared -shared diff --git a/tests/queries/0_stateless/01700_system_zookeeper_path_in.sql b/tests/queries/0_stateless/01700_system_zookeeper_path_in.sql index cf4bc7650e7..3b321d3cea5 100644 --- a/tests/queries/0_stateless/01700_system_zookeeper_path_in.sql +++ b/tests/queries/0_stateless/01700_system_zookeeper_path_in.sql @@ -8,17 +8,17 @@ CREATE TABLE sample_table ( ENGINE ReplicatedMergeTree('/clickhouse/{database}/01700_system_zookeeper_path_in/{shard}', '{replica}') ORDER BY tuple(); -SELECT name FROM system.zookeeper WHERE path = '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1' AND name like 'block%' ORDER BY name; -SELECT name FROM system.zookeeper WHERE path = '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1/replicas' AND name LIKE '%r1%' ORDER BY name; +SELECT name FROM system.zookeeper WHERE path = '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard') AND name like 'block%' ORDER BY name; +SELECT 'r1' FROM system.zookeeper WHERE path = '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard') || '/replicas' AND name LIKE '%'|| getMacro('replica') ||'%' ORDER BY name; SELECT '========'; -SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1') AND name LIKE 'block%' ORDER BY name; -SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1/replicas') AND name LIKE '%r1%' ORDER BY name; +SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard')) AND name LIKE 'block%' ORDER BY name; +SELECT 'r1' FROM system.zookeeper WHERE path IN ('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard') || '/replicas') AND name LIKE '%' || getMacro('replica') || '%' ORDER BY name; SELECT '========'; -SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1', - '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1/replicas') AND name LIKE 'block%' ORDER BY name; +SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard'), + '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard') || '/replicas') AND name LIKE 'block%' ORDER BY name; SELECT '========'; -SELECT name FROM system.zookeeper WHERE path IN (SELECT concat('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1/', name) - FROM system.zookeeper WHERE (name != 'replicas' AND name NOT LIKE 'leader_election%' AND path = '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1')) ORDER BY name; +SELECT name FROM system.zookeeper WHERE path IN (SELECT concat('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard') || '/', name) + FROM system.zookeeper WHERE (name != 'replicas' AND name NOT LIKE 'leader_election%' AND name NOT LIKE 'zero_copy_%' AND path = '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard'))) ORDER BY name; DROP TABLE IF EXISTS sample_table; diff --git a/tests/queries/0_stateless/02439_merge_selecting_partitions.sql b/tests/queries/0_stateless/02439_merge_selecting_partitions.sql index 1d01fde56d6..0142afba7f2 100644 --- a/tests/queries/0_stateless/02439_merge_selecting_partitions.sql +++ b/tests/queries/0_stateless/02439_merge_selecting_partitions.sql @@ -21,7 +21,7 @@ select sleepEachRow(3) as higher_probability_of_reproducing_the_issue format Nul system flush logs; -- it should not list unneeded partitions where we cannot merge anything -select * from system.zookeeper_log where path like '/test/02439/s1/' || currentDatabase() || '/block_numbers/%' +select * from system.zookeeper_log where path like '/test/02439/' || getMacro('shard') || '/' || currentDatabase() || '/block_numbers/%' and op_num in ('List', 'SimpleList', 'FilteredList') and path not like '%/block_numbers/1' and path not like '%/block_numbers/123' and event_time >= now() - interval 1 minute From 01000e8b9edddbbe337d5e6287c20b9b88a64cc2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 14:57:02 +0100 Subject: [PATCH 154/192] Merge with master --- contrib/abseil-cpp-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/abseil-cpp-cmake/CMakeLists.txt b/contrib/abseil-cpp-cmake/CMakeLists.txt index e84b4d46c4a..e6c3268c57a 100644 --- a/contrib/abseil-cpp-cmake/CMakeLists.txt +++ b/contrib/abseil-cpp-cmake/CMakeLists.txt @@ -2683,6 +2683,7 @@ absl_cc_library( "${DIR}/status.h" SRCS "${DIR}/internal/status_internal.h" + "${DIR}/internal/status_internal.cc" "${DIR}/status.cc" "${DIR}/status_payload_printer.h" "${DIR}/status_payload_printer.cc" @@ -2761,7 +2762,6 @@ absl_cc_library( "${DIR}/has_absl_stringify.h" "${DIR}/internal/damerau_levenshtein_distance.h" "${DIR}/internal/string_constant.h" - "${DIR}/internal/has_absl_stringify.h" "${DIR}/match.h" "${DIR}/numbers.h" "${DIR}/str_cat.h" From 4d5becb4deff91f4e9675624175f2207ea2d9ccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 17 Nov 2023 15:01:54 +0100 Subject: [PATCH 155/192] Adapt test_storage_s3/test.py::test_predefined_connection_configuration --- tests/integration/test_storage_s3/test.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 01ade1acc4d..3dd3c9e39d0 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -944,13 +944,6 @@ def test_predefined_connection_configuration(started_cluster): instance.query("GRANT SELECT ON *.* TO user") instance.query(f"drop table if exists {name}", user="user") - error = instance.query_and_get_error( - f"CREATE TABLE {name} (id UInt32) ENGINE = S3(s3_conf1, format='CSV')" - ) - assert ( - "To execute this query, it's necessary to have the grant NAMED COLLECTION ON s3_conf1" - in error - ) error = instance.query_and_get_error( f"CREATE TABLE {name} (id UInt32) ENGINE = S3(s3_conf1, format='CSV')", user="user", @@ -975,11 +968,6 @@ def test_predefined_connection_configuration(started_cluster): ) assert result == instance.query("SELECT number FROM numbers(10)") - error = instance.query_and_get_error("SELECT * FROM s3(no_collection)") - assert ( - "To execute this query, it's necessary to have the grant NAMED COLLECTION ON no_collection" - in error - ) error = instance.query_and_get_error("SELECT * FROM s3(no_collection)", user="user") assert ( "To execute this query, it's necessary to have the grant NAMED COLLECTION ON no_collection" From a530d8c80db5cd03e567f7eb6962824cfc12b9f2 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 17 Nov 2023 15:25:43 +0000 Subject: [PATCH 156/192] Fix flaky test #56926 --- .../02494_query_cache_events.reference | 3 --- .../0_stateless/02494_query_cache_events.sql | 15 ++------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/tests/queries/0_stateless/02494_query_cache_events.reference b/tests/queries/0_stateless/02494_query_cache_events.reference index 9bcd2820f27..00510f3a0c6 100644 --- a/tests/queries/0_stateless/02494_query_cache_events.reference +++ b/tests/queries/0_stateless/02494_query_cache_events.reference @@ -1,7 +1,4 @@ ---- 1 -0 1 ---- 1 0 1 1 0 diff --git a/tests/queries/0_stateless/02494_query_cache_events.sql b/tests/queries/0_stateless/02494_query_cache_events.sql index 05c0acad4b8..f92e71cb50f 100644 --- a/tests/queries/0_stateless/02494_query_cache_events.sql +++ b/tests/queries/0_stateless/02494_query_cache_events.sql @@ -4,20 +4,7 @@ -- Start with empty query cache QC SYSTEM DROP QUERY CACHE; --- Run a query with QC on. The first execution is a QC miss. -SELECT '---'; SELECT 1 SETTINGS use_query_cache = true; - -SYSTEM FLUSH LOGS; -SELECT ProfileEvents['QueryCacheHits'], ProfileEvents['QueryCacheMisses'] -FROM system.query_log -WHERE type = 'QueryFinish' - AND current_database = currentDatabase() - AND query = 'SELECT 1 SETTINGS use_query_cache = true;'; - - --- Run previous query again with query cache on -SELECT '---'; SELECT 1 SETTINGS use_query_cache = true; SYSTEM FLUSH LOGS; @@ -28,4 +15,6 @@ WHERE type = 'QueryFinish' AND query = 'SELECT 1 SETTINGS use_query_cache = true;' ORDER BY event_time_microseconds; +-- (The 1st execution was a cache miss, the 2nd execution was a cache hit) + SYSTEM DROP QUERY CACHE; From dd626d51cbc3b7307b7d48279d31f02dca8ec302 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 17 Nov 2023 16:36:19 +0100 Subject: [PATCH 157/192] Fix perf tests report when there are no tests (#56881) * fix perf tests report when there are no tests * Automatic style fix * Update docker/test/performance-comparison/compare.sh --------- Co-authored-by: robot-clickhouse --- docker/test/performance-comparison/compare.sh | 10 +++++ docker/test/performance-comparison/report.py | 42 ++++++++++++++----- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 7d6de732489..f10236b7135 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -189,6 +189,8 @@ function run_tests test_prefix=right/performance fi + run_only_changed_tests=0 + # Determine which tests to run. if [ -v CHPC_TEST_GREP ] then @@ -203,6 +205,7 @@ function run_tests # tests. The lists of changed files are prepared in entrypoint.sh because # it has the repository. test_files=($(sed "s/tests\/performance/${test_prefix//\//\\/}/" changed-test-definitions.txt)) + run_only_changed_tests=1 else # The default -- run all tests found in the test dir. test_files=($(ls "$test_prefix"/*.xml)) @@ -226,6 +229,13 @@ function run_tests test_files=("${test_files[@]}") fi + if [ "$run_only_changed_tests" -ne 0 ]; then + if [ ${#test_files[@]} -eq 0 ]; then + time "$script_dir/report.py" --no-tests-run > report.html + exit 0 + fi + fi + # For PRs w/o changes in test definitons, test only a subset of queries, # and run them less times. If the corresponding environment variables are # already set, keep those values. diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 7da30ba7a08..c2bc773bd54 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -19,6 +19,7 @@ parser.add_argument( choices=["main", "all-queries"], help="Which report to build", ) +parser.add_argument("--no-tests-run", action="store_true", default=False) args = parser.parse_args() tables = [] @@ -354,6 +355,36 @@ if args.report == "main": add_tested_commits() + def print_status(status, message): + print( + ( + """ + + + """.format( + status=status, message=message + ) + ) + ) + + if args.no_tests_run: + for t in tables: + print(t) + print( + "

No tests to run. Only changed tests were run, but all changed tests are from another batch.

" + ) + print( + f""" + + {os.getenv("CHPC_ADD_REPORT_LINKS") or ''} + + + """ + ) + # Why failure? Because otherwise we will not notice if we have a bug that leads to 0 tests being run + print_status("failure", "No tests changed, nothing to run") + exit(0) + run_error_rows = tsvRows("run-errors.tsv") error_tests += len(run_error_rows) addSimpleTable("Run Errors", ["Test", "Error"], run_error_rows) @@ -646,16 +677,7 @@ if args.report == "main": status = "failure" message = "Errors while building the report." - print( - ( - """ - - - """.format( - status=status, message=message - ) - ) - ) + print_status(status, message) elif args.report == "all-queries": print((header_template.format())) From 29e64347b939638b70d4b9397b9f2d1972f75008 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Fri, 17 Nov 2023 17:37:14 +0100 Subject: [PATCH 158/192] improve exception message --- src/Storages/MergeTree/DataPartsExchange.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 0192fb1868b..43642b3ba79 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -398,7 +398,8 @@ MergeTreeData::DataPartPtr Service::findPart(const String & name) if (!pred_result) throw Exception( ErrorCodes::ABORTED, - "Part {} is in PreActive state for {} ms. Another host has to be asked.", + "Could not exchange part {} as it's in preActive state ({} ms) and it uses zero copy replication. " + "This is expected behaviour and the client will retry fetching the part automatically.", name, wait_timeout_ms); return part; From 6366819f120f5fd108acd60079921ff6aa595116 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 17 Nov 2023 16:52:20 +0000 Subject: [PATCH 159/192] Fix generating deep nested columns in CapnProto/Protobuf schemas --- src/Formats/StructureToFormatSchemaUtils.cpp | 4 ++ ...apnp_protobuf_auto_schema_nested.reference | 52 +++++++++++++++++++ ...02920_capnp_protobuf_auto_schema_nested.sh | 21 ++++++++ 3 files changed, 77 insertions(+) create mode 100644 tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.reference create mode 100755 tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.sh diff --git a/src/Formats/StructureToFormatSchemaUtils.cpp b/src/Formats/StructureToFormatSchemaUtils.cpp index 47701fa4f81..c56ff821a4a 100644 --- a/src/Formats/StructureToFormatSchemaUtils.cpp +++ b/src/Formats/StructureToFormatSchemaUtils.cpp @@ -96,6 +96,10 @@ NamesAndTypesList collectNested(const NamesAndTypesList & names_and_types, bool nested[field_name].emplace_back(nested_name, type); } + /// Collect nested recursively. + for (auto & [field_name, elements] : nested) + elements = collectNested(elements, allow_split_by_underscore, format_name); + for (const auto & [field_name, elements]: nested) result.emplace_back(field_name, std::make_shared(elements.getTypes(), elements.getNames())); diff --git a/tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.reference b/tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.reference new file mode 100644 index 00000000000..9874bc57142 --- /dev/null +++ b/tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.reference @@ -0,0 +1,52 @@ + +message Message +{ + message H + { + uint32 k = 1; + } + H h = 1; + message A + { + uint32 g = 1; + message B + { + uint32 c = 1; + uint32 f = 2; + message D + { + uint32 e = 1; + } + D d = 3; + } + B b = 2; + } + A a = 2; +} +46 (45,(42,44,43)) + +struct Message +{ + struct H + { + k @0 : UInt8; + } + h @0 : H; + struct A + { + g @0 : UInt8; + struct B + { + c @0 : UInt8; + f @1 : UInt8; + struct D + { + e @0 : UInt8; + } + d @2 : D; + } + b @1 : B; + } + a @1 : A; +} +(46) (45,(42,44,(43))) diff --git a/tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.sh b/tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.sh new file mode 100755 index 00000000000..aee6b866719 --- /dev/null +++ b/tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +SCHEMA_FILE=$CLICKHOUSE_TEST_UNIQUE_NAME-schema +FILE=$CLICKHOUSE_TEST_UNIQUE_NAME + +$CLICKHOUSE_LOCAL -q "select 42 as \`a.b.c\`, 43 as \`a.b.d.e\`, 44 as \`a.b.f\`, 45 as \`a.g\`, 46 as \`h.k\` format Protobuf settings output_format_schema='$SCHEMA_FILE.proto'" > $FILE.pb +tail -n +2 $SCHEMA_FILE.proto +$CLICKHOUSE_LOCAL -q "select * from file('$FILE.pb') settings format_schema='$SCHEMA_FILE:Message'" + +$CLICKHOUSE_LOCAL -q "select 42 as a_b_c, 43 as a_b_d_e, 44 as a_b_f, 45 as a_g, 46 as h_k format CapnProto settings output_format_schema='$SCHEMA_FILE.capnp'" > $FILE.capnp +tail -n +2 $SCHEMA_FILE.capnp +$CLICKHOUSE_LOCAL -q "select * from file('$FILE.capnp') settings format_schema='$SCHEMA_FILE:Message'" + +rm $SCHEMA_FILE* +rm $FILE.* + From fcce5409f47c4db15ffedded1789e34f9e528010 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 16 Nov 2023 15:17:19 +0000 Subject: [PATCH 160/192] Simplify --- cmake/cpu_features.cmake | 50 ++++++++++++++-------------- contrib/aws-cmake/AwsSIMD.cmake | 2 +- contrib/fastops-cmake/CMakeLists.txt | 4 +-- contrib/rocksdb-cmake/CMakeLists.txt | 6 ++-- 4 files changed, 29 insertions(+), 33 deletions(-) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index 765e36403ad..484c139f1a7 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -134,60 +134,60 @@ elseif (ARCH_AMD64) # ClickHouse can be cross-compiled (e.g. on an ARM host for x86) but it is also possible to build ClickHouse on x86 w/o AVX for x86 w/ # AVX. We only assume that the compiler can emit certain SIMD instructions, we don't care if the host system is able to run the binary. - SET (HAVE_SSSE3 1) - SET (HAVE_SSE41 1) - SET (HAVE_SSE42 1) - SET (HAVE_PCLMULQDQ 1) - SET (HAVE_POPCNT 1) - SET (HAVE_AVX 1) - SET (HAVE_AVX2 1) - SET (HAVE_AVX512 1) - SET (HAVE_AVX512_VBMI 1) - SET (HAVE_BMI 1) - SET (HAVE_BMI2 1) - - if (HAVE_SSSE3 AND ENABLE_SSSE3) + if (ENABLE_SSSE3) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mssse3") endif () - if (HAVE_SSE41 AND ENABLE_SSE41) + + if (ENABLE_SSE41) set (COMPILER_FLAGS "${COMPILER_FLAGS} -msse4.1") endif () - if (HAVE_SSE42 AND ENABLE_SSE42) + + if (ENABLE_SSE42) set (COMPILER_FLAGS "${COMPILER_FLAGS} -msse4.2") endif () - if (HAVE_PCLMULQDQ AND ENABLE_PCLMULQDQ) + + if (ENABLE_PCLMULQDQ) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mpclmul") endif () - if (HAVE_POPCNT AND ENABLE_POPCNT) + + if (ENABLE_POPCNT) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mpopcnt") endif () - if (HAVE_AVX AND ENABLE_AVX) + + if (ENABLE_AVX) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx") endif () - if (HAVE_AVX2 AND ENABLE_AVX2) + + if (ENABLE_AVX2) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx2") endif () - if (HAVE_AVX512 AND ENABLE_AVX512) + + if (ENABLE_AVX512) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512f -mavx512bw -mavx512vl") endif () - if (HAVE_AVX512 AND ENABLE_AVX512 AND HAVE_AVX512_VBMI AND ENABLE_AVX512_VBMI) + + if (ENABLE_AVX512 AND ENABLE_AVX512_VBMI) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512vbmi") endif () - if (HAVE_BMI AND ENABLE_BMI) + + if (ENABLE_BMI) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi") endif () - if (HAVE_BMI2 AND HAVE_AVX2 AND ENABLE_AVX2 AND ENABLE_BMI2) + + if (ENABLE_AVX2 AND ENABLE_BMI2) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi2") endif () + if (ENABLE_AVX512_FOR_SPEC_OP) set (X86_INTRINSICS_FLAGS "") - if (HAVE_BMI) + if (1) set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mbmi") endif () - if (HAVE_AVX512) + if (1) set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx512f -mavx512bw -mavx512vl -mprefer-vector-width=256") endif () endif () + else () # RISC-V + exotic platforms endif () diff --git a/contrib/aws-cmake/AwsSIMD.cmake b/contrib/aws-cmake/AwsSIMD.cmake index a2f50f27d4e..24f7628e86f 100644 --- a/contrib/aws-cmake/AwsSIMD.cmake +++ b/contrib/aws-cmake/AwsSIMD.cmake @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0. if (USE_CPU_EXTENSIONS) - if (HAVE_AVX2) + if (ENABLE_AVX2) set (AVX2_CFLAGS "-mavx -mavx2") set (HAVE_AVX2_INTRINSICS 1) set (HAVE_MM256_EXTRACT_EPI64 1) diff --git a/contrib/fastops-cmake/CMakeLists.txt b/contrib/fastops-cmake/CMakeLists.txt index e9aa4803583..1b09b736b2a 100644 --- a/contrib/fastops-cmake/CMakeLists.txt +++ b/contrib/fastops-cmake/CMakeLists.txt @@ -13,12 +13,10 @@ set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/fastops") set(SRCS "") -if(HAVE_AVX) +if(ARCH_AMD64) set (SRCS ${SRCS} "${LIBRARY_DIR}/fastops/avx/ops_avx.cpp") set_source_files_properties("${LIBRARY_DIR}/fastops/avx/ops_avx.cpp" PROPERTIES COMPILE_FLAGS "-mavx -DNO_AVX2") -endif() -if(HAVE_AVX2) set (SRCS ${SRCS} "${LIBRARY_DIR}/fastops/avx2/ops_avx2.cpp") set_source_files_properties("${LIBRARY_DIR}/fastops/avx2/ops_avx2.cpp" PROPERTIES COMPILE_FLAGS "-mavx2 -mfma") endif() diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt index 2b6c48f0b38..7d7666dff87 100644 --- a/contrib/rocksdb-cmake/CMakeLists.txt +++ b/contrib/rocksdb-cmake/CMakeLists.txt @@ -93,11 +93,9 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm64|ARM64") endif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm64|ARM64") -if(HAVE_SSE42) +if(ENABLE_AVX2 AND ENABLE_PCLMULQDQ) add_definitions(-DHAVE_SSE42) add_definitions(-DHAVE_PCLMUL) -elseif(FORCE_SSE42) - message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled") endif() set (HAVE_THREAD_LOCAL 1) @@ -429,7 +427,7 @@ set(SOURCES ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/util/memarena.cc rocksdb_build_version.cc) -if(HAVE_SSE42) +if(ENABLE_SSE42 AND ENABLE_PCLMULQDQ) set_source_files_properties( "${ROCKSDB_SOURCE_DIR}/util/crc32c.cc" PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul") From c51429b1ef006b2bced059624312af5c1dc1f07f Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 16 Nov 2023 15:24:47 +0000 Subject: [PATCH 161/192] Simplify more --- cmake/cpu_features.cmake | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index 484c139f1a7..cfa9c314bc0 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -150,6 +150,10 @@ elseif (ARCH_AMD64) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mpclmul") endif () + if (ENABLE_BMI) + set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi") + endif () + if (ENABLE_POPCNT) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mpopcnt") endif () @@ -160,32 +164,20 @@ elseif (ARCH_AMD64) if (ENABLE_AVX2) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx2") + if (ENABLE_BMI2) + set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi2") + endif () endif () if (ENABLE_AVX512) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512f -mavx512bw -mavx512vl") - endif () - - if (ENABLE_AVX512 AND ENABLE_AVX512_VBMI) - set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512vbmi") - endif () - - if (ENABLE_BMI) - set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi") - endif () - - if (ENABLE_AVX2 AND ENABLE_BMI2) - set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi2") + if (ENABLE_AVX512_VBMI) + set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512vbmi") + endif () endif () if (ENABLE_AVX512_FOR_SPEC_OP) - set (X86_INTRINSICS_FLAGS "") - if (1) - set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mbmi") - endif () - if (1) - set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx512f -mavx512bw -mavx512vl -mprefer-vector-width=256") - endif () + set (X86_INTRINSICS_FLAGS "-mbmi -mavx512f -mavx512bw -mavx512vl -mprefer-vector-width=256") endif () else () From a18b7155919d7a7ffc10814043dde3e2c913f620 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Sat, 18 Nov 2023 15:44:45 +0000 Subject: [PATCH 162/192] Fix a bug --- src/Storages/AlterCommands.cpp | 17 +++++++++++++++-- .../02916_addcolumn_nested.reference | 1 + .../0_stateless/02916_addcolumn_nested.sql | 5 +++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 7eeaa2d4594..f5293c52bb0 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -33,6 +33,8 @@ #include #include +#include + namespace DB { @@ -403,10 +405,21 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) const auto transformed_columns = temporary_metadata.columns.getAll(); - for (auto it = transformed_columns.rbegin(); it != transformed_columns.rend(); it++) + auto add_column = [&](const String & name) { - const auto & transformed_column = temporary_metadata.columns.get(it->name); + const auto & transformed_column = temporary_metadata.columns.get(name); metadata.columns.add(transformed_column, after_column, first); + }; + + if (!after_column.empty() || first) + { + for (const auto & col: transformed_columns | std::views::reverse) + add_column(col.name); + } + else + { + for (const auto & col: transformed_columns) + add_column(col.name); } } else diff --git a/tests/queries/0_stateless/02916_addcolumn_nested.reference b/tests/queries/0_stateless/02916_addcolumn_nested.reference index 869d4336c62..7d79cd8731f 100644 --- a/tests/queries/0_stateless/02916_addcolumn_nested.reference +++ b/tests/queries/0_stateless/02916_addcolumn_nested.reference @@ -1,3 +1,4 @@ CREATE TABLE default.nested_table\n(\n `id` UInt64,\n `first` Nested(a Int8, b String)\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192 CREATE TABLE default.nested_table\n(\n `id` UInt64,\n `second.c` Array(Int8),\n `second.d` Array(String),\n `first` Nested(a Int8, b String)\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192 CREATE TABLE default.nested_table\n(\n `third` Nested(e Int8, f String),\n `id` UInt64,\n `second.c` Array(Int8),\n `second.d` Array(String),\n `first` Nested(a Int8, b String)\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.nested_table\n(\n `third` Nested(e Int8, f String),\n `id` UInt64,\n `second.c` Array(Int8),\n `second.d` Array(String),\n `first` Nested(a Int8, b String),\n `fourth.g` Array(Int8),\n `fourth.h` Array(String)\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02916_addcolumn_nested.sql b/tests/queries/0_stateless/02916_addcolumn_nested.sql index b23854824b5..1e64fca6a15 100644 --- a/tests/queries/0_stateless/02916_addcolumn_nested.sql +++ b/tests/queries/0_stateless/02916_addcolumn_nested.sql @@ -14,4 +14,9 @@ SET flatten_nested = 0; ALTER TABLE nested_table ADD COLUMN third Nested(e Int8, f String) FIRST; SHOW CREATE nested_table; +SET flatten_nested = 1; + +ALTER TABLE nested_table ADD COLUMN fourth Nested(g Int8, h String); +SHOW CREATE nested_table; + DROP TABLE nested_table; From 6eedd1649db62fcfbeab6eda8039e6217e155c49 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Sat, 18 Nov 2023 17:55:38 +0000 Subject: [PATCH 163/192] Resubmit: Better except for SSL authentication --- src/Server/TCPHandler.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 1da9806b4f5..1c2c16496f0 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1431,8 +1431,11 @@ void TCPHandler::receiveHello() getClientAddress(client_info)); return; } - catch (...) + catch (const Exception & e) { + if (e.code() != DB::ErrorCodes::AUTHENTICATION_FAILED) + throw; + tryLogCurrentException(log, "SSL authentication failed, falling back to password authentication"); } } From d56cbda1850974da1f4cf7d66ef52d002d3b7244 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Nov 2023 19:07:59 +0100 Subject: [PATCH 164/192] Add metrics for the number of queued jobs, which is useful for the IO thread pool --- programs/benchmark/Benchmark.cpp | 3 +- programs/copier/ClusterCopier.cpp | 3 +- src/Backups/BackupsWorker.cpp | 7 ++- src/Common/AsyncLoader.cpp | 1 + src/Common/AsyncLoader.h | 5 ++- src/Common/CurrentMetrics.cpp | 43 +++++++++++++++++++ src/Common/ThreadPool.cpp | 40 ++++++++--------- src/Common/ThreadPool.h | 14 ++++-- src/Common/examples/parallel_aggregation.cpp | 3 +- src/Common/examples/parallel_aggregation2.cpp | 3 +- .../examples/thread_creation_latency.cpp | 7 +-- src/Common/tests/gtest_async_loader.cpp | 2 + .../gtest_thread_pool_concurrent_wait.cpp | 5 ++- .../tests/gtest_thread_pool_global_full.cpp | 7 +-- src/Common/tests/gtest_thread_pool_limit.cpp | 3 +- src/Common/tests/gtest_thread_pool_loop.cpp | 3 +- .../gtest_thread_pool_schedule_exception.cpp | 5 ++- src/Coordination/Standalone/Context.cpp | 1 + src/Databases/DatabaseOnDisk.cpp | 3 +- src/Databases/DatabaseOrdinary.cpp | 3 +- src/Databases/TablesLoader.cpp | 3 +- .../CacheDictionaryUpdateQueue.cpp | 3 +- src/Dictionaries/HashedDictionary.cpp | 5 ++- src/Disks/IDisk.h | 5 ++- src/Disks/IO/ThreadPoolReader.cpp | 3 +- src/Disks/IO/ThreadPoolRemoteFSReader.cpp | 2 + .../AzureBlobStorage/AzureObjectStorage.cpp | 4 +- ...jectStorageRemoteMetadataRestoreHelper.cpp | 5 ++- .../ObjectStorageIteratorAsync.cpp | 1 + .../ObjectStorageIteratorAsync.h | 4 +- .../ObjectStorages/S3/S3ObjectStorage.cpp | 2 + src/IO/SharedThreadPools.cpp | 20 ++++++--- src/IO/SharedThreadPools.h | 5 ++- src/Interpreters/Aggregator.cpp | 7 +-- src/Interpreters/AsynchronousInsertQueue.cpp | 3 +- src/Interpreters/Context.cpp | 9 ++-- src/Interpreters/DDLWorker.cpp | 7 +-- src/Interpreters/DatabaseCatalog.cpp | 3 +- src/Interpreters/InterpreterSystemQuery.cpp | 3 +- src/Interpreters/loadMetadata.cpp | 5 ++- src/Interpreters/threadPoolCallbackRunner.h | 6 +-- src/Processors/Executors/PipelineExecutor.cpp | 3 +- .../Formats/Impl/DWARFBlockInputFormat.cpp | 3 +- .../Impl/ParallelFormattingOutputFormat.h | 3 +- .../Formats/Impl/ParallelParsingInputFormat.h | 3 +- .../Formats/Impl/ParquetBlockInputFormat.cpp | 3 +- .../Formats/Impl/ParquetBlockOutputFormat.cpp | 5 ++- .../Transforms/AggregatingTransform.h | 2 + src/Storages/Distributed/DistributedSink.cpp | 4 +- src/Storages/Hive/StorageHive.cpp | 3 +- .../MergeTree/MergeTreeBackgroundExecutor.cpp | 3 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 2 + src/Storages/StorageAzureBlob.cpp | 3 +- src/Storages/StorageDistributed.cpp | 3 +- src/Storages/StorageS3.cpp | 7 +-- src/Storages/System/StorageSystemReplicas.cpp | 3 +- utils/keeper-bench/Runner.cpp | 5 ++- 57 files changed, 222 insertions(+), 99 deletions(-) diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index ed3d4a1ea69..d6b8b38d84d 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -46,6 +46,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } namespace DB @@ -107,7 +108,7 @@ public: settings(settings_), shared_context(Context::createShared()), global_context(Context::createGlobal(shared_context.get())), - pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, concurrency) + pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, concurrency) { const auto secure = secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable; size_t connections_cnt = std::max(ports_.size(), hosts_.size()); diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index b2b4970d04f..7d58f35f62f 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -25,6 +25,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } namespace DB @@ -200,7 +201,7 @@ void ClusterCopier::discoverTablePartitions(const ConnectionTimeouts & timeouts, { /// Fetch partitions list from a shard { - ThreadPool thread_pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, num_threads ? num_threads : 2 * getNumberOfPhysicalCPUCores()); + ThreadPool thread_pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_threads ? num_threads : 2 * getNumberOfPhysicalCPUCores()); for (const TaskShardPtr & task_shard : task_table.all_shards) thread_pool.scheduleOrThrowOnError([this, timeouts, task_shard]() diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index f6020deabec..4e24269fb25 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -31,8 +31,10 @@ namespace CurrentMetrics { extern const Metric BackupsThreads; extern const Metric BackupsThreadsActive; + extern const Metric BackupsThreadsScheduled; extern const Metric RestoreThreads; extern const Metric RestoreThreadsActive; + extern const Metric RestoreThreadsScheduled; } namespace DB @@ -264,6 +266,7 @@ public: CurrentMetrics::Metric metric_threads; CurrentMetrics::Metric metric_active_threads; + CurrentMetrics::Metric metric_scheduled_threads; size_t max_threads = 0; /// What to do with a new job if a corresponding thread pool is already running `max_threads` jobs: @@ -279,6 +282,7 @@ public: { metric_threads = CurrentMetrics::BackupsThreads; metric_active_threads = CurrentMetrics::BackupsThreadsActive; + metric_active_threads = CurrentMetrics::BackupsThreadsScheduled; max_threads = num_backup_threads; /// We don't use thread pool queues for thread pools with a lot of tasks otherwise that queue could be memory-wasting. use_queue = (thread_pool_id != ThreadPoolId::BACKUP_COPY_FILES); @@ -291,6 +295,7 @@ public: { metric_threads = CurrentMetrics::RestoreThreads; metric_active_threads = CurrentMetrics::RestoreThreadsActive; + metric_active_threads = CurrentMetrics::RestoreThreadsScheduled; max_threads = num_restore_threads; use_queue = (thread_pool_id != ThreadPoolId::RESTORE_TABLES_DATA); break; @@ -301,7 +306,7 @@ public: chassert(max_threads != 0); size_t max_free_threads = 0; size_t queue_size = use_queue ? 0 : max_threads; - auto thread_pool = std::make_unique(metric_threads, metric_active_threads, max_threads, max_free_threads, queue_size); + auto thread_pool = std::make_unique(metric_threads, metric_active_threads, metric_scheduled_threads, max_threads, max_free_threads, queue_size); auto * thread_pool_ptr = thread_pool.get(); thread_pools.emplace(thread_pool_id, std::move(thread_pool)); return *thread_pool_ptr; diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp index 2e96d3eab7d..cff34ac036d 100644 --- a/src/Common/AsyncLoader.cpp +++ b/src/Common/AsyncLoader.cpp @@ -179,6 +179,7 @@ AsyncLoader::AsyncLoader(std::vector pool_initializers, bool lo .thread_pool = std::make_unique( init.metric_threads, init.metric_active_threads, + init.metric_scheduled_threads, init.max_threads, /* max_free_threads = */ 0, init.max_threads), diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h index 77905319f00..0496549001e 100644 --- a/src/Common/AsyncLoader.h +++ b/src/Common/AsyncLoader.h @@ -271,8 +271,8 @@ inline LoadTaskPtrs joinTasks(const LoadTaskPtrs & tasks1, const LoadTaskPtrs & // Basic usage example: // // Start async_loader with two thread pools (0=fg, 1=bg): // AsyncLoader async_loader({ -// {"FgPool", CurrentMetrics::AsyncLoaderThreads, CurrentMetrics::AsyncLoaderThreadsActive, .max_threads = 2, .priority{0}} -// {"BgPool", CurrentMetrics::AsyncLoaderThreads, CurrentMetrics::AsyncLoaderThreadsActive, .max_threads = 1, .priority{1}} +// {"FgPool", CurrentMetrics::AsyncLoaderThreads, ..., .max_threads = 2, .priority{0}} +// {"BgPool", CurrentMetrics::AsyncLoaderThreads, ..., .max_threads = 1, .priority{1}} // }); // // // Create and schedule a task consisting of three jobs. Job1 has no dependencies and is run first. @@ -368,6 +368,7 @@ public: String name; Metric metric_threads; Metric metric_active_threads; + Metric metric_scheduled_threads; size_t max_threads; Priority priority; }; diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index c929f4d86e2..5a4b6e80f75 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -71,92 +71,135 @@ M(RWLockActiveWriters, "Number of threads holding write lock in a table RWLock.") \ M(GlobalThread, "Number of threads in global thread pool.") \ M(GlobalThreadActive, "Number of threads in global thread pool running a task.") \ + M(GlobalThreadScheduled, "Number of queued or active jobs in global thread pool.") \ M(LocalThread, "Number of threads in local thread pools. The threads in local thread pools are taken from the global thread pool.") \ M(LocalThreadActive, "Number of threads in local thread pools running a task.") \ + M(LocalThreadScheduled, "Number of queued or active jobs in local thread pools.") \ M(MergeTreeDataSelectExecutorThreads, "Number of threads in the MergeTreeDataSelectExecutor thread pool.") \ M(MergeTreeDataSelectExecutorThreadsActive, "Number of threads in the MergeTreeDataSelectExecutor thread pool running a task.") \ + M(MergeTreeDataSelectExecutorThreadsScheduled, "Number of queued or active jobs in the MergeTreeDataSelectExecutor thread pool.") \ M(BackupsThreads, "Number of threads in the thread pool for BACKUP.") \ M(BackupsThreadsActive, "Number of threads in thread pool for BACKUP running a task.") \ + M(BackupsThreadsScheduled, "Number of queued or active jobs for BACKUP.") \ M(RestoreThreads, "Number of threads in the thread pool for RESTORE.") \ M(RestoreThreadsActive, "Number of threads in the thread pool for RESTORE running a task.") \ + M(RestoreThreadsScheduled, "Number of queued or active jobs for RESTORE.") \ M(MarksLoaderThreads, "Number of threads in thread pool for loading marks.") \ M(MarksLoaderThreadsActive, "Number of threads in the thread pool for loading marks running a task.") \ + M(MarksLoaderThreadsScheduled, "Number of queued or active jobs in the thread pool for loading marks.") \ M(IOPrefetchThreads, "Number of threads in the IO prefertch thread pool.") \ M(IOPrefetchThreadsActive, "Number of threads in the IO prefetch thread pool running a task.") \ + M(IOPrefetchThreadsScheduled, "Number of queued or active jobs in the IO prefetch thread pool.") \ M(IOWriterThreads, "Number of threads in the IO writer thread pool.") \ M(IOWriterThreadsActive, "Number of threads in the IO writer thread pool running a task.") \ + M(IOWriterThreadsScheduled, "Number of queued or active jobs in the IO writer thread pool.") \ M(IOThreads, "Number of threads in the IO thread pool.") \ M(IOThreadsActive, "Number of threads in the IO thread pool running a task.") \ + M(IOThreadsScheduled, "Number of queued or active jobs in the IO thread pool.") \ M(ThreadPoolRemoteFSReaderThreads, "Number of threads in the thread pool for remote_filesystem_read_method=threadpool.") \ M(ThreadPoolRemoteFSReaderThreadsActive, "Number of threads in the thread pool for remote_filesystem_read_method=threadpool running a task.") \ + M(ThreadPoolRemoteFSReaderThreadsScheduled, "Number of queued or active jobs in the thread pool for remote_filesystem_read_method=threadpool.") \ M(ThreadPoolFSReaderThreads, "Number of threads in the thread pool for local_filesystem_read_method=threadpool.") \ M(ThreadPoolFSReaderThreadsActive, "Number of threads in the thread pool for local_filesystem_read_method=threadpool running a task.") \ + M(ThreadPoolFSReaderThreadsScheduled, "Number of queued or active jobs in the thread pool for local_filesystem_read_method=threadpool.") \ M(BackupsIOThreads, "Number of threads in the BackupsIO thread pool.") \ M(BackupsIOThreadsActive, "Number of threads in the BackupsIO thread pool running a task.") \ + M(BackupsIOThreadsScheduled, "Number of queued or active jobs in the BackupsIO thread pool.") \ M(DiskObjectStorageAsyncThreads, "Obsolete metric, shows nothing.") \ M(DiskObjectStorageAsyncThreadsActive, "Obsolete metric, shows nothing.") \ M(StorageHiveThreads, "Number of threads in the StorageHive thread pool.") \ M(StorageHiveThreadsActive, "Number of threads in the StorageHive thread pool running a task.") \ + M(StorageHiveThreadsScheduled, "Number of queued or active jobs in the StorageHive thread pool.") \ M(TablesLoaderThreads, "Number of threads in the tables loader thread pool.") \ M(TablesLoaderThreadsActive, "Number of threads in the tables loader thread pool running a task.") \ + M(TablesLoaderThreadsScheduled, "Number of queued or active jobs in the tables loader thread pool.") \ M(DatabaseOrdinaryThreads, "Number of threads in the Ordinary database thread pool.") \ M(DatabaseOrdinaryThreadsActive, "Number of threads in the Ordinary database thread pool running a task.") \ + M(DatabaseOrdinaryThreadsScheduled, "Number of queued or active jobs in the Ordinary database thread pool.") \ M(DatabaseOnDiskThreads, "Number of threads in the DatabaseOnDisk thread pool.") \ M(DatabaseOnDiskThreadsActive, "Number of threads in the DatabaseOnDisk thread pool running a task.") \ + M(DatabaseOnDiskThreadsScheduled, "Number of queued or active jobs in the DatabaseOnDisk thread pool.") \ M(DatabaseCatalogThreads, "Number of threads in the DatabaseCatalog thread pool.") \ M(DatabaseCatalogThreadsActive, "Number of threads in the DatabaseCatalog thread pool running a task.") \ + M(DatabaseCatalogThreadsScheduled, "Number of queued or active jobs in the DatabaseCatalog thread pool.") \ M(DestroyAggregatesThreads, "Number of threads in the thread pool for destroy aggregate states.") \ M(DestroyAggregatesThreadsActive, "Number of threads in the thread pool for destroy aggregate states running a task.") \ + M(DestroyAggregatesThreadsScheduled, "Number of queued or active jobs in the thread pool for destroy aggregate states.") \ M(HashedDictionaryThreads, "Number of threads in the HashedDictionary thread pool.") \ M(HashedDictionaryThreadsActive, "Number of threads in the HashedDictionary thread pool running a task.") \ + M(HashedDictionaryThreadsScheduled, "Number of queued or active jobs in the HashedDictionary thread pool.") \ M(CacheDictionaryThreads, "Number of threads in the CacheDictionary thread pool.") \ M(CacheDictionaryThreadsActive, "Number of threads in the CacheDictionary thread pool running a task.") \ + M(CacheDictionaryThreadsScheduled, "Number of queued or active jobs in the CacheDictionary thread pool.") \ M(ParallelFormattingOutputFormatThreads, "Number of threads in the ParallelFormattingOutputFormatThreads thread pool.") \ M(ParallelFormattingOutputFormatThreadsActive, "Number of threads in the ParallelFormattingOutputFormatThreads thread pool running a task.") \ + M(ParallelFormattingOutputFormatThreadsScheduled, "Number of queued or active jobs in the ParallelFormattingOutputFormatThreads thread pool.") \ M(ParallelParsingInputFormatThreads, "Number of threads in the ParallelParsingInputFormat thread pool.") \ M(ParallelParsingInputFormatThreadsActive, "Number of threads in the ParallelParsingInputFormat thread pool running a task.") \ + M(ParallelParsingInputFormatThreadsScheduled, "Number of queued or active jobs in the ParallelParsingInputFormat thread pool.") \ M(MergeTreeBackgroundExecutorThreads, "Number of threads in the MergeTreeBackgroundExecutor thread pool.") \ M(MergeTreeBackgroundExecutorThreadsActive, "Number of threads in the MergeTreeBackgroundExecutor thread pool running a task.") \ + M(MergeTreeBackgroundExecutorThreadsScheduled, "Number of queued or active jobs in the MergeTreeBackgroundExecutor thread pool.") \ M(AsynchronousInsertThreads, "Number of threads in the AsynchronousInsert thread pool.") \ M(AsynchronousInsertThreadsActive, "Number of threads in the AsynchronousInsert thread pool running a task.") \ + M(AsynchronousInsertThreadsScheduled, "Number of queued or active jobs in the AsynchronousInsert thread pool.") \ M(StartupSystemTablesThreads, "Number of threads in the StartupSystemTables thread pool.") \ M(StartupSystemTablesThreadsActive, "Number of threads in the StartupSystemTables thread pool running a task.") \ + M(StartupSystemTablesThreadsScheduled, "Number of queued or active jobs in the StartupSystemTables thread pool.") \ M(AggregatorThreads, "Number of threads in the Aggregator thread pool.") \ M(AggregatorThreadsActive, "Number of threads in the Aggregator thread pool running a task.") \ + M(AggregatorThreadsScheduled, "Number of queued or active jobs in the Aggregator thread pool.") \ M(DDLWorkerThreads, "Number of threads in the DDLWorker thread pool for ON CLUSTER queries.") \ M(DDLWorkerThreadsActive, "Number of threads in the DDLWORKER thread pool for ON CLUSTER queries running a task.") \ + M(DDLWorkerThreadsScheduled, "Number of queued or active jobs in the DDLWORKER thread pool for ON CLUSTER queries.") \ M(StorageDistributedThreads, "Number of threads in the StorageDistributed thread pool.") \ M(StorageDistributedThreadsActive, "Number of threads in the StorageDistributed thread pool running a task.") \ + M(StorageDistributedThreadsScheduled, "Number of queued or active jobs in the StorageDistributed thread pool.") \ M(DistributedInsertThreads, "Number of threads used for INSERT into Distributed.") \ M(DistributedInsertThreadsActive, "Number of threads used for INSERT into Distributed running a task.") \ + M(DistributedInsertThreadsScheduled, "Number of queued or active jobs used for INSERT into Distributed.") \ M(StorageS3Threads, "Number of threads in the StorageS3 thread pool.") \ M(StorageS3ThreadsActive, "Number of threads in the StorageS3 thread pool running a task.") \ + M(StorageS3ThreadsScheduled, "Number of queued or active jobs in the StorageS3 thread pool.") \ M(ObjectStorageS3Threads, "Number of threads in the S3ObjectStorage thread pool.") \ M(ObjectStorageS3ThreadsActive, "Number of threads in the S3ObjectStorage thread pool running a task.") \ + M(ObjectStorageS3ThreadsScheduled, "Number of queued or active jobs in the S3ObjectStorage thread pool.") \ M(ObjectStorageAzureThreads, "Number of threads in the AzureObjectStorage thread pool.") \ M(ObjectStorageAzureThreadsActive, "Number of threads in the AzureObjectStorage thread pool running a task.") \ + M(ObjectStorageAzureThreadsScheduled, "Number of queued or active jobs in the AzureObjectStorage thread pool.") \ M(MergeTreePartsLoaderThreads, "Number of threads in the MergeTree parts loader thread pool.") \ M(MergeTreePartsLoaderThreadsActive, "Number of threads in the MergeTree parts loader thread pool running a task.") \ + M(MergeTreePartsLoaderThreadsScheduled, "Number of queued or active jobs in the MergeTree parts loader thread pool.") \ M(MergeTreeOutdatedPartsLoaderThreads, "Number of threads in the threadpool for loading Outdated data parts.") \ M(MergeTreeOutdatedPartsLoaderThreadsActive, "Number of active threads in the threadpool for loading Outdated data parts.") \ + M(MergeTreeOutdatedPartsLoaderThreadsScheduled, "Number of queued or active jobs in the threadpool for loading Outdated data parts.") \ M(MergeTreePartsCleanerThreads, "Number of threads in the MergeTree parts cleaner thread pool.") \ M(MergeTreePartsCleanerThreadsActive, "Number of threads in the MergeTree parts cleaner thread pool running a task.") \ + M(MergeTreePartsCleanerThreadsScheduled, "Number of queued or active jobs in the MergeTree parts cleaner thread pool.") \ M(IDiskCopierThreads, "Number of threads for copying data between disks of different types.") \ M(IDiskCopierThreadsActive, "Number of threads for copying data between disks of different types running a task.") \ + M(IDiskCopierThreadsScheduled, "Number of queued or active jobs for copying data between disks of different types.") \ M(SystemReplicasThreads, "Number of threads in the system.replicas thread pool.") \ M(SystemReplicasThreadsActive, "Number of threads in the system.replicas thread pool running a task.") \ + M(SystemReplicasThreadsScheduled, "Number of queued or active jobs in the system.replicas thread pool.") \ M(RestartReplicaThreads, "Number of threads in the RESTART REPLICA thread pool.") \ M(RestartReplicaThreadsActive, "Number of threads in the RESTART REPLICA thread pool running a task.") \ + M(RestartReplicaThreadsScheduled, "Number of queued or active jobs in the RESTART REPLICA thread pool.") \ M(QueryPipelineExecutorThreads, "Number of threads in the PipelineExecutor thread pool.") \ M(QueryPipelineExecutorThreadsActive, "Number of threads in the PipelineExecutor thread pool running a task.") \ + M(QueryPipelineExecutorThreadsScheduled, "Number of queued or active jobs in the PipelineExecutor thread pool.") \ M(ParquetDecoderThreads, "Number of threads in the ParquetBlockInputFormat thread pool.") \ M(ParquetDecoderThreadsActive, "Number of threads in the ParquetBlockInputFormat thread pool running a task.") \ + M(ParquetDecoderThreadsScheduled, "Number of queued or active jobs in the ParquetBlockInputFormat thread pool.") \ M(ParquetEncoderThreads, "Number of threads in ParquetBlockOutputFormat thread pool.") \ M(ParquetEncoderThreadsActive, "Number of threads in ParquetBlockOutputFormat thread pool running a task.") \ + M(ParquetEncoderThreadsScheduled, "Number of queued or active jobs in ParquetBlockOutputFormat thread pool.") \ M(DWARFReaderThreads, "Number of threads in the DWARFBlockInputFormat thread pool.") \ M(DWARFReaderThreadsActive, "Number of threads in the DWARFBlockInputFormat thread pool running a task.") \ + M(DWARFReaderThreadsScheduled, "Number of queued or active jobs in the DWARFBlockInputFormat thread pool.") \ M(OutdatedPartsLoadingThreads, "Number of threads in the threadpool for loading Outdated data parts.") \ M(OutdatedPartsLoadingThreadsActive, "Number of active threads in the threadpool for loading Outdated data parts.") \ + M(OutdatedPartsLoadingThreadsScheduled, "Number of queued or active jobs in the threadpool for loading Outdated data parts.") \ M(DistributedBytesToInsert, "Number of pending bytes to process for asynchronous insertion into Distributed tables. Number of bytes for every shard is summed.") \ M(BrokenDistributedBytesToInsert, "Number of bytes for asynchronous insertion into Distributed tables that has been marked as broken. Number of bytes for every shard is summed.") \ M(DistributedFilesToInsert, "Number of pending files to process for asynchronous insertion into Distributed tables. Number of files for every shard is summed.") \ diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 4a5bdeffcee..8cba13373b9 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -25,13 +25,14 @@ namespace CurrentMetrics { extern const Metric GlobalThread; extern const Metric GlobalThreadActive; + extern const Metric GlobalThreadScheduled; } static constexpr auto DEFAULT_THREAD_NAME = "ThreadPool"; template -ThreadPoolImpl::ThreadPoolImpl(Metric metric_threads_, Metric metric_active_threads_) - : ThreadPoolImpl(metric_threads_, metric_active_threads_, getNumberOfPhysicalCPUCores()) +ThreadPoolImpl::ThreadPoolImpl(Metric metric_threads_, Metric metric_active_threads_, Metric metric_scheduled_jobs_) + : ThreadPoolImpl(metric_threads_, metric_active_threads_, metric_scheduled_jobs_, getNumberOfPhysicalCPUCores()) { } @@ -40,8 +41,9 @@ template ThreadPoolImpl::ThreadPoolImpl( Metric metric_threads_, Metric metric_active_threads_, + Metric metric_scheduled_jobs_, size_t max_threads_) - : ThreadPoolImpl(metric_threads_, metric_active_threads_, max_threads_, max_threads_, max_threads_) + : ThreadPoolImpl(metric_threads_, metric_active_threads_, metric_scheduled_jobs_, max_threads_, max_threads_, max_threads_) { } @@ -49,12 +51,14 @@ template ThreadPoolImpl::ThreadPoolImpl( Metric metric_threads_, Metric metric_active_threads_, + Metric metric_scheduled_jobs_, size_t max_threads_, size_t max_free_threads_, size_t queue_size_, bool shutdown_on_exception_) : metric_threads(metric_threads_) , metric_active_threads(metric_active_threads_) + , metric_scheduled_jobs(metric_scheduled_jobs_) , max_threads(max_threads_) , max_free_threads(std::min(max_free_threads_, max_threads)) , queue_size(queue_size_ ? std::max(queue_size_, max_threads) : 0 /* zero means the queue is unlimited */) @@ -187,6 +191,7 @@ ReturnType ThreadPoolImpl::scheduleImpl(Job job, Priority priority, std: jobs.emplace(std::move(job), priority, + metric_scheduled_jobs, /// Tracing context on this thread is used as parent context for the sub-thread that runs the job propagate_opentelemetry_tracing_context ? DB::OpenTelemetry::CurrentContext() : DB::OpenTelemetry::TracingContextOnThread(), /// capture_frame_pointers @@ -346,13 +351,8 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ /// This is inside the loop to also reset previous thread names set inside the jobs. setThreadName(DEFAULT_THREAD_NAME); - /// A copy of parent trace context - DB::OpenTelemetry::TracingContextOnThread parent_thread_trace_context; - - std::vector thread_frame_pointers; - /// Get a job from the queue. - Job job; + std::optional job_data; { std::unique_lock lock(mutex); @@ -393,12 +393,8 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ } /// boost::priority_queue does not provide interface for getting non-const reference to an element - /// to prevent us from modifying its priority. We have to use const_cast to force move semantics on JobWithPriority::job. - job = std::move(const_cast(jobs.top().job)); - parent_thread_trace_context = std::move(const_cast(jobs.top().thread_trace_context)); - DB::Exception::enable_job_stack_trace = jobs.top().enable_job_stack_trace; - if (DB::Exception::enable_job_stack_trace) - thread_frame_pointers = std::move(const_cast &>(jobs.top().frame_pointers)); + /// to prevent us from modifying its priority. We have to use const_cast to force move semantics on JobWithPriority. + job_data = std::move(const_cast(jobs.top())); jobs.pop(); /// We don't run jobs after `shutdown` is set, but we have to properly dequeue all jobs and finish them. @@ -412,18 +408,17 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ ALLOW_ALLOCATIONS_IN_SCOPE; /// Set up tracing context for this thread by its parent context. - DB::OpenTelemetry::TracingContextHolder thread_trace_context("ThreadPool::worker()", parent_thread_trace_context); + DB::OpenTelemetry::TracingContextHolder thread_trace_context("ThreadPool::worker()", job_data->thread_trace_context); /// Run the job. try { if (DB::Exception::enable_job_stack_trace) - DB::Exception::thread_frame_pointers = std::move(thread_frame_pointers); - + DB::Exception::thread_frame_pointers = std::move(job_data->frame_pointers); CurrentMetrics::Increment metric_active_pool_threads(metric_active_threads); - job(); + job_data->job(); if (thread_trace_context.root_span.isTraceEnabled()) { @@ -437,13 +432,13 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ else { /// If the thread name is not set, use the type name of the job instead - thread_trace_context.root_span.operation_name = demangle(job.target_type().name()); + thread_trace_context.root_span.operation_name = demangle(job_data->job.target_type().name()); } } /// job should be reset before decrementing scheduled_jobs to /// ensure that the Job destroyed before wait() returns. - job = {}; + job_data.reset(); } catch (...) { @@ -452,7 +447,7 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ /// job should be reset before decrementing scheduled_jobs to /// ensure that the Job destroyed before wait() returns. - job = {}; + job_data.reset(); } job_is_done = true; @@ -475,6 +470,7 @@ GlobalThreadPool::GlobalThreadPool( : FreeThreadPool( CurrentMetrics::GlobalThread, CurrentMetrics::GlobalThreadActive, + CurrentMetrics::GlobalThreadScheduled, max_threads_, max_free_threads_, queue_size_, diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h index f5721146e09..c8eefedd838 100644 --- a/src/Common/ThreadPool.h +++ b/src/Common/ThreadPool.h @@ -41,18 +41,20 @@ public: using Metric = CurrentMetrics::Metric; /// Maximum number of threads is based on the number of physical cores. - ThreadPoolImpl(Metric metric_threads_, Metric metric_active_threads_); + ThreadPoolImpl(Metric metric_threads_, Metric metric_active_threads_, Metric metric_scheduled_jobs_); /// Size is constant. Up to num_threads are created on demand and then run until shutdown. explicit ThreadPoolImpl( Metric metric_threads_, Metric metric_active_threads_, + Metric metric_scheduled_jobs_, size_t max_threads_); /// queue_size - maximum number of running plus scheduled jobs. It can be greater than max_threads. Zero means unlimited. ThreadPoolImpl( Metric metric_threads_, Metric metric_active_threads_, + Metric metric_scheduled_jobs_, size_t max_threads_, size_t max_free_threads_, size_t queue_size_, @@ -113,6 +115,7 @@ private: Metric metric_threads; Metric metric_active_threads; + Metric metric_scheduled_jobs; size_t max_threads; size_t max_free_threads; @@ -127,14 +130,19 @@ private: { Job job; Priority priority; + CurrentMetrics::Increment metric_increment; DB::OpenTelemetry::TracingContextOnThread thread_trace_context; /// Call stacks of all jobs' schedulings leading to this one std::vector frame_pointers; bool enable_job_stack_trace = false; - JobWithPriority(Job job_, Priority priority_, const DB::OpenTelemetry::TracingContextOnThread & thread_trace_context_, bool capture_frame_pointers = false) - : job(job_), priority(priority_), thread_trace_context(thread_trace_context_), enable_job_stack_trace(capture_frame_pointers) + JobWithPriority( + Job job_, Priority priority_, CurrentMetrics::Metric metric, + const DB::OpenTelemetry::TracingContextOnThread & thread_trace_context_, + bool capture_frame_pointers) + : job(job_), priority(priority_), metric_increment(metric), + thread_trace_context(thread_trace_context_), enable_job_stack_trace(capture_frame_pointers) { if (!capture_frame_pointers) return; diff --git a/src/Common/examples/parallel_aggregation.cpp b/src/Common/examples/parallel_aggregation.cpp index cf7a3197fef..20f5f1c5224 100644 --- a/src/Common/examples/parallel_aggregation.cpp +++ b/src/Common/examples/parallel_aggregation.cpp @@ -33,6 +33,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } struct SmallLock @@ -254,7 +255,7 @@ int main(int argc, char ** argv) std::cerr << std::fixed << std::setprecision(2); - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, num_threads); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_threads); Source data(n); diff --git a/src/Common/examples/parallel_aggregation2.cpp b/src/Common/examples/parallel_aggregation2.cpp index 1b0ad760490..e7136707dbd 100644 --- a/src/Common/examples/parallel_aggregation2.cpp +++ b/src/Common/examples/parallel_aggregation2.cpp @@ -29,6 +29,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } template @@ -281,7 +282,7 @@ int main(int argc, char ** argv) std::cerr << std::fixed << std::setprecision(2); - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, num_threads); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_threads); Source data(n); diff --git a/src/Common/examples/thread_creation_latency.cpp b/src/Common/examples/thread_creation_latency.cpp index 2434759c968..60fb27dc345 100644 --- a/src/Common/examples/thread_creation_latency.cpp +++ b/src/Common/examples/thread_creation_latency.cpp @@ -19,6 +19,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } namespace DB @@ -72,7 +73,7 @@ int main(int argc, char ** argv) test(n, "Create and destroy ThreadPool each iteration", [] { - ThreadPool tp(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 1); + ThreadPool tp(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 1); tp.scheduleOrThrowOnError(f); tp.wait(); }); @@ -93,7 +94,7 @@ int main(int argc, char ** argv) }); { - ThreadPool tp(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 1); + ThreadPool tp(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 1); test(n, "Schedule job for Threadpool each iteration", [&tp] { @@ -103,7 +104,7 @@ int main(int argc, char ** argv) } { - ThreadPool tp(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 128); + ThreadPool tp(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 128); test(n, "Schedule job for Threadpool with 128 threads each iteration", [&tp] { diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp index dfcbf27b9fc..28d47593e78 100644 --- a/src/Common/tests/gtest_async_loader.cpp +++ b/src/Common/tests/gtest_async_loader.cpp @@ -21,6 +21,7 @@ namespace CurrentMetrics { extern const Metric TablesLoaderThreads; extern const Metric TablesLoaderThreadsActive; + extern const Metric TablesLoaderThreadsScheduled; } namespace DB::ErrorCodes @@ -62,6 +63,7 @@ struct AsyncLoaderTest .name = fmt::format("Pool{}", pool_id), .metric_threads = CurrentMetrics::TablesLoaderThreads, .metric_active_threads = CurrentMetrics::TablesLoaderThreadsActive, + .metric_scheduled_threads = CurrentMetrics::TablesLoaderThreadsScheduled, .max_threads = desc.max_threads, .priority = desc.priority }); diff --git a/src/Common/tests/gtest_thread_pool_concurrent_wait.cpp b/src/Common/tests/gtest_thread_pool_concurrent_wait.cpp index f93017129dd..ddaff3382db 100644 --- a/src/Common/tests/gtest_thread_pool_concurrent_wait.cpp +++ b/src/Common/tests/gtest_thread_pool_concurrent_wait.cpp @@ -12,6 +12,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } TEST(ThreadPool, ConcurrentWait) @@ -25,14 +26,14 @@ TEST(ThreadPool, ConcurrentWait) constexpr size_t num_threads = 4; constexpr size_t num_jobs = 4; - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, num_threads); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_threads); for (size_t i = 0; i < num_jobs; ++i) pool.scheduleOrThrowOnError(worker); constexpr size_t num_waiting_threads = 4; - ThreadPool waiting_pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, num_waiting_threads); + ThreadPool waiting_pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_waiting_threads); for (size_t i = 0; i < num_waiting_threads; ++i) waiting_pool.scheduleOrThrowOnError([&pool] { pool.wait(); }); diff --git a/src/Common/tests/gtest_thread_pool_global_full.cpp b/src/Common/tests/gtest_thread_pool_global_full.cpp index 1b2ded9c7e1..4507998be3c 100644 --- a/src/Common/tests/gtest_thread_pool_global_full.cpp +++ b/src/Common/tests/gtest_thread_pool_global_full.cpp @@ -11,6 +11,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } /// Test what happens if local ThreadPool cannot create a ThreadFromGlobalPool. @@ -34,7 +35,7 @@ TEST(ThreadPool, GlobalFull1) auto func = [&] { ++counter; while (counter != num_jobs) {} }; - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, num_jobs); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_jobs); for (size_t i = 0; i < capacity; ++i) pool.scheduleOrThrowOnError(func); @@ -72,11 +73,11 @@ TEST(ThreadPool, GlobalFull2) std::atomic counter = 0; auto func = [&] { ++counter; while (counter != capacity + 1) {} }; - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, capacity, 0, capacity); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, capacity, 0, capacity); for (size_t i = 0; i < capacity; ++i) pool.scheduleOrThrowOnError(func); - ThreadPool another_pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 1); + ThreadPool another_pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 1); EXPECT_THROW(another_pool.scheduleOrThrowOnError(func), DB::Exception); ++counter; diff --git a/src/Common/tests/gtest_thread_pool_limit.cpp b/src/Common/tests/gtest_thread_pool_limit.cpp index b47c8cdad18..66c6f8dc122 100644 --- a/src/Common/tests/gtest_thread_pool_limit.cpp +++ b/src/Common/tests/gtest_thread_pool_limit.cpp @@ -8,6 +8,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } /// Test for thread self-removal when number of free threads in pool is too large. @@ -16,7 +17,7 @@ namespace CurrentMetrics template int test() { - Pool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 10, 2, 10); + Pool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 10, 2, 10); std::atomic counter{0}; for (size_t i = 0; i < 10; ++i) diff --git a/src/Common/tests/gtest_thread_pool_loop.cpp b/src/Common/tests/gtest_thread_pool_loop.cpp index 170a888ff72..4257c0b73a5 100644 --- a/src/Common/tests/gtest_thread_pool_loop.cpp +++ b/src/Common/tests/gtest_thread_pool_loop.cpp @@ -9,6 +9,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } TEST(ThreadPool, Loop) @@ -18,7 +19,7 @@ TEST(ThreadPool, Loop) for (size_t i = 0; i < 1000; ++i) { size_t threads = 16; - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, threads); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, threads); for (size_t j = 0; j < threads; ++j) pool.scheduleOrThrowOnError([&] { ++res; }); pool.wait(); diff --git a/src/Common/tests/gtest_thread_pool_schedule_exception.cpp b/src/Common/tests/gtest_thread_pool_schedule_exception.cpp index d8e00b5314c..1b4f27e59e0 100644 --- a/src/Common/tests/gtest_thread_pool_schedule_exception.cpp +++ b/src/Common/tests/gtest_thread_pool_schedule_exception.cpp @@ -9,11 +9,12 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } static bool check() { - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 10); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 10); /// The throwing thread. pool.scheduleOrThrowOnError([] { throw std::runtime_error("Hello, world!"); }); @@ -53,7 +54,7 @@ TEST(ThreadPool, ExceptionFromSchedule) static bool check2() { - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 2); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 2); try { diff --git a/src/Coordination/Standalone/Context.cpp b/src/Coordination/Standalone/Context.cpp index ef23e85e54b..4001673e01a 100644 --- a/src/Coordination/Standalone/Context.cpp +++ b/src/Coordination/Standalone/Context.cpp @@ -25,6 +25,7 @@ namespace CurrentMetrics extern const Metric BackgroundSchedulePoolSize; extern const Metric IOWriterThreads; extern const Metric IOWriterThreadsActive; + extern const Metric IOWriterThreadsScheduled; } namespace DB diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 96c084a261c..549711150b8 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -30,6 +30,7 @@ namespace CurrentMetrics { extern const Metric DatabaseOnDiskThreads; extern const Metric DatabaseOnDiskThreadsActive; + extern const Metric DatabaseOnDiskThreadsScheduled; } namespace DB @@ -627,7 +628,7 @@ void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const Iterat } /// Read and parse metadata in parallel - ThreadPool pool(CurrentMetrics::DatabaseOnDiskThreads, CurrentMetrics::DatabaseOnDiskThreadsActive); + ThreadPool pool(CurrentMetrics::DatabaseOnDiskThreads, CurrentMetrics::DatabaseOnDiskThreadsActive, CurrentMetrics::DatabaseOnDiskThreadsScheduled); for (const auto & file : metadata_files) { pool.scheduleOrThrowOnError([&]() diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 51d37b84e14..1b2a7b9d5e2 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -34,6 +34,7 @@ namespace CurrentMetrics { extern const Metric DatabaseOrdinaryThreads; extern const Metric DatabaseOrdinaryThreadsActive; + extern const Metric DatabaseOrdinaryThreadsScheduled; } namespace DB @@ -106,7 +107,7 @@ void DatabaseOrdinary::loadStoredObjects(ContextMutablePtr local_context, Loadin std::atomic dictionaries_processed{0}; std::atomic tables_processed{0}; - ThreadPool pool(CurrentMetrics::DatabaseOrdinaryThreads, CurrentMetrics::DatabaseOrdinaryThreadsActive); + ThreadPool pool(CurrentMetrics::DatabaseOrdinaryThreads, CurrentMetrics::DatabaseOrdinaryThreadsActive, CurrentMetrics::DatabaseOrdinaryThreadsScheduled); /// We must attach dictionaries before attaching tables /// because while we're attaching tables we may need to have some dictionaries attached diff --git a/src/Databases/TablesLoader.cpp b/src/Databases/TablesLoader.cpp index f8b4e7fe33b..5cc7e77a6de 100644 --- a/src/Databases/TablesLoader.cpp +++ b/src/Databases/TablesLoader.cpp @@ -15,6 +15,7 @@ namespace CurrentMetrics { extern const Metric TablesLoaderThreads; extern const Metric TablesLoaderThreadsActive; + extern const Metric TablesLoaderThreadsScheduled; } namespace DB @@ -32,7 +33,7 @@ TablesLoader::TablesLoader(ContextMutablePtr global_context_, Databases database , referential_dependencies("ReferentialDeps") , loading_dependencies("LoadingDeps") , all_loading_dependencies("LoadingDeps") - , pool(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive) + , pool(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive, CurrentMetrics::TablesLoaderThreadsScheduled) { metadata.default_database = global_context->getCurrentDatabase(); log = &Poco::Logger::get("TablesLoader"); diff --git a/src/Dictionaries/CacheDictionaryUpdateQueue.cpp b/src/Dictionaries/CacheDictionaryUpdateQueue.cpp index 3aa3cbb9a4b..1e9b1da390a 100644 --- a/src/Dictionaries/CacheDictionaryUpdateQueue.cpp +++ b/src/Dictionaries/CacheDictionaryUpdateQueue.cpp @@ -9,6 +9,7 @@ namespace CurrentMetrics { extern const Metric CacheDictionaryThreads; extern const Metric CacheDictionaryThreadsActive; + extern const Metric CacheDictionaryThreadsScheduled; } namespace DB @@ -33,7 +34,7 @@ CacheDictionaryUpdateQueue::CacheDictionaryUpdateQueue( , configuration(configuration_) , update_func(std::move(update_func_)) , update_queue(configuration.max_update_queue_size) - , update_pool(CurrentMetrics::CacheDictionaryThreads, CurrentMetrics::CacheDictionaryThreadsActive, configuration.max_threads_for_updates) + , update_pool(CurrentMetrics::CacheDictionaryThreads, CurrentMetrics::CacheDictionaryThreadsActive, CurrentMetrics::CacheDictionaryThreadsScheduled, configuration.max_threads_for_updates) { for (size_t i = 0; i < configuration.max_threads_for_updates; ++i) update_pool.scheduleOrThrowOnError([this] { updateThreadFunction(); }); diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp index 0556e2bb266..9c5dfeef6ca 100644 --- a/src/Dictionaries/HashedDictionary.cpp +++ b/src/Dictionaries/HashedDictionary.cpp @@ -32,6 +32,7 @@ namespace CurrentMetrics { extern const Metric HashedDictionaryThreads; extern const Metric HashedDictionaryThreadsActive; + extern const Metric HashedDictionaryThreadsScheduled; } namespace DB @@ -59,7 +60,7 @@ public: explicit ParallelDictionaryLoader(HashedDictionary & dictionary_) : dictionary(dictionary_) , shards(dictionary.configuration.shards) - , pool(CurrentMetrics::HashedDictionaryThreads, CurrentMetrics::HashedDictionaryThreadsActive, shards) + , pool(CurrentMetrics::HashedDictionaryThreads, CurrentMetrics::HashedDictionaryThreadsActive, CurrentMetrics::HashedDictionaryThreadsScheduled, shards) , shards_queues(shards) { UInt64 backlog = dictionary.configuration.shard_load_queue_backlog; @@ -229,7 +230,7 @@ HashedDictionary::~HashedDictionary() return; size_t shards = std::max(configuration.shards, 1); - ThreadPool pool(CurrentMetrics::HashedDictionaryThreads, CurrentMetrics::HashedDictionaryThreadsActive, shards); + ThreadPool pool(CurrentMetrics::HashedDictionaryThreads, CurrentMetrics::HashedDictionaryThreadsActive, CurrentMetrics::HashedDictionaryThreadsScheduled, shards); size_t hash_tables_count = 0; auto schedule_destroy = [&hash_tables_count, &pool](auto & container) diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 6911fd86db2..5fcf9a9dce4 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -38,6 +38,7 @@ namespace CurrentMetrics { extern const Metric IDiskCopierThreads; extern const Metric IDiskCopierThreadsActive; + extern const Metric IDiskCopierThreadsScheduled; } namespace DB @@ -117,13 +118,13 @@ public: /// Default constructor. IDisk(const String & name_, const Poco::Util::AbstractConfiguration & config, const String & config_prefix) : name(name_) - , copying_thread_pool(CurrentMetrics::IDiskCopierThreads, CurrentMetrics::IDiskCopierThreadsActive, config.getUInt(config_prefix + ".thread_pool_size", 16)) + , copying_thread_pool(CurrentMetrics::IDiskCopierThreads, CurrentMetrics::IDiskCopierThreadsActive, CurrentMetrics::IDiskCopierThreadsScheduled, config.getUInt(config_prefix + ".thread_pool_size", 16)) { } explicit IDisk(const String & name_) : name(name_) - , copying_thread_pool(CurrentMetrics::IDiskCopierThreads, CurrentMetrics::IDiskCopierThreadsActive, 16) + , copying_thread_pool(CurrentMetrics::IDiskCopierThreads, CurrentMetrics::IDiskCopierThreadsActive, CurrentMetrics::IDiskCopierThreadsScheduled, 16) { } diff --git a/src/Disks/IO/ThreadPoolReader.cpp b/src/Disks/IO/ThreadPoolReader.cpp index cd3f2d8dea0..15cbcdf5b9b 100644 --- a/src/Disks/IO/ThreadPoolReader.cpp +++ b/src/Disks/IO/ThreadPoolReader.cpp @@ -64,6 +64,7 @@ namespace CurrentMetrics extern const Metric Read; extern const Metric ThreadPoolFSReaderThreads; extern const Metric ThreadPoolFSReaderThreadsActive; + extern const Metric ThreadPoolFSReaderThreadsScheduled; } @@ -88,7 +89,7 @@ static bool hasBugInPreadV2() #endif ThreadPoolReader::ThreadPoolReader(size_t pool_size, size_t queue_size_) - : pool(std::make_unique(CurrentMetrics::ThreadPoolFSReaderThreads, CurrentMetrics::ThreadPoolFSReaderThreadsActive, pool_size, pool_size, queue_size_)) + : pool(std::make_unique(CurrentMetrics::ThreadPoolFSReaderThreads, CurrentMetrics::ThreadPoolFSReaderThreadsActive, CurrentMetrics::ThreadPoolFSReaderThreadsScheduled, pool_size, pool_size, queue_size_)) { } diff --git a/src/Disks/IO/ThreadPoolRemoteFSReader.cpp b/src/Disks/IO/ThreadPoolRemoteFSReader.cpp index 0ec5e0fd6c1..ac599bb3547 100644 --- a/src/Disks/IO/ThreadPoolRemoteFSReader.cpp +++ b/src/Disks/IO/ThreadPoolRemoteFSReader.cpp @@ -29,6 +29,7 @@ namespace CurrentMetrics extern const Metric RemoteRead; extern const Metric ThreadPoolRemoteFSReaderThreads; extern const Metric ThreadPoolRemoteFSReaderThreadsActive; + extern const Metric ThreadPoolRemoteFSReaderThreadsScheduled; } namespace DB @@ -59,6 +60,7 @@ namespace ThreadPoolRemoteFSReader::ThreadPoolRemoteFSReader(size_t pool_size, size_t queue_size_) : pool(std::make_unique(CurrentMetrics::ThreadPoolRemoteFSReaderThreads, CurrentMetrics::ThreadPoolRemoteFSReaderThreadsActive, + CurrentMetrics::ThreadPoolRemoteFSReaderThreadsScheduled, pool_size, pool_size, queue_size_)) { } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index fcb82daca95..068e2aebab1 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -14,11 +14,12 @@ #include #include + namespace CurrentMetrics { extern const Metric ObjectStorageAzureThreads; extern const Metric ObjectStorageAzureThreadsActive; - + extern const Metric ObjectStorageAzureThreadsScheduled; } namespace DB @@ -45,6 +46,7 @@ public: : IObjectStorageIteratorAsync( CurrentMetrics::ObjectStorageAzureThreads, CurrentMetrics::ObjectStorageAzureThreadsActive, + CurrentMetrics::ObjectStorageAzureThreadsScheduled, "ListObjectAzure") , client(client_) { diff --git a/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp b/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp index 33b98cd328c..0314e0a7e92 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp @@ -13,6 +13,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } namespace DB @@ -156,7 +157,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::migrateToRestorableSchema() { LOG_INFO(disk->log, "Start migration to restorable schema for disk {}", disk->name); - ThreadPool pool{CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive}; + ThreadPool pool{CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled}; for (const auto & root : data_roots) if (disk->exists(root)) @@ -355,7 +356,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFiles(IObjectStorage * { LOG_INFO(disk->log, "Starting restore files for disk {}", disk->name); - ThreadPool pool{CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive}; + ThreadPool pool{CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled}; auto restore_files = [this, &source_object_storage, &restore_information, &pool](const RelativePathsWithMetadata & objects) { std::vector keys_names; diff --git a/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.cpp b/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.cpp index 7425f629a5a..990e66fc4e5 100644 --- a/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.cpp +++ b/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.cpp @@ -2,6 +2,7 @@ #include + namespace DB { diff --git a/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.h b/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.h index b0dd3cef39c..a6abe03bac9 100644 --- a/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.h +++ b/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.h @@ -6,6 +6,7 @@ #include #include + namespace DB { @@ -15,8 +16,9 @@ public: IObjectStorageIteratorAsync( CurrentMetrics::Metric threads_metric, CurrentMetrics::Metric threads_active_metric, + CurrentMetrics::Metric threads_scheduled_metric, const std::string & thread_name) - : list_objects_pool(threads_metric, threads_active_metric, 1) + : list_objects_pool(threads_metric, threads_active_metric, threads_scheduled_metric, 1) , list_objects_scheduler(threadPoolCallbackRunner(list_objects_pool, thread_name)) { } diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index b36185249af..3af316bf0cf 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -37,6 +37,7 @@ namespace CurrentMetrics { extern const Metric ObjectStorageS3Threads; extern const Metric ObjectStorageS3ThreadsActive; + extern const Metric ObjectStorageS3ThreadsScheduled; } @@ -105,6 +106,7 @@ public: : IObjectStorageIteratorAsync( CurrentMetrics::ObjectStorageS3Threads, CurrentMetrics::ObjectStorageS3ThreadsActive, + CurrentMetrics::ObjectStorageS3ThreadsScheduled, "ListObjectS3") , client(client_) { diff --git a/src/IO/SharedThreadPools.cpp b/src/IO/SharedThreadPools.cpp index 6a0e953f0ef..4a0ef173669 100644 --- a/src/IO/SharedThreadPools.cpp +++ b/src/IO/SharedThreadPools.cpp @@ -7,14 +7,19 @@ namespace CurrentMetrics { extern const Metric IOThreads; extern const Metric IOThreadsActive; + extern const Metric IOThreadsScheduled; extern const Metric BackupsIOThreads; extern const Metric BackupsIOThreadsActive; + extern const Metric BackupsIOThreadsScheduled; extern const Metric MergeTreePartsLoaderThreads; extern const Metric MergeTreePartsLoaderThreadsActive; + extern const Metric MergeTreePartsLoaderThreadsScheduled; extern const Metric MergeTreePartsCleanerThreads; extern const Metric MergeTreePartsCleanerThreadsActive; + extern const Metric MergeTreePartsCleanerThreadsScheduled; extern const Metric MergeTreeOutdatedPartsLoaderThreads; extern const Metric MergeTreeOutdatedPartsLoaderThreadsActive; + extern const Metric MergeTreeOutdatedPartsLoaderThreadsScheduled; } namespace DB @@ -29,10 +34,12 @@ namespace ErrorCodes StaticThreadPool::StaticThreadPool( const String & name_, CurrentMetrics::Metric threads_metric_, - CurrentMetrics::Metric threads_active_metric_) + CurrentMetrics::Metric threads_active_metric_, + CurrentMetrics::Metric threads_scheduled_metric_) : name(name_) , threads_metric(threads_metric_) , threads_active_metric(threads_active_metric_) + , threads_scheduled_metric(threads_scheduled_metric_) { } @@ -47,6 +54,7 @@ void StaticThreadPool::initialize(size_t max_threads, size_t max_free_threads, s instance = std::make_unique( threads_metric, threads_active_metric, + threads_scheduled_metric, max_threads, max_free_threads, queue_size, @@ -110,31 +118,31 @@ void StaticThreadPool::setMaxTurboThreads(size_t max_threads_turbo_) StaticThreadPool & getIOThreadPool() { - static StaticThreadPool instance("IOThreadPool", CurrentMetrics::IOThreads, CurrentMetrics::IOThreadsActive); + static StaticThreadPool instance("IOThreadPool", CurrentMetrics::IOThreads, CurrentMetrics::IOThreadsActive, CurrentMetrics::IOThreadsScheduled); return instance; } StaticThreadPool & getBackupsIOThreadPool() { - static StaticThreadPool instance("BackupsIOThreadPool", CurrentMetrics::BackupsIOThreads, CurrentMetrics::BackupsIOThreadsActive); + static StaticThreadPool instance("BackupsIOThreadPool", CurrentMetrics::BackupsIOThreads, CurrentMetrics::BackupsIOThreadsActive, CurrentMetrics::BackupsIOThreadsScheduled); return instance; } StaticThreadPool & getActivePartsLoadingThreadPool() { - static StaticThreadPool instance("MergeTreePartsLoaderThreadPool", CurrentMetrics::MergeTreePartsLoaderThreads, CurrentMetrics::MergeTreePartsLoaderThreadsActive); + static StaticThreadPool instance("MergeTreePartsLoaderThreadPool", CurrentMetrics::MergeTreePartsLoaderThreads, CurrentMetrics::MergeTreePartsLoaderThreadsActive, CurrentMetrics::MergeTreeOutdatedPartsLoaderThreadsScheduled); return instance; } StaticThreadPool & getPartsCleaningThreadPool() { - static StaticThreadPool instance("MergeTreePartsCleanerThreadPool", CurrentMetrics::MergeTreePartsCleanerThreads, CurrentMetrics::MergeTreePartsCleanerThreadsActive); + static StaticThreadPool instance("MergeTreePartsCleanerThreadPool", CurrentMetrics::MergeTreePartsCleanerThreads, CurrentMetrics::MergeTreePartsCleanerThreadsActive, CurrentMetrics::MergeTreePartsCleanerThreadsScheduled); return instance; } StaticThreadPool & getOutdatedPartsLoadingThreadPool() { - static StaticThreadPool instance("MergeTreeOutdatedPartsLoaderThreadPool", CurrentMetrics::MergeTreeOutdatedPartsLoaderThreads, CurrentMetrics::MergeTreeOutdatedPartsLoaderThreadsActive); + static StaticThreadPool instance("MergeTreeOutdatedPartsLoaderThreadPool", CurrentMetrics::MergeTreeOutdatedPartsLoaderThreads, CurrentMetrics::MergeTreeOutdatedPartsLoaderThreadsActive, CurrentMetrics::MergeTreeOutdatedPartsLoaderThreadsScheduled); return instance; } diff --git a/src/IO/SharedThreadPools.h b/src/IO/SharedThreadPools.h index 188a2a4f003..f37f3acefe7 100644 --- a/src/IO/SharedThreadPools.h +++ b/src/IO/SharedThreadPools.h @@ -8,6 +8,7 @@ #include #include + namespace DB { @@ -17,7 +18,8 @@ public: StaticThreadPool( const String & name_, CurrentMetrics::Metric threads_metric_, - CurrentMetrics::Metric threads_active_metric_); + CurrentMetrics::Metric threads_active_metric_, + CurrentMetrics::Metric threads_scheduled_metric_); ThreadPool & get(); @@ -34,6 +36,7 @@ private: const String name; const CurrentMetrics::Metric threads_metric; const CurrentMetrics::Metric threads_active_metric; + const CurrentMetrics::Metric threads_scheduled_metric; std::unique_ptr instance; std::mutex mutex; diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 129c02f032b..e2ddfbe3418 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -63,6 +63,7 @@ namespace CurrentMetrics extern const Metric TemporaryFilesForAggregation; extern const Metric AggregatorThreads; extern const Metric AggregatorThreadsActive; + extern const Metric AggregatorThreadsScheduled; } namespace DB @@ -2466,7 +2467,7 @@ BlocksList Aggregator::convertToBlocks(AggregatedDataVariants & data_variants, b std::unique_ptr thread_pool; if (max_threads > 1 && data_variants.sizeWithoutOverflowRow() > 100000 /// TODO Make a custom threshold. && data_variants.isTwoLevel()) /// TODO Use the shared thread pool with the `merge` function. - thread_pool = std::make_unique(CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, max_threads); + thread_pool = std::make_unique(CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, CurrentMetrics::AggregatorThreadsScheduled, max_threads); if (data_variants.without_key) blocks.emplace_back(prepareBlockAndFillWithoutKey( @@ -2656,7 +2657,7 @@ void NO_INLINE Aggregator::mergeDataOnlyExistingKeysImpl( void NO_INLINE Aggregator::mergeWithoutKeyDataImpl( ManyAggregatedDataVariants & non_empty_data) const { - ThreadPool thread_pool{CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, params.max_threads}; + ThreadPool thread_pool{CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, CurrentMetrics::AggregatorThreadsScheduled, params.max_threads}; AggregatedDataVariantsPtr & res = non_empty_data[0]; @@ -3144,7 +3145,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari std::unique_ptr thread_pool; if (max_threads > 1 && total_input_rows > 100000) /// TODO Make a custom threshold. - thread_pool = std::make_unique(CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, max_threads); + thread_pool = std::make_unique(CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, CurrentMetrics::AggregatorThreadsScheduled, max_threads); for (const auto & bucket_blocks : bucket_to_blocks) { diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 04285a06a65..a0750122a5c 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -39,6 +39,7 @@ namespace CurrentMetrics extern const Metric PendingAsyncInsert; extern const Metric AsynchronousInsertThreads; extern const Metric AsynchronousInsertThreadsActive; + extern const Metric AsynchronousInsertThreadsScheduled; } namespace ProfileEvents @@ -175,7 +176,7 @@ AsynchronousInsertQueue::AsynchronousInsertQueue(ContextPtr context_, size_t poo , pool_size(pool_size_) , flush_on_shutdown(flush_on_shutdown_) , queue_shards(pool_size) - , pool(CurrentMetrics::AsynchronousInsertThreads, CurrentMetrics::AsynchronousInsertThreadsActive, pool_size) + , pool(CurrentMetrics::AsynchronousInsertThreads, CurrentMetrics::AsynchronousInsertThreadsActive, CurrentMetrics::AsynchronousInsertThreadsScheduled, pool_size) { if (!pool_size) throw Exception(ErrorCodes::BAD_ARGUMENTS, "pool_size cannot be zero"); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index c6640d582a2..c13be020933 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -135,10 +135,13 @@ namespace CurrentMetrics extern const Metric BackgroundCommonPoolSize; extern const Metric MarksLoaderThreads; extern const Metric MarksLoaderThreadsActive; + extern const Metric MarksLoaderThreadsScheduled; extern const Metric IOPrefetchThreads; extern const Metric IOPrefetchThreadsActive; + extern const Metric IOPrefetchThreadsScheduled; extern const Metric IOWriterThreads; extern const Metric IOWriterThreadsActive; + extern const Metric IOWriterThreadsScheduled; } @@ -2542,7 +2545,7 @@ ThreadPool & Context::getLoadMarksThreadpool() const auto pool_size = config.getUInt(".load_marks_threadpool_pool_size", 50); auto queue_size = config.getUInt(".load_marks_threadpool_queue_size", 1000000); shared->load_marks_threadpool = std::make_unique( - CurrentMetrics::MarksLoaderThreads, CurrentMetrics::MarksLoaderThreadsActive, pool_size, pool_size, queue_size); + CurrentMetrics::MarksLoaderThreads, CurrentMetrics::MarksLoaderThreadsActive, CurrentMetrics::MarksLoaderThreadsScheduled, pool_size, pool_size, queue_size); }); return *shared->load_marks_threadpool; @@ -2725,7 +2728,7 @@ ThreadPool & Context::getPrefetchThreadpool() const auto pool_size = config.getUInt(".prefetch_threadpool_pool_size", 100); auto queue_size = config.getUInt(".prefetch_threadpool_queue_size", 1000000); shared->prefetch_threadpool = std::make_unique( - CurrentMetrics::IOPrefetchThreads, CurrentMetrics::IOPrefetchThreadsActive, pool_size, pool_size, queue_size); + CurrentMetrics::IOPrefetchThreads, CurrentMetrics::IOPrefetchThreadsActive, CurrentMetrics::IOPrefetchThreadsScheduled, pool_size, pool_size, queue_size); }); return *shared->prefetch_threadpool; @@ -4765,7 +4768,7 @@ ThreadPool & Context::getThreadPoolWriter() const auto queue_size = config.getUInt(".threadpool_writer_queue_size", 1000000); shared->threadpool_writer = std::make_unique( - CurrentMetrics::IOWriterThreads, CurrentMetrics::IOWriterThreadsActive, pool_size, pool_size, queue_size); + CurrentMetrics::IOWriterThreads, CurrentMetrics::IOWriterThreadsActive, CurrentMetrics::IOWriterThreadsScheduled, pool_size, pool_size, queue_size); }); return *shared->threadpool_writer; diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index a9930036e7e..a43c61f3d66 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -46,6 +46,7 @@ namespace CurrentMetrics { extern const Metric DDLWorkerThreads; extern const Metric DDLWorkerThreadsActive; + extern const Metric DDLWorkerThreadsScheduled; } namespace DB @@ -93,7 +94,7 @@ DDLWorker::DDLWorker( { LOG_WARNING(log, "DDLWorker is configured to use multiple threads. " "It's not recommended because queries can be reordered. Also it may cause some unknown issues to appear."); - worker_pool = std::make_unique(CurrentMetrics::DDLWorkerThreads, CurrentMetrics::DDLWorkerThreadsActive, pool_size); + worker_pool = std::make_unique(CurrentMetrics::DDLWorkerThreads, CurrentMetrics::DDLWorkerThreadsActive, CurrentMetrics::DDLWorkerThreadsScheduled, pool_size); } queue_dir = zk_root_dir; @@ -1113,7 +1114,7 @@ void DDLWorker::runMainThread() /// It will wait for all threads in pool to finish and will not rethrow exceptions (if any). /// We create new thread pool to forget previous exceptions. if (1 < pool_size) - worker_pool = std::make_unique(CurrentMetrics::DDLWorkerThreads, CurrentMetrics::DDLWorkerThreadsActive, pool_size); + worker_pool = std::make_unique(CurrentMetrics::DDLWorkerThreads, CurrentMetrics::DDLWorkerThreadsActive, CurrentMetrics::DDLWorkerThreadsScheduled, pool_size); /// Clear other in-memory state, like server just started. current_tasks.clear(); last_skipped_entry_name.reset(); @@ -1152,7 +1153,7 @@ void DDLWorker::runMainThread() initialized = false; /// Wait for pending async tasks if (1 < pool_size) - worker_pool = std::make_unique(CurrentMetrics::DDLWorkerThreads, CurrentMetrics::DDLWorkerThreadsActive, pool_size); + worker_pool = std::make_unique(CurrentMetrics::DDLWorkerThreads, CurrentMetrics::DDLWorkerThreadsActive, CurrentMetrics::DDLWorkerThreadsScheduled, pool_size); LOG_INFO(log, "Lost ZooKeeper connection, will try to connect again: {}", getCurrentExceptionMessage(true)); } else diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 8dc90e1a2f1..e0ce2b99b14 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -44,6 +44,7 @@ namespace CurrentMetrics extern const Metric TablesToDropQueueSize; extern const Metric DatabaseCatalogThreads; extern const Metric DatabaseCatalogThreadsActive; + extern const Metric DatabaseCatalogThreadsScheduled; } namespace DB @@ -1024,7 +1025,7 @@ void DatabaseCatalog::loadMarkedAsDroppedTables() LOG_INFO(log, "Found {} partially dropped tables. Will load them and retry removal.", dropped_metadata.size()); - ThreadPool pool(CurrentMetrics::DatabaseCatalogThreads, CurrentMetrics::DatabaseCatalogThreadsActive); + ThreadPool pool(CurrentMetrics::DatabaseCatalogThreads, CurrentMetrics::DatabaseCatalogThreadsActive, CurrentMetrics::DatabaseCatalogThreadsScheduled); for (const auto & elem : dropped_metadata) { pool.scheduleOrThrowOnError([&]() diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index b42745b726c..ff95b3014dc 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -76,6 +76,7 @@ namespace CurrentMetrics { extern const Metric RestartReplicaThreads; extern const Metric RestartReplicaThreadsActive; + extern const Metric RestartReplicaThreadsScheduled; } namespace DB @@ -809,7 +810,7 @@ void InterpreterSystemQuery::restartReplicas(ContextMutablePtr system_context) size_t threads = std::min(static_cast(getNumberOfPhysicalCPUCores()), replica_names.size()); LOG_DEBUG(log, "Will restart {} replicas using {} threads", replica_names.size(), threads); - ThreadPool pool(CurrentMetrics::RestartReplicaThreads, CurrentMetrics::RestartReplicaThreadsActive, threads); + ThreadPool pool(CurrentMetrics::RestartReplicaThreads, CurrentMetrics::RestartReplicaThreadsActive, CurrentMetrics::RestartReplicaThreadsScheduled, threads); for (auto & replica : replica_names) { diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index 3612dbfdc4e..44b57396bd5 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -32,6 +32,7 @@ namespace CurrentMetrics { extern const Metric StartupSystemTablesThreads; extern const Metric StartupSystemTablesThreadsActive; + extern const Metric StartupSystemTablesThreadsScheduled; } namespace DB @@ -377,7 +378,7 @@ static void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, cons if (!tables_started) { /// It's not quite correct to run DDL queries while database is not started up. - ThreadPool pool(CurrentMetrics::StartupSystemTablesThreads, CurrentMetrics::StartupSystemTablesThreadsActive); + ThreadPool pool(CurrentMetrics::StartupSystemTablesThreads, CurrentMetrics::StartupSystemTablesThreadsActive, CurrentMetrics::StartupSystemTablesThreadsScheduled); DatabaseCatalog::instance().getSystemDatabase()->startupTables(pool, LoadingStrictnessLevel::FORCE_RESTORE); } @@ -472,7 +473,7 @@ void convertDatabasesEnginesIfNeed(ContextMutablePtr context) void startupSystemTables() { - ThreadPool pool(CurrentMetrics::StartupSystemTablesThreads, CurrentMetrics::StartupSystemTablesThreadsActive); + ThreadPool pool(CurrentMetrics::StartupSystemTablesThreads, CurrentMetrics::StartupSystemTablesThreadsActive, CurrentMetrics::StartupSystemTablesThreadsScheduled); DatabaseCatalog::instance().getSystemDatabase()->startupTables(pool, LoadingStrictnessLevel::FORCE_RESTORE); } diff --git a/src/Interpreters/threadPoolCallbackRunner.h b/src/Interpreters/threadPoolCallbackRunner.h index eb90b61cf31..2b943110273 100644 --- a/src/Interpreters/threadPoolCallbackRunner.h +++ b/src/Interpreters/threadPoolCallbackRunner.h @@ -24,9 +24,10 @@ ThreadPoolCallbackRunner threadPoolCallbackRunner(ThreadPool & if (thread_group) CurrentThread::attachToGroup(thread_group); - SCOPE_EXIT_SAFE({ + SCOPE_EXIT_SAFE( + { { - /// Release all captutred resources before detaching thread group + /// Release all captured resources before detaching thread group /// Releasing has to use proper memory tracker which has been set here before callback [[maybe_unused]] auto tmp = std::move(my_callback); @@ -34,7 +35,6 @@ ThreadPoolCallbackRunner threadPoolCallbackRunner(ThreadPool & if (thread_group) CurrentThread::detachFromGroupIfNotDetached(); - }); setThreadName(thread_name.data()); diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 37af391fba3..812b64ccdb8 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -22,6 +22,7 @@ namespace CurrentMetrics { extern const Metric QueryPipelineExecutorThreads; extern const Metric QueryPipelineExecutorThreadsActive; + extern const Metric QueryPipelineExecutorThreadsScheduled; } namespace DB @@ -332,7 +333,7 @@ void PipelineExecutor::initializeExecution(size_t num_threads, bool concurrency_ tasks.fill(queue); if (num_threads > 1) - pool = std::make_unique(CurrentMetrics::QueryPipelineExecutorThreads, CurrentMetrics::QueryPipelineExecutorThreadsActive, num_threads); + pool = std::make_unique(CurrentMetrics::QueryPipelineExecutorThreads, CurrentMetrics::QueryPipelineExecutorThreadsActive, CurrentMetrics::QueryPipelineExecutorThreadsScheduled, num_threads); } void PipelineExecutor::spawnThreads() diff --git a/src/Processors/Formats/Impl/DWARFBlockInputFormat.cpp b/src/Processors/Formats/Impl/DWARFBlockInputFormat.cpp index c5f8059f93c..19ed5c94dfd 100644 --- a/src/Processors/Formats/Impl/DWARFBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/DWARFBlockInputFormat.cpp @@ -28,6 +28,7 @@ namespace CurrentMetrics { extern const Metric DWARFReaderThreads; extern const Metric DWARFReaderThreadsActive; + extern const Metric DWARFReaderThreadsScheduled; } namespace DB @@ -238,7 +239,7 @@ void DWARFBlockInputFormat::initializeIfNeeded() LOG_DEBUG(&Poco::Logger::get("DWARF"), "{} units, reading in {} threads", units_queue.size(), num_threads); - pool.emplace(CurrentMetrics::DWARFReaderThreads, CurrentMetrics::DWARFReaderThreadsActive, num_threads); + pool.emplace(CurrentMetrics::DWARFReaderThreads, CurrentMetrics::DWARFReaderThreadsActive, CurrentMetrics::DWARFReaderThreadsScheduled, num_threads); for (size_t i = 0; i < num_threads; ++i) pool->scheduleOrThrowOnError( [this, thread_group = CurrentThread::getGroup()]() diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index bf8968dd376..c2f08479730 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -21,6 +21,7 @@ namespace CurrentMetrics { extern const Metric ParallelFormattingOutputFormatThreads; extern const Metric ParallelFormattingOutputFormatThreadsActive; + extern const Metric ParallelFormattingOutputFormatThreadsScheduled; } namespace DB @@ -80,7 +81,7 @@ public: explicit ParallelFormattingOutputFormat(Params params) : IOutputFormat(params.header, params.out) , internal_formatter_creator(params.internal_formatter_creator) - , pool(CurrentMetrics::ParallelFormattingOutputFormatThreads, CurrentMetrics::ParallelFormattingOutputFormatThreadsActive, params.max_threads_for_parallel_formatting) + , pool(CurrentMetrics::ParallelFormattingOutputFormatThreads, CurrentMetrics::ParallelFormattingOutputFormatThreadsActive, CurrentMetrics::ParallelFormattingOutputFormatThreadsScheduled, params.max_threads_for_parallel_formatting) { LOG_TEST(&Poco::Logger::get("ParallelFormattingOutputFormat"), "Parallel formatting is being used"); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index f61dc3fbc78..be9e50b854b 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -18,6 +18,7 @@ namespace CurrentMetrics { extern const Metric ParallelParsingInputFormatThreads; extern const Metric ParallelParsingInputFormatThreadsActive; + extern const Metric ParallelParsingInputFormatThreadsScheduled; } namespace DB @@ -101,7 +102,7 @@ public: , min_chunk_bytes(params.min_chunk_bytes) , max_block_size(params.max_block_size) , is_server(params.is_server) - , pool(CurrentMetrics::ParallelParsingInputFormatThreads, CurrentMetrics::ParallelParsingInputFormatThreadsActive, params.max_threads) + , pool(CurrentMetrics::ParallelParsingInputFormatThreads, CurrentMetrics::ParallelParsingInputFormatThreadsActive, CurrentMetrics::ParallelParsingInputFormatThreadsScheduled, params.max_threads) { // One unit for each thread, including segmentator and reader, plus a // couple more units so that the segmentation thread doesn't spuriously diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index c9ac2438fc0..d37c2dc1160 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -28,6 +28,7 @@ namespace CurrentMetrics { extern const Metric ParquetDecoderThreads; extern const Metric ParquetDecoderThreadsActive; + extern const Metric ParquetDecoderThreadsScheduled; } namespace DB @@ -377,7 +378,7 @@ ParquetBlockInputFormat::ParquetBlockInputFormat( , pending_chunks(PendingChunk::Compare { .row_group_first = format_settings_.parquet.preserve_order }) { if (max_decoding_threads > 1) - pool = std::make_unique(CurrentMetrics::ParquetDecoderThreads, CurrentMetrics::ParquetDecoderThreadsActive, max_decoding_threads); + pool = std::make_unique(CurrentMetrics::ParquetDecoderThreads, CurrentMetrics::ParquetDecoderThreadsActive, CurrentMetrics::ParquetDecoderThreadsScheduled, max_decoding_threads); } ParquetBlockInputFormat::~ParquetBlockInputFormat() diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index fbf8b3a7c87..fb9f853dc01 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -13,6 +13,7 @@ namespace CurrentMetrics { extern const Metric ParquetEncoderThreads; extern const Metric ParquetEncoderThreadsActive; + extern const Metric ParquetEncoderThreadsScheduled; } namespace DB @@ -79,7 +80,9 @@ ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, const Blo { if (format_settings.parquet.parallel_encoding && format_settings.max_threads > 1) pool = std::make_unique( - CurrentMetrics::ParquetEncoderThreads, CurrentMetrics::ParquetEncoderThreadsActive, + CurrentMetrics::ParquetEncoderThreads, + CurrentMetrics::ParquetEncoderThreadsActive, + CurrentMetrics::ParquetEncoderThreadsScheduled, format_settings.max_threads); using C = FormatSettings::ParquetCompression; diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 3420cdeaa50..61a6acd6bc8 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -13,6 +13,7 @@ namespace CurrentMetrics { extern const Metric DestroyAggregatesThreads; extern const Metric DestroyAggregatesThreadsActive; + extern const Metric DestroyAggregatesThreadsScheduled; } namespace DB @@ -95,6 +96,7 @@ struct ManyAggregatedData const auto pool = std::make_unique( CurrentMetrics::DestroyAggregatesThreads, CurrentMetrics::DestroyAggregatesThreadsActive, + CurrentMetrics::DestroyAggregatesThreadsScheduled, variants.size()); for (auto && variant : variants) diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index 65a4aa2741a..8c58e304203 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -31,8 +31,6 @@ #include #include #include -#include -#include #include #include @@ -43,6 +41,7 @@ namespace CurrentMetrics extern const Metric DistributedSend; extern const Metric DistributedInsertThreads; extern const Metric DistributedInsertThreadsActive; + extern const Metric DistributedInsertThreadsScheduled; } namespace ProfileEvents @@ -465,6 +464,7 @@ void DistributedSink::writeSync(const Block & block) pool.emplace( CurrentMetrics::DistributedInsertThreads, CurrentMetrics::DistributedInsertThreadsActive, + CurrentMetrics::DistributedInsertThreadsScheduled, max_threads, max_threads, jobs_count); if (!throttler && (settings.max_network_bandwidth || settings.max_network_bytes)) diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index 1587354452e..f03136e4edf 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -46,6 +46,7 @@ namespace CurrentMetrics { extern const Metric StorageHiveThreads; extern const Metric StorageHiveThreadsActive; + extern const Metric StorageHiveThreadsScheduled; } namespace DB @@ -861,7 +862,7 @@ HiveFiles StorageHive::collectHiveFiles( Int64 hive_max_query_partitions = context_->getSettings().max_partitions_to_read; /// Mutext to protect hive_files, which maybe appended in multiple threads std::mutex hive_files_mutex; - ThreadPool pool{CurrentMetrics::StorageHiveThreads, CurrentMetrics::StorageHiveThreadsActive, max_threads}; + ThreadPool pool{CurrentMetrics::StorageHiveThreads, CurrentMetrics::StorageHiveThreadsActive, CurrentMetrics::StorageHiveThreadsScheduled, max_threads}; if (!partitions.empty()) { for (const auto & partition : partitions) diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp index 1a7a0b5b2c1..a5f503718b6 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp @@ -14,6 +14,7 @@ namespace CurrentMetrics { extern const Metric MergeTreeBackgroundExecutorThreads; extern const Metric MergeTreeBackgroundExecutorThreadsActive; + extern const Metric MergeTreeBackgroundExecutorThreadsScheduled; } namespace DB @@ -40,7 +41,7 @@ MergeTreeBackgroundExecutor::MergeTreeBackgroundExecutor( , metric(metric_) , max_tasks_metric(max_tasks_metric_, 2 * max_tasks_count) // active + pending , pool(std::make_unique( - CurrentMetrics::MergeTreeBackgroundExecutorThreads, CurrentMetrics::MergeTreeBackgroundExecutorThreadsActive)) + CurrentMetrics::MergeTreeBackgroundExecutorThreads, CurrentMetrics::MergeTreeBackgroundExecutorThreadsActive, CurrentMetrics::MergeTreeBackgroundExecutorThreadsScheduled)) { if (max_tasks_count == 0) throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Task count for MergeTreeBackgroundExecutor must not be zero"); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index e521491c2d5..d1a285b8818 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -53,6 +53,7 @@ namespace CurrentMetrics { extern const Metric MergeTreeDataSelectExecutorThreads; extern const Metric MergeTreeDataSelectExecutorThreadsActive; + extern const Metric MergeTreeDataSelectExecutorThreadsScheduled; } namespace DB @@ -1075,6 +1076,7 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd ThreadPool pool( CurrentMetrics::MergeTreeDataSelectExecutorThreads, CurrentMetrics::MergeTreeDataSelectExecutorThreadsActive, + CurrentMetrics::MergeTreeDataSelectExecutorThreadsScheduled, num_threads); for (size_t part_index = 0; part_index < parts.size(); ++part_index) diff --git a/src/Storages/StorageAzureBlob.cpp b/src/Storages/StorageAzureBlob.cpp index 2e0703a8df3..b43f25b0fff 100644 --- a/src/Storages/StorageAzureBlob.cpp +++ b/src/Storages/StorageAzureBlob.cpp @@ -53,6 +53,7 @@ namespace CurrentMetrics { extern const Metric ObjectStorageAzureThreads; extern const Metric ObjectStorageAzureThreadsActive; + extern const Metric ObjectStorageAzureThreadsScheduled; } namespace ProfileEvents @@ -1087,7 +1088,7 @@ StorageAzureBlobSource::StorageAzureBlobSource( , file_iterator(file_iterator_) , need_only_count(need_only_count_) , query_info(query_info_) - , create_reader_pool(CurrentMetrics::ObjectStorageAzureThreads, CurrentMetrics::ObjectStorageAzureThreadsActive, 1) + , create_reader_pool(CurrentMetrics::ObjectStorageAzureThreads, CurrentMetrics::ObjectStorageAzureThreadsActive, CurrentMetrics::ObjectStorageAzureThreadsScheduled, 1) , create_reader_scheduler(threadPoolCallbackRunner(create_reader_pool, "AzureReader")) { reader = createReader(); diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 94ce525bc38..2dedc8abdda 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -134,6 +134,7 @@ namespace CurrentMetrics { extern const Metric StorageDistributedThreads; extern const Metric StorageDistributedThreadsActive; + extern const Metric StorageDistributedThreadsScheduled; } namespace DB @@ -1214,7 +1215,7 @@ void StorageDistributed::initializeFromDisk() const auto & disks = data_volume->getDisks(); /// Make initialization for large number of disks parallel. - ThreadPool pool(CurrentMetrics::StorageDistributedThreads, CurrentMetrics::StorageDistributedThreadsActive, disks.size()); + ThreadPool pool(CurrentMetrics::StorageDistributedThreads, CurrentMetrics::StorageDistributedThreadsActive, CurrentMetrics::StorageDistributedThreadsScheduled, disks.size()); for (const DiskPtr & disk : disks) { diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 63ed84680c9..80ee1e9339d 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -77,6 +77,7 @@ namespace CurrentMetrics { extern const Metric StorageS3Threads; extern const Metric StorageS3ThreadsActive; + extern const Metric StorageS3ThreadsScheduled; } namespace ProfileEvents @@ -147,7 +148,7 @@ public: , virtual_columns(virtual_columns_) , read_keys(read_keys_) , request_settings(request_settings_) - , list_objects_pool(CurrentMetrics::StorageS3Threads, CurrentMetrics::StorageS3ThreadsActive, 1) + , list_objects_pool(CurrentMetrics::StorageS3Threads, CurrentMetrics::StorageS3ThreadsActive, CurrentMetrics::StorageS3ThreadsScheduled, 1) , list_objects_scheduler(threadPoolCallbackRunner(list_objects_pool, "ListObjects")) , file_progress_callback(file_progress_callback_) { @@ -499,7 +500,7 @@ StorageS3Source::ReadTaskIterator::ReadTaskIterator( size_t max_threads_count) : callback(callback_) { - ThreadPool pool(CurrentMetrics::StorageS3Threads, CurrentMetrics::StorageS3ThreadsActive, max_threads_count); + ThreadPool pool(CurrentMetrics::StorageS3Threads, CurrentMetrics::StorageS3ThreadsActive, CurrentMetrics::StorageS3ThreadsScheduled, max_threads_count); auto pool_scheduler = threadPoolCallbackRunner(pool, "S3ReadTaskItr"); std::vector> keys; @@ -564,7 +565,7 @@ StorageS3Source::StorageS3Source( , file_iterator(file_iterator_) , max_parsing_threads(max_parsing_threads_) , need_only_count(need_only_count_) - , create_reader_pool(CurrentMetrics::StorageS3Threads, CurrentMetrics::StorageS3ThreadsActive, 1) + , create_reader_pool(CurrentMetrics::StorageS3Threads, CurrentMetrics::StorageS3ThreadsActive, CurrentMetrics::StorageS3ThreadsScheduled, 1) , create_reader_scheduler(threadPoolCallbackRunner(create_reader_pool, "CreateS3Reader")) { } diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index ffefd41327d..d9a12095443 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -23,6 +23,7 @@ namespace CurrentMetrics { extern const Metric SystemReplicasThreads; extern const Metric SystemReplicasThreadsActive; + extern const Metric SystemReplicasThreadsScheduled; } namespace DB @@ -59,7 +60,7 @@ private: public: explicit StatusRequestsPool(size_t max_threads) - : thread_pool(CurrentMetrics::SystemReplicasThreads, CurrentMetrics::SystemReplicasThreadsActive, max_threads) + : thread_pool(CurrentMetrics::SystemReplicasThreads, CurrentMetrics::SystemReplicasThreadsActive, CurrentMetrics::SystemReplicasThreadsScheduled, max_threads) , log(&Poco::Logger::get("StatusRequestsPool")) {} diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index 13855c6d94e..611ca948c53 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -10,10 +10,12 @@ #include #include + namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } namespace DB::ErrorCodes @@ -106,7 +108,7 @@ Runner::Runner( std::cerr << "---- Run options ----\n" << std::endl; - pool.emplace(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, concurrency); + pool.emplace(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, concurrency); queue.emplace(concurrency); } @@ -461,4 +463,3 @@ Runner::~Runner() pool->wait(); generator->cleanup(*connections[0]); } - From 593f04a6b5825182940e1f241989d801b57ed93d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Nov 2023 20:19:24 +0100 Subject: [PATCH 165/192] Fix style --- src/IO/SharedThreadPools.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/SharedThreadPools.cpp b/src/IO/SharedThreadPools.cpp index 4a0ef173669..6af5aab7a38 100644 --- a/src/IO/SharedThreadPools.cpp +++ b/src/IO/SharedThreadPools.cpp @@ -130,7 +130,7 @@ StaticThreadPool & getBackupsIOThreadPool() StaticThreadPool & getActivePartsLoadingThreadPool() { - static StaticThreadPool instance("MergeTreePartsLoaderThreadPool", CurrentMetrics::MergeTreePartsLoaderThreads, CurrentMetrics::MergeTreePartsLoaderThreadsActive, CurrentMetrics::MergeTreeOutdatedPartsLoaderThreadsScheduled); + static StaticThreadPool instance("MergeTreePartsLoaderThreadPool", CurrentMetrics::MergeTreePartsLoaderThreads, CurrentMetrics::MergeTreePartsLoaderThreadsActive, CurrentMetrics::MergeTreePartsLoaderThreadsScheduled); return instance; } From 75cebb3c29c54979218f0f0f80f46609851059b9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Nov 2023 20:20:21 +0100 Subject: [PATCH 166/192] Fix build --- src/Coordination/Standalone/Context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Coordination/Standalone/Context.cpp b/src/Coordination/Standalone/Context.cpp index 4001673e01a..6942e866f23 100644 --- a/src/Coordination/Standalone/Context.cpp +++ b/src/Coordination/Standalone/Context.cpp @@ -263,7 +263,7 @@ ThreadPool & Context::getThreadPoolWriter() const auto queue_size = config.getUInt(".threadpool_writer_queue_size", 1000000); shared->threadpool_writer = std::make_unique( - CurrentMetrics::IOWriterThreads, CurrentMetrics::IOWriterThreadsActive, pool_size, pool_size, queue_size); + CurrentMetrics::IOWriterThreads, CurrentMetrics::IOWriterThreadsActive, CurrentMetrics::IOWriterThreadsScheduled, pool_size, pool_size, queue_size); }); return *shared->threadpool_writer; From 7c81d9b2b73ee35f8cf317397c2bd23dcd2336ea Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Sat, 18 Nov 2023 19:38:33 +0000 Subject: [PATCH 167/192] Fix test_keeper_auth --- tests/integration/test_keeper_auth/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_keeper_auth/test.py b/tests/integration/test_keeper_auth/test.py index e247984cc6a..78fbf84bbe2 100644 --- a/tests/integration/test_keeper_auth/test.py +++ b/tests/integration/test_keeper_auth/test.py @@ -1,6 +1,7 @@ import pytest import time from helpers.cluster import ClickHouseCluster +from helpers import keeper_utils from kazoo.client import KazooClient, KazooState from kazoo.security import ACL, make_digest_acl, make_acl from kazoo.exceptions import ( @@ -26,6 +27,7 @@ SUPERAUTH = "super:admin" def started_cluster(): try: cluster.start() + keeper_utils.wait_until_connected(cluster, node) yield cluster From a894671e8af10d4b5b70d79beccdba6be2e12174 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Sat, 18 Nov 2023 17:53:32 -0800 Subject: [PATCH 168/192] [Docs] Add perf tip for COUNT(DISTINCT expr) --- docs/en/sql-reference/aggregate-functions/reference/count.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/sql-reference/aggregate-functions/reference/count.md b/docs/en/sql-reference/aggregate-functions/reference/count.md index a98c8e50174..a40108a331a 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/count.md +++ b/docs/en/sql-reference/aggregate-functions/reference/count.md @@ -34,6 +34,10 @@ The `SELECT count() FROM table` query is optimized by default using metadata fro However `SELECT count(nullable_column) FROM table` query can be optimized by enabling the [optimize_functions_to_subcolumns](../../../operations/settings/settings.md#optimize-functions-to-subcolumns) setting. With `optimize_functions_to_subcolumns = 1` the function reads only [null](../../../sql-reference/data-types/nullable.md#finding-null) subcolumn instead of reading and processing the whole column data. The query `SELECT count(n) FROM table` transforms to `SELECT sum(NOT n.null) FROM table`. +**Improving COUNT(DISTINCT expr) performance** + +If your `COUNT(DISTINCT expr)` query is slow, consider adding a [`GROUP BY`](../../../sql-reference/statements/select/group-by.md) clause as this improves parallelization. You can also use a [projection](../../../sql-reference/statements/alter/projection.md) to create an index on the target column used with `COUNT(DISTINCT target_col)`. + **Examples** Example 1: From 96e87322b525351b7d241c9e2c6ca125d2178b20 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Sat, 18 Nov 2023 18:10:43 -0800 Subject: [PATCH 169/192] [Docs] Add insert_distributed_sync to Core Settings docs --- docs/en/operations/settings/settings.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index a22bd6e33e5..e61934d2168 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2716,6 +2716,10 @@ Default value: `0`. - [Distributed Table Engine](../../engines/table-engines/special/distributed.md/#distributed) - [Managing Distributed Tables](../../sql-reference/statements/system.md/#query-language-system-distributed) +## insert_distributed_sync {#insert_distributed_sync} + +Alias for [`distributed_foreground_insert`](#distributed_foreground_insert). + ## insert_shard_id {#insert_shard_id} If not `0`, specifies the shard of [Distributed](../../engines/table-engines/special/distributed.md/#distributed) table into which the data will be inserted synchronously. From 5e1da38720d16672bba07da1b9b54fc081b1464e Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Sat, 18 Nov 2023 18:28:29 -0800 Subject: [PATCH 170/192] [Docs] Add details on why partitions improve query perf --- .../table-engines/mergetree-family/custom-partitioning-key.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md index 7e564b23676..97d37e476ae 100644 --- a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -14,7 +14,7 @@ You should never use too granular of partitioning. Don't partition your data by Partitioning is available for the [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) family tables (including [replicated](../../../engines/table-engines/mergetree-family/replication.md) tables). [Materialized views](../../../engines/table-engines/special/materializedview.md#materializedview) based on MergeTree tables support partitioning, as well. -A partition is a logical combination of records in a table by a specified criterion. You can set a partition by an arbitrary criterion, such as by month, by day, or by event type. Each partition is stored separately to simplify manipulations of this data. When accessing the data, ClickHouse uses the smallest subset of partitions possible. +A partition is a logical combination of records in a table by a specified criterion. You can set a partition by an arbitrary criterion, such as by month, by day, or by event type. Each partition is stored separately to simplify manipulations of this data. When accessing the data, ClickHouse uses the smallest subset of partitions possible. Partitions improve performance for queries containing a partitioning key because ClickHouse will filter for that partition before selecting the parts and granules within the partition. The partition is specified in the `PARTITION BY expr` clause when [creating a table](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). The partition key can be any expression from the table columns. For example, to specify partitioning by month, use the expression `toYYYYMM(date_column)`: From 8062fb578c7b6c65d4d24efa28bb47e8c8783ae8 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Sat, 18 Nov 2023 18:38:49 -0800 Subject: [PATCH 171/192] [Docs] Recommend ReplacingMergeTree for frequent updates --- docs/en/engines/table-engines/mergetree-family/mergetree.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index e615c9ad9d3..d250cfd1a08 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -6,7 +6,7 @@ sidebar_label: MergeTree # MergeTree -The `MergeTree` engine and other engines of this family (`*MergeTree`) are the most robust ClickHouse table engines. +The `MergeTree` engine and other engines of this family (`*MergeTree`) are the most common and most robust ClickHouse table engines. Engines in the `MergeTree` family are designed for inserting a very large amount of data into a table. The data is quickly written to the table part by part, then rules are applied for merging the parts in the background. This method is much more efficient than continually rewriting the data in storage during insert. @@ -32,6 +32,8 @@ Main features: The [Merge](/docs/en/engines/table-engines/special/merge.md/#merge) engine does not belong to the `*MergeTree` family. ::: +If you need to update rows frequently, we recommend using the [`ReplacingMergeTree`](/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md). Using `ALTER TALBE my_table UPDATE` to update rows triggers a mutation, which causes parts to be re-written and uses IO/resources. With `ReplacingMergeTree`, you can simply insert the updated rows and the old rows will be replaced according to the table sorting key. + ## Creating a Table {#table_engine-mergetree-creating-a-table} ``` sql From a398e3f51e499c5523e62ac4c296b0203304e672 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Sat, 18 Nov 2023 18:40:14 -0800 Subject: [PATCH 172/192] [Docs] Fix typo --- docs/en/engines/table-engines/mergetree-family/mergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index d250cfd1a08..810b7596150 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -6,7 +6,7 @@ sidebar_label: MergeTree # MergeTree -The `MergeTree` engine and other engines of this family (`*MergeTree`) are the most common and most robust ClickHouse table engines. +The `MergeTree` engine and other engines of this family (`*MergeTree`) are the most commonly used and most robust ClickHouse table engines. Engines in the `MergeTree` family are designed for inserting a very large amount of data into a table. The data is quickly written to the table part by part, then rules are applied for merging the parts in the background. This method is much more efficient than continually rewriting the data in storage during insert. From f29777db2d77aee2d17034e6844377f65e5ddfe7 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Sat, 18 Nov 2023 18:41:20 -0800 Subject: [PATCH 173/192] [Docs] More typos --- docs/en/engines/table-engines/mergetree-family/mergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 810b7596150..f0bc45b9f53 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -32,7 +32,7 @@ Main features: The [Merge](/docs/en/engines/table-engines/special/merge.md/#merge) engine does not belong to the `*MergeTree` family. ::: -If you need to update rows frequently, we recommend using the [`ReplacingMergeTree`](/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md). Using `ALTER TALBE my_table UPDATE` to update rows triggers a mutation, which causes parts to be re-written and uses IO/resources. With `ReplacingMergeTree`, you can simply insert the updated rows and the old rows will be replaced according to the table sorting key. +If you need to update rows frequently, we recommend using the [`ReplacingMergeTree`](/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md) table engine. Using `ALTER TABLE my_table UPDATE` to update rows triggers a mutation, which causes parts to be re-written and uses IO/resources. With `ReplacingMergeTree`, you can simply insert the updated rows and the old rows will be replaced according to the table sorting key. ## Creating a Table {#table_engine-mergetree-creating-a-table} From 9d3c62ec418802371fcf7f466163d683c1ec262a Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Sat, 18 Nov 2023 18:51:35 -0800 Subject: [PATCH 174/192] [Docs] Recommend against OPTIMIZE FINAL in OPTIMIZE page --- docs/en/sql-reference/statements/optimize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md index 49843eaff9a..07b5a196096 100644 --- a/docs/en/sql-reference/statements/optimize.md +++ b/docs/en/sql-reference/statements/optimize.md @@ -5,7 +5,7 @@ sidebar_label: OPTIMIZE title: "OPTIMIZE Statement" --- -This query tries to initialize an unscheduled merge of data parts for tables. +This query tries to initialize an unscheduled merge of data parts for tables. Note that we generally recommend against using `OPTIMIZE TABLE ... FINAL` (see these [docs](/docs/en/optimize/avoidoptimizefinal)) as its use case is meant for administration, not for daily operations. :::note `OPTIMIZE` can’t fix the `Too many parts` error. From 097f80657c35c83ca522b412a040dfeaa36011d5 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Sun, 19 Nov 2023 11:14:24 +0100 Subject: [PATCH 175/192] Fewer concurrent requests in 02908_many_requests_to_system_replicas --- .../02908_many_requests_to_system_replicas.reference | 2 +- .../0_stateless/02908_many_requests_to_system_replicas.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference index d7850e59dec..af0e50ec332 100644 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference @@ -1,5 +1,5 @@ Creating 300 tables -Making making 500 requests to system.replicas +Making making 200 requests to system.replicas Query system.replicas while waiting for other concurrent requests to finish 0 900 diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh index c620fcf4bea..f93175529c0 100755 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e NUM_TABLES=300 -CONCURRENCY=500 +CONCURRENCY=200 echo "Creating $NUM_TABLES tables" From f27018c14196e938aa3b16cae387e46d2578501a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Nov 2023 15:21:11 +0100 Subject: [PATCH 176/192] Own CMake for GRPC --- cmake/limit_jobs.cmake | 4 +- contrib/grpc-cmake/CMakeLists.txt | 40 +- contrib/grpc-cmake/grpc.cmake | 1864 +++++++++++++++++++++++++++++ 3 files changed, 1868 insertions(+), 40 deletions(-) create mode 100644 contrib/grpc-cmake/grpc.cmake diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index 28ccb62e10c..a43e208ff0d 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -21,7 +21,7 @@ if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY) set (PARALLEL_COMPILE_JOBS 1) endif () if (PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES) - message(WARNING "The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.") + message(INFO "The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.") endif() endif () @@ -32,7 +32,7 @@ if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY) set (PARALLEL_LINK_JOBS 1) endif () if (PARALLEL_LINK_JOBS LESS NUMBER_OF_LOGICAL_CORES) - message(WARNING "The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.") + message(INFO "The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.") endif() endif () diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index 09ed2fe3f80..b8b5f5580c4 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -9,50 +9,14 @@ endif() set(_gRPC_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/grpc") set(_gRPC_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/grpc") -# Use re2 from ClickHouse contrib, not from gRPC third_party. -set(gRPC_RE2_PROVIDER "clickhouse" CACHE STRING "" FORCE) -set(_gRPC_RE2_INCLUDE_DIR "") -set(_gRPC_RE2_LIBRARIES ch_contrib::re2) - -# Use zlib from ClickHouse contrib, not from gRPC third_party. -set(gRPC_ZLIB_PROVIDER "clickhouse" CACHE STRING "" FORCE) -set(_gRPC_ZLIB_INCLUDE_DIR "") -set(_gRPC_ZLIB_LIBRARIES ch_contrib::zlib) - -# Use protobuf from ClickHouse contrib, not from gRPC third_party. -set(gRPC_PROTOBUF_PROVIDER "clickhouse" CACHE STRING "" FORCE) -set(_gRPC_PROTOBUF_LIBRARIES ch_contrib::protobuf) -set(_gRPC_PROTOBUF_PROTOC "protoc") -set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE $) -set(_gRPC_PROTOBUF_PROTOC_LIBRARIES ch_contrib::protoc) - if(TARGET OpenSSL::SSL) set(gRPC_USE_UNSECURE_LIBRARIES FALSE) else() set(gRPC_USE_UNSECURE_LIBRARIES TRUE) endif() -# Use OpenSSL from ClickHouse contrib, not from gRPC third_party. -set(gRPC_SSL_PROVIDER "clickhouse" CACHE STRING "" FORCE) -set(_gRPC_SSL_INCLUDE_DIR "") -set(_gRPC_SSL_LIBRARIES OpenSSL::Crypto OpenSSL::SSL) - -# Use abseil-cpp from ClickHouse contrib, not from gRPC third_party. -set(gRPC_ABSL_PROVIDER "clickhouse" CACHE STRING "" FORCE) - -# We don't want to build C# extensions. -set(gRPC_BUILD_CSHARP_EXT OFF) - -# TODO: Remove this. We generally like to compile with C++23 but grpc isn't ready yet. -set (CMAKE_CXX_STANDARD 20) - -set(_gRPC_CARES_LIBRARIES ch_contrib::c-ares) -set(gRPC_CARES_PROVIDER "clickhouse" CACHE STRING "" FORCE) -add_subdirectory("${_gRPC_SOURCE_DIR}" "${_gRPC_BINARY_DIR}") - -# The contrib/grpc/CMakeLists.txt redefined the PROTOBUF_GENERATE_GRPC_CPP() function for its own purposes, -# so we need to redefine it back. -include("${ClickHouse_SOURCE_DIR}/contrib/grpc-cmake/protobuf_generate_grpc.cmake") +include(grpc.cmake) +include(protobuf_generate_grpc.cmake) set(gRPC_CPP_PLUGIN $) set(gRPC_PYTHON_PLUGIN $) diff --git a/contrib/grpc-cmake/grpc.cmake b/contrib/grpc-cmake/grpc.cmake new file mode 100644 index 00000000000..43d4edd191e --- /dev/null +++ b/contrib/grpc-cmake/grpc.cmake @@ -0,0 +1,1864 @@ +# This file was edited for ClickHouse. + +# GRPC global cmake file +# This currently builds C and C++ code. +# This file has been automatically generated from a template file. +# Please look at the templates directory instead. +# This file can be regenerated from the template by running +# tools/buildgen/generate_projects.sh +# +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# We want to use C++23, but GRPC is not ready +set (CMAKE_CXX_STANDARD 20) + +set(_gRPC_ZLIB_INCLUDE_DIR "") +set(_gRPC_ZLIB_LIBRARIES ch_contrib::zlib) + +set(_gRPC_CARES_LIBRARIES ch_contrib::c-ares) + +set(_gRPC_RE2_INCLUDE_DIR "") +set(_gRPC_RE2_LIBRARIES ch_contrib::re2) + +set(_gRPC_SSL_INCLUDE_DIR "") +set(_gRPC_SSL_LIBRARIES OpenSSL::Crypto OpenSSL::SSL) + +set(_gRPC_PROTOBUF_LIBRARIES ch_contrib::protobuf) +set(_gRPC_PROTOBUF_PROTOC "protoc") +set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE $) +set(_gRPC_PROTOBUF_PROTOC_LIBRARIES ch_contrib::protoc) + + +if(UNIX) + if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") + set(_gRPC_PLATFORM_LINUX ON) + elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(_gRPC_PLATFORM_MAC ON) + elseif(${CMAKE_SYSTEM_NAME} MATCHES "iOS") + set(_gRPC_PLATFORM_IOS ON) + elseif(${CMAKE_SYSTEM_NAME} MATCHES "Android") + set(_gRPC_PLATFORM_ANDROID ON) + else() + set(_gRPC_PLATFORM_POSIX ON) + endif() +endif() + +if(UNIX AND NOT HAIKU) + # -pthread does more than -lpthread + set(THREADS_PREFER_PTHREAD_FLAG ON) + find_package(Threads) + set(_gRPC_ALLTARGETS_LIBRARIES ${CMAKE_DL_LIBS} Threads::Threads) + if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) + set(_gRPC_ALLTARGETS_LIBRARIES ${_gRPC_ALLTARGETS_LIBRARIES} rt) + endif() +endif() + +set(_gRPC_ADDRESS_SORTING_INCLUDE_DIR "${_gRPC_SOURCE_DIR}/third_party/address_sorting/include") +set(_gRPC_ADDRESS_SORTING_LIBRARIES address_sorting) + +set(UPB_ROOT_DIR ${_gRPC_SOURCE_DIR}/third_party/upb) + +set(_gRPC_UPB_INCLUDE_DIR "${UPB_ROOT_DIR}" "${_gRPC_SOURCE_DIR}/third_party/utf8_range") +set(_gRPC_UPB_GRPC_GENERATED_DIR "${_gRPC_SOURCE_DIR}/src//core/ext/upb-generated" "${_gRPC_SOURCE_DIR}/src//core/ext/upbdefs-generated") + +set(_gRPC_UPB_LIBRARIES upb) + +set(_gRPC_XXHASH_INCLUDE_DIR "${_gRPC_SOURCE_DIR}/third_party/xxhash") + +add_library(address_sorting + ${_gRPC_SOURCE_DIR}/third_party/address_sorting/address_sorting.c + ${_gRPC_SOURCE_DIR}/third_party/address_sorting/address_sorting_posix.c + ${_gRPC_SOURCE_DIR}/third_party/address_sorting/address_sorting_windows.c +) + +target_compile_features(address_sorting PUBLIC cxx_std_14) + +target_include_directories(address_sorting + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(address_sorting + ${_gRPC_ALLTARGETS_LIBRARIES} +) + + +add_library(gpr + ${_gRPC_SOURCE_DIR}/src/core/lib/config/config_vars.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/config/config_vars_non_generated.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/config/load_config.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_local.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/alloc.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/android/log.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/atm.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/iphone/cpu.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/linux/cpu.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/linux/log.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/log.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/msys/tmpfile.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/posix/cpu.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/posix/log.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/posix/string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/posix/sync.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/posix/time.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/posix/tmpfile.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/sync.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/sync_abseil.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/time.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/time_precise.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/windows/cpu.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/windows/log.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/windows/string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/windows/string_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/windows/sync.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/windows/time.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/windows/tmpfile.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/wrap_memcpy.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/crash.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/examine_stack.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/fork.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/host_port.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/linux/env.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/mpscq.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/posix/env.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/posix/stat.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/posix/thd.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/strerror.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/tchar.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/time_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/windows/env.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/windows/stat.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/windows/thd.cc +) + +target_compile_features(gpr PUBLIC cxx_std_14) + +target_include_directories(gpr + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(gpr + ${_gRPC_ALLTARGETS_LIBRARIES} + absl::base + absl::core_headers + absl::flags + absl::flags_marshalling + absl::any_invocable + absl::memory + absl::random_random + absl::status + absl::cord + absl::str_format + absl::strings + absl::synchronization + absl::time + absl::optional + absl::variant +) +if(_gRPC_PLATFORM_ANDROID) + target_link_libraries(gpr + android + log + ) +endif() + + +add_library(grpc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/backend_metrics/backend_metric_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/census/grpc_context.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/channel_idle/channel_idle_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/channel_idle/idle_filter_state.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backend_metric.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backup_poller.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/channel_connectivity.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_channelz.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_service_config.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/config_selector.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/dynamic_filters.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/global_subchannel_pool.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/http_proxy_mapper.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/endpoint_list.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/health_check_client.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/rls/rls.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/static_stride_scheduler.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/weighted_round_robin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/local_subchannel_pool.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/binder/binder_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/event_engine/event_engine_client_channel_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/event_engine/service_config_helper.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/polling_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_service_config.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_throttle.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel_pool_interface.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel_stream_client.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/deadline/deadline_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/fault_injection/fault_injection_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/client/http_client_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/client_authority_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/http_filters_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/message_compress/compression_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/server/http_server_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/message_size/message_size_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/rbac/rbac_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/rbac/rbac_service_config_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/stateful_session/stateful_session_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/gcp/metadata_query.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/alpn/alpn.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/chttp2_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/chttp2_server.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_decoder.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_encoder.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_transport.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/decode_huff.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/flow_control.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_data.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_goaway.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_ping.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_settings.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_window_update.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parse_result.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_settings.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/huffsyms.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/max_concurrent_streams_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/parsing.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_abuse_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_callbacks.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/write_size_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/writing.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/inproc/inproc_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/inproc/inproc_transport.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/certs.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/clusters.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/config_dump.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/config_dump_shared.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/init_dump.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/listeners.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/memory.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/metrics.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/mutex_stats.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/server_info.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/tap.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/annotations/deprecation.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/annotations/resource.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/accesslog/v3/accesslog.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/bootstrap/v3/bootstrap.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/cluster/v3/circuit_breaker.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/cluster/v3/cluster.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/cluster/v3/filter.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/cluster/v3/outlier_detection.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/common/matcher/v3/matcher.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/address.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/backoff.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/base.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/config_source.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/event_service_config.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/extension.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/grpc_method_list.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/grpc_service.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/health_check.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/http_uri.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/protocol.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/proxy_protocol.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/resolver.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/socket_option.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/substitution_format_string.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/udp_socket_config.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint_components.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/endpoint/v3/load_report.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/listener/v3/api_listener.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/listener/v3/listener.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/listener/v3/listener_components.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/listener/v3/quic_config.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/listener/v3/udp_listener_config.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/metrics/v3/metrics_service.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/metrics/v3/stats.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/overload/v3/overload.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/rbac/v3/rbac.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/route/v3/route.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/route/v3/route_components.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/route/v3/scoped_route.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/tap/v3/common.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/datadog.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/dynamic_ot.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/http_tracer.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/lightstep.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/opencensus.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/opentelemetry.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/service.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/skywalking.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/trace.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/xray.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/zipkin.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/data/accesslog/v3/accesslog.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/clusters/aggregate/v3/cluster.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/filters/common/fault/v3/fault.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/filters/http/fault/v3/fault.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/filters/http/rbac/v3/rbac.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/filters/http/router/v3/router.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/common/v3/common.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/cert.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/common.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/secret.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/service/discovery/v3/ads.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/service/discovery/v3/discovery.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/service/load_stats/v3/lrs.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/service/status/v3/csds.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/http/v3/cookie.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/http/v3/path_transformation.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/filter_state.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/http_inputs.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/metadata.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/node.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/number.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/path.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/regex.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/status_code_input.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/string.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/struct.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/value.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/metadata/v3/metadata.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/tracing/v3/custom_tag.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/hash_policy.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/http.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/http_status.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/percent.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/range.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/ratelimit_strategy.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/ratelimit_unit.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/semantic_version.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/token_bucket.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/annotations.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/expr/v1alpha1/checked.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/http.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/httpbody.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/any.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/duration.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/empty.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/struct.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/timestamp.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/wrappers.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/rpc/status.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/opencensus/proto/trace/v1/trace_config.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/altscontext.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/lb/v1/load_balancer.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls_config.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/annotations/migrate.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/annotations/security.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/annotations/sensitive.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/annotations/status.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/annotations/versioning.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/validate/validate.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/annotations/v3/migrate.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/annotations/v3/security.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/annotations/v3/sensitive.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/annotations/v3/status.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/annotations/v3/versioning.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/authority.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/cidr.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/collection_entry.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/context_params.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/extension.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/resource.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/resource_locator.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/resource_name.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/service/orca/v3/orca.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/cel.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/domain.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/http_inputs.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/ip.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/matcher.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/range.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/regex.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/string.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/v3/cel.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/v3/range.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/v3/typed_struct.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/certs.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/clusters.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump_shared.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/init_dump.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/listeners.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/memory.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/metrics.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/mutex_stats.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/server_info.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/tap.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/annotations/deprecation.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/annotations/resource.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/accesslog/v3/accesslog.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/bootstrap/v3/bootstrap.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/circuit_breaker.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/cluster.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/filter.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/outlier_detection.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/common/matcher/v3/matcher.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/address.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/backoff.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/base.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/config_source.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/event_service_config.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/extension.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/grpc_method_list.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/grpc_service.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/health_check.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/http_uri.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/protocol.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/proxy_protocol.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/resolver.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/socket_option.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/substitution_format_string.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/udp_socket_config.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/endpoint/v3/endpoint.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/endpoint/v3/endpoint_components.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/endpoint/v3/load_report.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/listener/v3/api_listener.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener_components.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/listener/v3/quic_config.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/listener/v3/udp_listener_config.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/metrics/v3/metrics_service.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/metrics/v3/stats.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/overload/v3/overload.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/rbac/v3/rbac.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/route/v3/route.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/route/v3/route_components.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/route/v3/scoped_route.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/tap/v3/common.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/datadog.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/dynamic_ot.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/http_tracer.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/lightstep.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/opencensus.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/opentelemetry.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/service.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/skywalking.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/trace.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/xray.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/zipkin.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/data/accesslog/v3/accesslog.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/clusters/aggregate/v3/cluster.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/filters/common/fault/v3/fault.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/fault/v3/fault.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/router/v3/router.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/cert.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/common.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/secret.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/service/discovery/v3/ads.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/service/discovery/v3/discovery.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/service/load_stats/v3/lrs.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/service/status/v3/csds.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/http/v3/cookie.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/http/v3/path_transformation.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/filter_state.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/http_inputs.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/metadata.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/node.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/number.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/path.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/regex.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/status_code_input.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/string.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/struct.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/value.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/metadata/v3/metadata.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/tracing/v3/custom_tag.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/hash_policy.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/http.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/http_status.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/percent.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/range.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/ratelimit_strategy.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/ratelimit_unit.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/semantic_version.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/token_bucket.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/api/annotations.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/api/expr/v1alpha1/checked.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/api/expr/v1alpha1/syntax.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/api/http.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/api/httpbody.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/protobuf/any.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/protobuf/descriptor.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/protobuf/duration.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/protobuf/empty.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/protobuf/struct.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/protobuf/timestamp.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/protobuf/wrappers.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/rpc/status.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/opencensus/proto/trace/v1/trace_config.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/src/proto/grpc/lookup/v1/rls_config.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/udpa/annotations/migrate.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/udpa/annotations/security.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/udpa/annotations/sensitive.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/udpa/annotations/status.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/udpa/annotations/versioning.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/validate/validate.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/annotations/v3/migrate.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/annotations/v3/security.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/annotations/v3/sensitive.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/annotations/v3/status.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/annotations/v3/versioning.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/authority.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/cidr.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/collection_entry.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/context_params.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/extension.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/resource.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/resource_locator.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/resource_name.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/cel.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/domain.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/http_inputs.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/ip.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/matcher.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/range.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/regex.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/string.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/v3/cel.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/v3/range.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/v3/typed_struct.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/certificate_provider_store.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/file_watcher_certificate_provider_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_api.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_audit_logger_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_bootstrap.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_bootstrap_grpc.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_certificate_provider.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_channel_stack_modifier.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_client.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_client_grpc.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_client_stats.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_cluster.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_cluster_specifier_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_common_types.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_health_status.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_http_fault_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_http_filters.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_http_rbac_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_http_stateful_session_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_lb_policy_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_listener.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_route_config.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_routing.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_server_config_fetcher.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_transport_grpc.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/address_utils/parse_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/address_utils/sockaddr_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/backoff/backoff.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/backoff/random_early_detection.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/call_tracer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_args.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_args_preconditioning.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack_builder.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack_builder_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channelz.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channelz_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/connected_channel.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/promise_based_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/server_call_tracer_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/status_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/compression/compression.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/compression/compression_internal.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/compression/message_compress.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/config/core_configuration.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/event_log.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/histogram_view.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/stats.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/stats_data.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/ares_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/cf_engine/cf_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/cf_engine/cfstream_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/cf_engine/dns_service_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/channel_args_endpoint_config.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/default_event_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/default_event_engine_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/event_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/forkable.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/memory_allocator.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/ev_poll_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/internal_errqueue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/lockfree_event.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_engine_listener.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/timer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/timer_heap.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/timer_manager.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/traced_buffer_list.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/resolved_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/shim.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/slice.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/slice_buffer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/tcp_socket_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_pool/thread_count.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_pool/thread_pool_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thready_event_engine/thready_event_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/time_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/iocp.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/win_socket.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/windows_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/windows_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/windows_listener.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/work_queue/basic_work_queue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/experiments/config.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/experiments/experiments.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/load_file.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/per_cpu.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/ref_counted_string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/status_helper.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/time.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/time_averaged_stats.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/validation_errors.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/work_serializer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/handshaker/proxy_mapper_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/http/format_request.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/http/httpcli.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/http/httpcli_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/http/parser.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/buffer_list.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/call_combiner.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/cfstream_handle.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/closure.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/combiner.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/dualstack_socket_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/error.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/error_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_apple.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_epoll1_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_poll_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/event_engine_shims/closure.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/event_engine_shims/endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/event_engine_shims/tcp_client.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/exec_ctx.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/executor.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_fallback.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_host_name_max.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_sysconf.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/internal_errqueue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iocp_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_internal.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/load_file.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/lockfree_event.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/polling_entity.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/sockaddr_utils_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_factory_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_mutator.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_common_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/systemd_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_common.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_generic.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_heap.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_manager.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix_noop.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/vsock.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_eventfd.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_nospecial.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_pipe.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/json/json_object_loader.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/json/json_reader.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/json/json_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/json/json_writer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/load_balancing/lb_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/load_balancing/lb_policy_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/matchers/matchers.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/activity.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/party.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/sleep.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resolver/resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resolver/resolver_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resolver/server_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/api.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/arena.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/memory_quota.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/periodic_update.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/resource_quota.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/thread_quota.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/audit_logging.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/evaluate_args.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/grpc_authorization_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/grpc_server_authz_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/matchers.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/rbac_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/stdout_logger.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/certificate_provider/certificate_provider_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/context/security_context.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/alts_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/call_creds_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/channel_creds_registry_init.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/composite/composite_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/external/aws_external_account_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/external/aws_request_signer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/external/external_account_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/external/file_external_account_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/external/url_external_account_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/fake/fake_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/google_default/credentials_generic.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/google_default/google_default_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/iam/iam_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/insecure/insecure_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/json_token.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/jwt_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/jwt_verifier.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/local/local_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/plugin/plugin_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/ssl/ssl_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/grpc_tls_certificate_match.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/tls_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/tls_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/xds/xds_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/alts/alts_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/fake/fake_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/insecure/insecure_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_fallback.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_supported.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/local/local_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/tls/tls_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/client_auth_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/secure_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/security_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/server_auth_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/tsi_error.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/util/json_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/service_config/service_config_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/service_config/service_config_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/b64.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/percent_encoding.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice_buffer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice_refcount.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice_string_helpers.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/api_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/builtins.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer_reader.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call_details.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call_log_batch.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel_init.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel_ping.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel_stack_type.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/event_string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/init.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/init_internally.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/lame_client.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/metadata_array.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/server.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/validate_metadata.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/version.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/batch_builder.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/bdp_estimator.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/connectivity_state.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/error_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/handshaker_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/http_connect_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/metadata_batch.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/parsed_metadata.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/pid_controller.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/status_conversion.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/tcp_connect_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/timeout_encoding.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/transport.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/transport_op_string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/uri/uri_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry_extra.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/crypt/aes_gcm.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/crypt/gsec.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_counter.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_crypter.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_frame_protector.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_seal_privacy_integrity_crypter.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_unseal_privacy_integrity_crypter.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/frame_handler.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_handshaker_client.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_shared_resource.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_tsi_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/transport_security_common_api.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/fake_transport_security.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/local_transport_security.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl/key_logging/ssl_key_logging.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_boringssl.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_cache.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_openssl.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl_transport_security.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl_transport_security_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/transport_security.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/transport_security_grpc.cc +) + +target_compile_features(grpc PUBLIC cxx_std_14) + +target_include_directories(grpc + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(grpc + ${_gRPC_ALLTARGETS_LIBRARIES} + ${_gRPC_RE2_LIBRARIES} + upb_json_lib + upb_textformat_lib + ${_gRPC_ZLIB_LIBRARIES} + absl::algorithm_container + absl::cleanup + absl::flat_hash_map + absl::flat_hash_set + absl::inlined_vector + absl::bind_front + absl::function_ref + absl::hash + absl::type_traits + absl::random_bit_gen_ref + absl::random_distributions + absl::statusor + absl::span + absl::utility + ${_gRPC_CARES_LIBRARIES} + gpr + ${_gRPC_SSL_LIBRARIES} + ${_gRPC_ADDRESS_SORTING_LIBRARIES} +) +if(_gRPC_PLATFORM_IOS OR _gRPC_PLATFORM_MAC) + target_link_libraries(grpc "-framework CoreFoundation") +endif() + +add_library(grpc_unsecure + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/backend_metrics/backend_metric_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/census/grpc_context.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/channel_idle/channel_idle_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/channel_idle/idle_filter_state.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backend_metric.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backup_poller.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/channel_connectivity.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_channelz.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_service_config.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/config_selector.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/dynamic_filters.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/global_subchannel_pool.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/http_proxy_mapper.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/endpoint_list.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/health_check_client.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/rls/rls.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/static_stride_scheduler.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/weighted_round_robin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/local_subchannel_pool.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/binder/binder_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/event_engine/event_engine_client_channel_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/event_engine/service_config_helper.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/polling_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_service_config.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_throttle.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel_pool_interface.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel_stream_client.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/deadline/deadline_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/fault_injection/fault_injection_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/client/http_client_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/client_authority_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/http_filters_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/message_compress/compression_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/server/http_server_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/message_size/message_size_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/chttp2_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/chttp2_server.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_decoder.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_encoder.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_transport.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/decode_huff.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/flow_control.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_data.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_goaway.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_ping.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_settings.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_window_update.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parse_result.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_settings.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/huffsyms.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/max_concurrent_streams_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/parsing.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_abuse_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_callbacks.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/write_size_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/writing.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/inproc/inproc_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/inproc/inproc_transport.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/annotations.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/http.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/any.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/duration.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/empty.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/struct.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/timestamp.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/wrappers.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/rpc/status.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/altscontext.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/lb/v1/load_balancer.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/validate/validate.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/service/orca/v3/orca.upb.c + ${_gRPC_SOURCE_DIR}/src/core/lib/address_utils/parse_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/address_utils/sockaddr_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/backoff/backoff.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/backoff/random_early_detection.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/call_tracer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_args.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_args_preconditioning.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack_builder.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack_builder_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channelz.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channelz_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/connected_channel.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/promise_based_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/server_call_tracer_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/status_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/compression/compression.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/compression/compression_internal.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/compression/message_compress.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/config/core_configuration.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/event_log.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/histogram_view.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/stats.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/stats_data.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/ares_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/cf_engine/cf_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/cf_engine/cfstream_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/cf_engine/dns_service_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/channel_args_endpoint_config.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/default_event_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/default_event_engine_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/event_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/forkable.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/memory_allocator.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/ev_poll_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/internal_errqueue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/lockfree_event.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_engine_listener.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/timer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/timer_heap.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/timer_manager.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/traced_buffer_list.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/resolved_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/shim.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/slice.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/slice_buffer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/tcp_socket_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_pool/thread_count.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_pool/thread_pool_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thready_event_engine/thready_event_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/time_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/iocp.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/win_socket.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/windows_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/windows_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/windows_listener.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/work_queue/basic_work_queue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/experiments/config.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/experiments/experiments.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/load_file.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/per_cpu.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/ref_counted_string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/status_helper.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/time.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/time_averaged_stats.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/validation_errors.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/work_serializer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/handshaker/proxy_mapper_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/http/format_request.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/http/httpcli.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/http/parser.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/buffer_list.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/call_combiner.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/cfstream_handle.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/closure.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/combiner.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/dualstack_socket_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/error.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/error_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_apple.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_epoll1_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_poll_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/event_engine_shims/closure.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/event_engine_shims/endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/event_engine_shims/tcp_client.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/exec_ctx.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/executor.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_fallback.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_host_name_max.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_sysconf.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/internal_errqueue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iocp_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_internal.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/load_file.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/lockfree_event.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/polling_entity.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/sockaddr_utils_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_factory_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_mutator.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_common_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/systemd_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_common.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_generic.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_heap.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_manager.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix_noop.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/vsock.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_eventfd.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_nospecial.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_pipe.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/json/json_object_loader.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/json/json_reader.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/json/json_writer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/load_balancing/lb_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/load_balancing/lb_policy_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/activity.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/party.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/sleep.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resolver/resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resolver/resolver_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resolver/server_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/api.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/arena.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/memory_quota.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/periodic_update.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/resource_quota.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/thread_quota.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/evaluate_args.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/grpc_server_authz_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/certificate_provider/certificate_provider_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/context/security_context.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/call_creds_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/composite/composite_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/fake/fake_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/insecure/insecure_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/plugin/plugin_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/tls_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/fake/fake_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/insecure/insecure_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_fallback.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_supported.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/client_auth_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/secure_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/security_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/server_auth_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/tsi_error.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/util/json_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/service_config/service_config_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/service_config/service_config_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/b64.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/percent_encoding.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice_buffer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice_refcount.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice_string_helpers.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/api_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/builtins.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer_reader.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call_details.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call_log_batch.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel_init.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel_ping.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel_stack_type.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/event_string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/init.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/init_internally.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/lame_client.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/metadata_array.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/server.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/validate_metadata.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/version.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/batch_builder.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/bdp_estimator.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/connectivity_state.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/error_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/handshaker_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/http_connect_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/metadata_batch.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/parsed_metadata.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/pid_controller.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/status_conversion.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/tcp_connect_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/timeout_encoding.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/transport.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/transport_op_string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/uri/uri_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry_noextra.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/transport_security_common_api.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/fake_transport_security.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/local_transport_security.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/transport_security.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/transport_security_grpc.cc + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/accessors.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/build_enum.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/decode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/internal/base92.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/link.c + ${gRPC_ADDITIONAL_DLL_SRC} +) + +target_compile_features(grpc_unsecure PUBLIC cxx_std_14) + +target_include_directories(grpc_unsecure + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(grpc_unsecure + ${_gRPC_ALLTARGETS_LIBRARIES} + upb_collections_lib + upb + ${_gRPC_ZLIB_LIBRARIES} + absl::algorithm_container + absl::cleanup + absl::flat_hash_map + absl::flat_hash_set + absl::inlined_vector + absl::bind_front + absl::function_ref + absl::hash + absl::type_traits + absl::random_bit_gen_ref + absl::random_distributions + absl::statusor + absl::span + absl::utility + ${_gRPC_CARES_LIBRARIES} + gpr + ${_gRPC_ADDRESS_SORTING_LIBRARIES} +) +if(_gRPC_PLATFORM_IOS OR _gRPC_PLATFORM_MAC) + target_link_libraries(grpc_unsecure "-framework CoreFoundation") +endif() + +add_library(upb + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/base/status.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/collections/array.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/collections/map.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/collections/map_sorter.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/hash/common.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/lex/atoi.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/lex/round_trip.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/lex/strtod.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/lex/unicode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mem/alloc.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mem/arena.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/extension_registry.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/internal/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/decode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/decode_fast.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/encode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/eps_copy_input_stream.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/reader.c +) + +target_compile_features(upb PUBLIC cxx_std_14) + +target_include_directories(upb + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(upb + ${_gRPC_ALLTARGETS_LIBRARIES} + utf8_range_lib +) + + +add_library(upb_collections_lib + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/base/status.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/collections/array.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/collections/map.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/collections/map_sorter.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/hash/common.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mem/alloc.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mem/arena.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/extension_registry.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/internal/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/message.c +) + +target_compile_features(upb_collections_lib PUBLIC cxx_std_14) + +target_include_directories(upb_collections_lib + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(upb_collections_lib + ${_gRPC_ALLTARGETS_LIBRARIES} +) + + + +add_library(upb_json_lib + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/json/decode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/json/encode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/accessors.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/build_enum.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/decode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/internal/base92.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/internal/encode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/link.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/def_builder.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/def_pool.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/def_type.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/desc_state.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/enum_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/enum_reserved_range.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/enum_value_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/extension_range.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/field_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/file_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/message_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/message_reserved_range.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/method_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/oneof_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/service_def.c +) + +target_compile_features(upb_json_lib PUBLIC cxx_std_14) + +target_include_directories(upb_json_lib + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(upb_json_lib + ${_gRPC_ALLTARGETS_LIBRARIES} + upb_collections_lib + upb +) + + +add_library(upb_textformat_lib + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/accessors.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/build_enum.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/decode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/internal/base92.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/internal/encode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/link.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/def_builder.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/def_pool.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/def_type.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/desc_state.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/enum_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/enum_reserved_range.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/enum_value_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/extension_range.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/field_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/file_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/message_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/message_reserved_range.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/method_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/oneof_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/service_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/text/encode.c +) + +target_compile_features(upb_textformat_lib PUBLIC cxx_std_14) + +target_include_directories(upb_textformat_lib + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(upb_textformat_lib + ${_gRPC_ALLTARGETS_LIBRARIES} + upb_collections_lib + upb +) + + +add_library(utf8_range_lib + ${_gRPC_SOURCE_DIR}/third_party/utf8_range/naive.c + ${_gRPC_SOURCE_DIR}/third_party/utf8_range/range2-neon.c + ${_gRPC_SOURCE_DIR}/third_party/utf8_range/range2-sse.c +) + +target_compile_features(utf8_range_lib PUBLIC cxx_std_14) + +target_include_directories(utf8_range_lib + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(utf8_range_lib + ${_gRPC_ALLTARGETS_LIBRARIES} +) + + +add_library(grpc++ + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/client/binder_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/client/channel_create.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/client/channel_create_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/client/connection_id_generator.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/client/endpoint_binder_pool.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/client/jni_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/client/security_policy_setting.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/security_policy/binder_security_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/server/binder_server.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/server/binder_server_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/transport/binder_transport.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/utils/ndk_binder.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/utils/transport_stream_receiver_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/wire_format/binder_android.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/wire_format/binder_constants.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/wire_format/transaction.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/wire_format/wire_reader_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/wire_format/wire_writer.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/channel_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_callback.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_interceptor.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_stats_interceptor.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel_internal.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel_posix.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/insecure_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/secure_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/xds_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/alarm.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/auth_property_iterator.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/channel_arguments.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/channel_filter.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/completion_queue_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/resource_quota_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/rpc_method.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/secure_auth_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/secure_channel_arguments.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/secure_create_auth_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/tls_certificate_provider.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/tls_certificate_verifier.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/tls_credentials_options.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/validate_service_config.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/version_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/async_generic_service.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/backend_metric_recorder.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/channel_argument_option.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/create_default_thread_pool.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/external_connection_acceptor_impl.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/health/default_health_check_service.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/health/health_check_service.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/health/health_check_service_server_builder_option.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/insecure_server_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/secure_server_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_builder.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_callback.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_posix.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/xds_server_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/thread_manager/thread_manager.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/byte_buffer_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/status.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/string_ref.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/time_cc.cc + ${gRPC_UPB_GEN_DUPL_SRC} +) + +target_compile_features(grpc++ PUBLIC cxx_std_14) + +target_include_directories(grpc++ + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(grpc++ + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc + ${_gRPC_PROTOBUF_LIBRARIES} +) + +add_library(grpc++_unsecure + ${_gRPC_SOURCE_DIR}/src/cpp/client/channel_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_callback.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_interceptor.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_stats_interceptor.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel_internal.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel_posix.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/insecure_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/alarm.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/channel_arguments.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/channel_filter.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/completion_queue_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/insecure_create_auth_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/resource_quota_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/rpc_method.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/validate_service_config.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/version_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/async_generic_service.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/backend_metric_recorder.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/channel_argument_option.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/create_default_thread_pool.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/external_connection_acceptor_impl.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/health/default_health_check_service.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/health/health_check_service.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/health/health_check_service_server_builder_option.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/insecure_server_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_builder.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_callback.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_posix.cc + ${_gRPC_SOURCE_DIR}/src/cpp/thread_manager/thread_manager.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/byte_buffer_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/status.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/string_ref.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/time_cc.cc + ${gRPC_UPB_GEN_DUPL_SRC} +) + +target_compile_features(grpc++_unsecure PUBLIC cxx_std_14) + +target_include_directories(grpc++_unsecure + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(grpc++_unsecure + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_unsecure + ${_gRPC_PROTOBUF_LIBRARIES} +) + +add_library(grpc_plugin_support + ${_gRPC_SOURCE_DIR}/src/compiler/cpp_generator.cc + ${_gRPC_SOURCE_DIR}/src/compiler/csharp_generator.cc + ${_gRPC_SOURCE_DIR}/src/compiler/node_generator.cc + ${_gRPC_SOURCE_DIR}/src/compiler/objective_c_generator.cc + ${_gRPC_SOURCE_DIR}/src/compiler/php_generator.cc + ${_gRPC_SOURCE_DIR}/src/compiler/proto_parser_helper.cc + ${_gRPC_SOURCE_DIR}/src/compiler/python_generator.cc + ${_gRPC_SOURCE_DIR}/src/compiler/ruby_generator.cc +) + +target_compile_features(grpc_plugin_support PUBLIC cxx_std_14) + +target_include_directories(grpc_plugin_support + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(grpc_plugin_support + ${_gRPC_ALLTARGETS_LIBRARIES} + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_PROTOBUF_PROTOC_LIBRARIES} +) + + +add_executable(grpc_cpp_plugin + ${_gRPC_SOURCE_DIR}/src/compiler/cpp_plugin.cc +) +target_compile_features(grpc_cpp_plugin PUBLIC cxx_std_14) +target_include_directories(grpc_cpp_plugin + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_SOURCE_DIR}/include + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) + +target_link_libraries(grpc_cpp_plugin + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_plugin_support +) From 3213443ee2a53699f341b72916ca6011fe6f87cd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Nov 2023 15:42:35 +0100 Subject: [PATCH 177/192] Own CMake for GRPC --- cmake/limit_jobs.cmake | 4 ++-- contrib/grpc-cmake/grpc.cmake | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index a43e208ff0d..9d693f65528 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -21,7 +21,7 @@ if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY) set (PARALLEL_COMPILE_JOBS 1) endif () if (PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES) - message(INFO "The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.") + message(INFORMATION "The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.") endif() endif () @@ -32,7 +32,7 @@ if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY) set (PARALLEL_LINK_JOBS 1) endif () if (PARALLEL_LINK_JOBS LESS NUMBER_OF_LOGICAL_CORES) - message(INFO "The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.") + message(INFORMATION "The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.") endif() endif () diff --git a/contrib/grpc-cmake/grpc.cmake b/contrib/grpc-cmake/grpc.cmake index 43d4edd191e..c2488539211 100644 --- a/contrib/grpc-cmake/grpc.cmake +++ b/contrib/grpc-cmake/grpc.cmake @@ -55,16 +55,6 @@ if(UNIX) endif() endif() -if(UNIX AND NOT HAIKU) - # -pthread does more than -lpthread - set(THREADS_PREFER_PTHREAD_FLAG ON) - find_package(Threads) - set(_gRPC_ALLTARGETS_LIBRARIES ${CMAKE_DL_LIBS} Threads::Threads) - if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) - set(_gRPC_ALLTARGETS_LIBRARIES ${_gRPC_ALLTARGETS_LIBRARIES} rt) - endif() -endif() - set(_gRPC_ADDRESS_SORTING_INCLUDE_DIR "${_gRPC_SOURCE_DIR}/third_party/address_sorting/include") set(_gRPC_ADDRESS_SORTING_LIBRARIES address_sorting) From 907f168e0dea4df5ac305fecf63c9c9a3820e433 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Nov 2023 15:43:09 +0100 Subject: [PATCH 178/192] Own CMake for GRPC --- cmake/limit_jobs.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index 9d693f65528..8e48fc9b9d8 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -21,7 +21,7 @@ if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY) set (PARALLEL_COMPILE_JOBS 1) endif () if (PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES) - message(INFORMATION "The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.") + message("The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.") endif() endif () @@ -32,7 +32,7 @@ if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY) set (PARALLEL_LINK_JOBS 1) endif () if (PARALLEL_LINK_JOBS LESS NUMBER_OF_LOGICAL_CORES) - message(INFORMATION "The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.") + message("The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.") endif() endif () From 60a17ee397bf98e07ece79bef06d6b025bc0dfe0 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 19 Nov 2023 15:27:59 +0000 Subject: [PATCH 179/192] Fix build --- src/Backups/BackupIO_S3.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 76b9fba7b83..ea3f57c27ff 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -253,7 +253,6 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, { LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination); copyS3File( - client, client, /* src_bucket */ s3_uri.bucket, /* src_key= */ fs::path(s3_uri.key) / source, From 4fc658cd1fe751e082ed4cf5bbda581c1b5a145a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Nov 2023 16:31:03 +0100 Subject: [PATCH 180/192] Fix build --- src/Backups/BackupIO_S3.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 76b9fba7b83..ea3f57c27ff 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -253,7 +253,6 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, { LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination); copyS3File( - client, client, /* src_bucket */ s3_uri.bucket, /* src_key= */ fs::path(s3_uri.key) / source, From 62a87665c551f4efa9ae5b98000604e64590e79e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Nov 2023 16:31:03 +0100 Subject: [PATCH 181/192] Fix build --- src/Backups/BackupIO_S3.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 76b9fba7b83..ea3f57c27ff 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -253,7 +253,6 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, { LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination); copyS3File( - client, client, /* src_bucket */ s3_uri.bucket, /* src_key= */ fs::path(s3_uri.key) / source, From a3c9f13ac916ddecf74fba96d7c85cfe9c3f7468 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 19 Nov 2023 15:33:58 +0000 Subject: [PATCH 182/192] Add exclude for tryBase64Decode to backward compat test (follow-up to #56913) Fixes #56969 --- .../integration/test_backward_compatibility/test_functions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_backward_compatibility/test_functions.py b/tests/integration/test_backward_compatibility/test_functions.py index 94771a624e2..b6b6ef28de5 100644 --- a/tests/integration/test_backward_compatibility/test_functions.py +++ b/tests/integration/test_backward_compatibility/test_functions.py @@ -153,6 +153,9 @@ def test_string_functions(start_cluster): # mandatory or optional). The former lib produces a value based on implicit padding, the latter lib throws an error. "FROM_BASE64", "base64Decode", + # PR #56913 (in v23.11) corrected the way tryBase64Decode() behaved with invalid inputs. Old versions return garbage, new versions + # return an empty string (as it was always documented). + "tryBase64Decode", # Removed in 23.9 "meiliMatch", ] From 0513c93829d8c6d5f3226d98593b325d2d68f9df Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sun, 19 Nov 2023 23:11:05 +0100 Subject: [PATCH 183/192] Prefer sccache to ccache by default --- cmake/ccache.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/ccache.cmake b/cmake/ccache.cmake index e8bf856332a..0df70d82d2c 100644 --- a/cmake/ccache.cmake +++ b/cmake/ccache.cmake @@ -9,10 +9,10 @@ if (CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" OR CMAKE_C_COMPILER_LAUNCHER MA return() endif() -set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the caching tools; valid options are 'auto' (ccache, then sccache), 'ccache', 'sccache', or 'disabled'") +set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the caching tools; valid options are 'auto' (sccache, then ccache), 'ccache', 'sccache', or 'disabled'") if(COMPILER_CACHE STREQUAL "auto") - find_program (CCACHE_EXECUTABLE NAMES ccache sccache) + find_program (CCACHE_EXECUTABLE NAMES sccache ccache) elseif (COMPILER_CACHE STREQUAL "ccache") find_program (CCACHE_EXECUTABLE ccache) elseif(COMPILER_CACHE STREQUAL "sccache") @@ -21,7 +21,7 @@ elseif(COMPILER_CACHE STREQUAL "disabled") message(STATUS "Using *ccache: no (disabled via configuration)") return() else() - message(${RECONFIGURE_MESSAGE_LEVEL} "The COMPILER_CACHE must be one of (auto|ccache|sccache|disabled), value: '${COMPILER_CACHE}'") + message(${RECONFIGURE_MESSAGE_LEVEL} "The COMPILER_CACHE must be one of (auto|sccache|ccache|disabled), value: '${COMPILER_CACHE}'") endif() From 33df68cd0149a901812cfb0276a097c042ed800f Mon Sep 17 00:00:00 2001 From: Bharat Nallan Chakravarthy Date: Sun, 19 Nov 2023 14:48:21 -0800 Subject: [PATCH 184/192] update 02003_memory_limit_in_client.sh --- tests/queries/0_stateless/02003_memory_limit_in_client.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02003_memory_limit_in_client.sh b/tests/queries/0_stateless/02003_memory_limit_in_client.sh index 2d2493828c8..4017c3771a6 100755 --- a/tests/queries/0_stateless/02003_memory_limit_in_client.sh +++ b/tests/queries/0_stateless/02003_memory_limit_in_client.sh @@ -1,4 +1,4 @@ -#!/usr/bin/bash -f +#!/usr/bin/env bash CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From f6e4c29669f85b15ae6c84938d5e5cc8ca2628d1 Mon Sep 17 00:00:00 2001 From: Peignon Melvyn Date: Mon, 20 Nov 2023 01:49:17 +0100 Subject: [PATCH 185/192] MaterializedMysql doc Add experimental flag for materializedMysql --- docs/en/engines/database-engines/materialized-mysql.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/en/engines/database-engines/materialized-mysql.md b/docs/en/engines/database-engines/materialized-mysql.md index b7e567c7b6c..f32698f84f6 100644 --- a/docs/en/engines/database-engines/materialized-mysql.md +++ b/docs/en/engines/database-engines/materialized-mysql.md @@ -7,7 +7,10 @@ sidebar_position: 70 # [experimental] MaterializedMySQL :::note -This is an experimental feature that should not be used in production. +This database engine is experimental. To use it, set `allow_experimental_database_materialized_mysql` to 1 in your configuration files or by using the `SET` command: +```sql +SET allow_experimental_database_materialized_mysql=1 +``` ::: Creates a ClickHouse database with all the tables existing in MySQL, and all the data in those tables. The ClickHouse server works as MySQL replica. It reads `binlog` and performs DDL and DML queries. From 96c603ef97ce63a617637b0c1f45053ce0b5899c Mon Sep 17 00:00:00 2001 From: Chuan-Zheng Lee Date: Mon, 20 Nov 2023 17:17:27 +1300 Subject: [PATCH 186/192] [Docs] MaterializedPostgreSQL: Change DETACH to DETACH PERMANENTLY If I'm not mistaken, ClickHouse/ClickHouse#35158 changed the syntax for dynamically removing tables from MaterializedPostgreSQL databases from `DETACH` to `DETACH PERMANENTLY`. Currently when just running `DETACH TABLE postgres_database.table_to_remove`, it shows an error: ``` DETACH TABLE not allowed, use DETACH PERMANENTLY. (NOT_IMPLEMENTED) ``` This adds the keyword `PERMANENTLY` to both places where `DETACH` occurs on the MaterializedPostgreSQL database engine page. --- docs/en/engines/database-engines/materialized-postgresql.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/engines/database-engines/materialized-postgresql.md b/docs/en/engines/database-engines/materialized-postgresql.md index 4e978947e36..3aa6dd01ea3 100644 --- a/docs/en/engines/database-engines/materialized-postgresql.md +++ b/docs/en/engines/database-engines/materialized-postgresql.md @@ -8,7 +8,7 @@ sidebar_position: 60 Creates a ClickHouse database with tables from PostgreSQL database. Firstly, database with engine `MaterializedPostgreSQL` creates a snapshot of PostgreSQL database and loads required tables. Required tables can include any subset of tables from any subset of schemas from specified database. Along with the snapshot database engine acquires LSN and once initial dump of tables is performed - it starts pulling updates from WAL. After database is created, newly added tables to PostgreSQL database are not automatically added to replication. They have to be added manually with `ATTACH TABLE db.table` query. -Replication is implemented with PostgreSQL Logical Replication Protocol, which does not allow to replicate DDL, but allows to know whether replication breaking changes happened (column type changes, adding/removing columns). Such changes are detected and according tables stop receiving updates. In this case you should use `ATTACH`/ `DETACH` queries to reload table completely. If DDL does not break replication (for example, renaming a column) table will still receive updates (insertion is done by position). +Replication is implemented with PostgreSQL Logical Replication Protocol, which does not allow to replicate DDL, but allows to know whether replication breaking changes happened (column type changes, adding/removing columns). Such changes are detected and according tables stop receiving updates. In this case you should use `ATTACH`/ `DETACH PERMANENTLY` queries to reload table completely. If DDL does not break replication (for example, renaming a column) table will still receive updates (insertion is done by position). :::note This database engine is experimental. To use it, set `allow_experimental_database_materialized_postgresql` to 1 in your configuration files or by using the `SET` command: @@ -63,7 +63,7 @@ Before version 22.1, adding a table to replication left a non-removed temporary It is possible to remove specific tables from replication: ``` sql -DETACH TABLE postgres_database.table_to_remove; +DETACH TABLE postgres_database.table_to_remove PERMANENTLY; ``` ## PostgreSQL schema {#schema} From f9a8df4296b9c4cc657cb14d6cd2cfaa84d1a6ba Mon Sep 17 00:00:00 2001 From: Aleksandr Musorin Date: Thu, 16 Nov 2023 13:57:58 +0100 Subject: [PATCH 187/192] Added comment to prevent using --remote to update submodules --- contrib/update-submodules.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/update-submodules.sh b/contrib/update-submodules.sh index b612d25352b..6d187f124a4 100755 --- a/contrib/update-submodules.sh +++ b/contrib/update-submodules.sh @@ -9,4 +9,8 @@ cd $GIT_DIR contrib/sparse-checkout/setup-sparse-checkout.sh git submodule init git submodule sync -git config --file .gitmodules --get-regexp .*path | sed 's/[^ ]* //' | xargs -I _ --max-procs 64 git submodule update --depth=1 --single-branch _ +# NOTE: do not use --remote for `git submodule update`[1] command, since the submodule references to the specific commit SHA1 in the subproject. +# It may cause unexpected behavior. Instead you need to commit a new SHA1 for a submodule. +# +# [1] - https://git-scm.com/book/en/v2/Git-Tools-Submodules +git config --file .gitmodules --get-regexp '.*path' | sed 's/[^ ]* //' | xargs -I _ --max-procs 64 git submodule update --depth=1 --single-branch _ From c3a3cf8d24d5ca1205c2fe9cb2d8d266ea276b00 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 20 Nov 2023 12:57:10 +0100 Subject: [PATCH 188/192] Make check for the limited cmake dependencies the part of sparse checkout --- contrib/update-submodules.sh | 8 ++++++++ docker/packager/binary/build.sh | 9 --------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/contrib/update-submodules.sh b/contrib/update-submodules.sh index 6d187f124a4..b12f3f924dc 100755 --- a/contrib/update-submodules.sh +++ b/contrib/update-submodules.sh @@ -14,3 +14,11 @@ git submodule sync # # [1] - https://git-scm.com/book/en/v2/Git-Tools-Submodules git config --file .gitmodules --get-regexp '.*path' | sed 's/[^ ]* //' | xargs -I _ --max-procs 64 git submodule update --depth=1 --single-branch _ + +# We don't want to depend on any third-party CMake files. +# To check it, find and delete them. +grep -o -P '"contrib/[^"]+"' .gitmodules | + grep -v -P 'contrib/(llvm-project|google-protobuf|grpc|abseil-cpp|corrosion)' | + xargs -I@ find @ \ + -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' \ + -delete diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index f943011df9d..fd9bfcaabb2 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -34,15 +34,6 @@ cd /build/build_docker rm -f CMakeCache.txt -# We don't want to depend on any third-party CMake files. -# To check it, find and delete them. - -grep -o -P '"contrib/[^"]+"' ../.gitmodules | - grep -v -P 'llvm-project|google-protobuf|grpc|abseil-cpp|corrosion' | - xargs -I@ find ../@ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | - xargs rm - - if [ -n "$MAKE_DEB" ]; then rm -rf /build/packages/root # NOTE: this is for backward compatibility with previous releases, From 779a8971e05cfbc0dc43aecb8e25e98a1f8c94c6 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 20 Nov 2023 13:33:39 +0100 Subject: [PATCH 189/192] Disable settings randomisation for `02896_memory_accounting_for_user.sh` (#56709) --- tests/queries/0_stateless/02896_memory_accounting_for_user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02896_memory_accounting_for_user.sh b/tests/queries/0_stateless/02896_memory_accounting_for_user.sh index 72f4be1475d..f3016671420 100755 --- a/tests/queries/0_stateless/02896_memory_accounting_for_user.sh +++ b/tests/queries/0_stateless/02896_memory_accounting_for_user.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-parallel, long +# Tags: no-parallel, long, no-random-settings CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From 5031f239c3ecb92d6da853ae2e14b82eadc0ba43 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 20 Nov 2023 14:28:59 +0100 Subject: [PATCH 190/192] Revert "s3 adaptive timeouts" --- base/poco/Net/src/HTTPServerSession.cpp | 1 + base/poco/Net/src/HTTPSession.cpp | 31 +---- docs/en/operations/settings/settings.md | 7 - src/Backups/BackupIO_S3.cpp | 9 +- src/Coordination/KeeperSnapshotManagerS3.cpp | 1 + src/Core/Settings.h | 3 +- .../ObjectStorages/S3/S3ObjectStorage.cpp | 58 +++++--- src/Disks/ObjectStorages/S3/S3ObjectStorage.h | 15 +- src/Disks/ObjectStorages/S3/diskSettings.cpp | 8 +- src/IO/ConnectionTimeouts.cpp | 82 ----------- src/IO/ConnectionTimeouts.h | 2 - src/IO/HTTPCommon.cpp | 12 +- src/IO/HTTPCommon.h | 2 - src/IO/ReadBufferFromS3.cpp | 24 ++-- src/IO/ReadBufferFromS3.h | 4 +- src/IO/S3/Client.cpp | 12 +- src/IO/S3/Client.h | 10 +- src/IO/S3/PocoHTTPClient.cpp | 111 ++++----------- src/IO/S3/PocoHTTPClient.h | 5 - src/IO/S3/copyS3File.cpp | 26 ++-- src/IO/S3/copyS3File.h | 7 + src/IO/S3/tests/gtest_aws_s3_client.cpp | 3 +- src/IO/WriteBufferFromS3.cpp | 4 +- src/IO/WriteBufferFromS3.h | 3 + src/IO/tests/gtest_writebuffer_s3.cpp | 1 + src/Storages/StorageS3.cpp | 3 + src/Storages/StorageS3.h | 1 + src/Storages/StorageS3Settings.h | 3 +- .../configs/inf_s3_retries.xml | 1 - .../configs/s3_retries.xml | 1 - .../configs/storage_conf.xml | 16 +-- .../test_checking_s3_blobs_paranoid/test.py | 129 ++++-------------- .../configs/config.d/storage_conf.xml | 2 - .../test_storage_s3/configs/defaultS3.xml | 5 + .../test_storage_s3/configs/s3_retry.xml | 4 +- .../s3_mocks/unstable_server.py | 17 +-- tests/integration/test_storage_s3/test.py | 9 -- 37 files changed, 202 insertions(+), 430 deletions(-) diff --git a/base/poco/Net/src/HTTPServerSession.cpp b/base/poco/Net/src/HTTPServerSession.cpp index d4f2b24879e..f6d3c4e5b92 100644 --- a/base/poco/Net/src/HTTPServerSession.cpp +++ b/base/poco/Net/src/HTTPServerSession.cpp @@ -26,6 +26,7 @@ HTTPServerSession::HTTPServerSession(const StreamSocket& socket, HTTPServerParam _maxKeepAliveRequests(pParams->getMaxKeepAliveRequests()) { setTimeout(pParams->getTimeout()); + this->socket().setReceiveTimeout(pParams->getTimeout()); } diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index 8f951b3102c..d2663baaf9f 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -93,34 +93,9 @@ void HTTPSession::setTimeout(const Poco::Timespan& timeout) void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco::Timespan& sendTimeout, const Poco::Timespan& receiveTimeout) { - try - { - _connectionTimeout = connectionTimeout; - - if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) { - _sendTimeout = sendTimeout; - - if (connected()) - _socket.setSendTimeout(_sendTimeout); - } - - if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) { - _receiveTimeout = receiveTimeout; - - if (connected()) - _socket.setReceiveTimeout(_receiveTimeout); - } - } - catch (NetException &) - { -#ifndef NDEBUG - throw; -#else - // mute exceptions in release - // just in case when changing settings on socket is not allowed - // however it should be OK for timeouts -#endif - } + _connectionTimeout = connectionTimeout; + _sendTimeout = sendTimeout; + _receiveTimeout = receiveTimeout; } diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index edc1c9bdfd7..e61934d2168 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -4826,10 +4826,3 @@ When set to `true` the metadata files are written with `VERSION_FULL_OBJECT_KEY` When set to `false` the metadata files are written with the previous format version, `VERSION_INLINE_DATA`. With that format only suffixes of object storage key names are are written to the metadata files. The prefix for all of object storage key names is set in configurations files at `storage_configuration.disks` section. Default value: `false`. - -## s3_use_adaptive_timeouts {#s3_use_adaptive_timeouts} - -When set to `true` than for all s3 requests first two attempts are made with low send and receive timeouts. -When set to `false` than all attempts are made with identical timeouts. - -Default value: `true`. diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index ea3f57c27ff..eb9dcf6b45a 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -55,9 +55,7 @@ namespace static_cast(context->getGlobalContext()->getSettingsRef().s3_max_redirects), static_cast(context->getGlobalContext()->getSettingsRef().s3_retry_attempts), context->getGlobalContext()->getSettingsRef().enable_s3_requests_logging, - /* for_disk_s3 = */ false, - request_settings.get_request_throttler, - request_settings.put_request_throttler, + /* for_disk_s3 = */ false, request_settings.get_request_throttler, request_settings.put_request_throttler, s3_uri.uri.getScheme()); client_configuration.endpointOverride = s3_uri.endpoint; @@ -169,6 +167,7 @@ void BackupReaderS3::copyFileToDisk(const String & path_in_backup, size_t file_s blob_path.size(), mode); copyS3File( + client, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, @@ -230,6 +229,7 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src { LOG_TRACE(log, "Copying file {} from disk {} to S3", src_path, src_disk->getName()); copyS3File( + client, client, /* src_bucket */ blob_path[1], /* src_key= */ blob_path[0], @@ -268,7 +268,7 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, void BackupWriterS3::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) { - copyDataToS3File(create_read_buffer, start_pos, length, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {}, + copyDataToS3File(create_read_buffer, start_pos, length, client, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {}, threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterS3")); } @@ -298,6 +298,7 @@ std::unique_ptr BackupWriterS3::writeFile(const String & file_name) { return std::make_unique( client, + client, // already has long timeout s3_uri.bucket, fs::path(s3_uri.key) / file_name, DBMS_DEFAULT_BUFFER_SIZE, diff --git a/src/Coordination/KeeperSnapshotManagerS3.cpp b/src/Coordination/KeeperSnapshotManagerS3.cpp index bedde0d7b39..302e05c8418 100644 --- a/src/Coordination/KeeperSnapshotManagerS3.cpp +++ b/src/Coordination/KeeperSnapshotManagerS3.cpp @@ -148,6 +148,7 @@ void KeeperSnapshotManagerS3::uploadSnapshotImpl(const SnapshotFileInfo & snapsh const auto create_writer = [&](const auto & key) { return WriteBufferFromS3( + s3_client->client, s3_client->client, s3_client->uri.bucket, key, diff --git a/src/Core/Settings.h b/src/Core/Settings.h index bb5e4322485..ac4c6b6c17f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -94,7 +94,6 @@ class IColumn; M(UInt64, s3_max_put_rps, 0, "Limit on S3 PUT request per second rate before throttling. Zero means unlimited.", 0) \ M(UInt64, s3_max_put_burst, 0, "Max number of requests that can be issued simultaneously before hitting request per second limit. By default (0) equals to `s3_max_put_rps`", 0) \ M(UInt64, s3_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ - M(Bool, s3_use_adaptive_timeouts, true, "When adaptive timeouts are enabled first two attempts are made with low receive and send timeout", 0) \ M(UInt64, azure_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ M(Bool, s3_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables.", 0) \ M(Bool, azure_truncate_on_insert, false, "Enables or disables truncate before insert in azure engine tables.", 0) \ @@ -105,7 +104,7 @@ class IColumn; M(Bool, s3_allow_parallel_part_upload, true, "Use multiple threads for s3 multipart upload. It may lead to slightly higher memory usage", 0) \ M(Bool, s3_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \ M(UInt64, s3_retry_attempts, 100, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries", 0) \ - M(UInt64, s3_request_timeout_ms, 30000, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \ + M(UInt64, s3_request_timeout_ms, 3000, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \ M(UInt64, s3_http_connection_pool_size, 1000, "How many reusable open connections to keep per S3 endpoint. Only applies to the S3 table engine and table function, not to S3 disks (for disks, use disk config instead). Global setting, can only be set in config, overriding it per session or per query has no effect.", 0) \ M(Bool, enable_s3_requests_logging, false, "Enable very explicit logging of S3 requests. Makes sense for debug only.", 0) \ M(String, s3queue_default_zookeeper_path, "/clickhouse/s3queue/", "Default zookeeper path prefix for S3Queue engine", 0) \ diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 308db389ee1..3af316bf0cf 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -155,7 +155,7 @@ private: bool S3ObjectStorage::exists(const StoredObject & object) const { auto settings_ptr = s3_settings.get(); - return S3::objectExists(*client.get(), bucket, object.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + return S3::objectExists(*clients.get()->client, bucket, object.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); } std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT @@ -174,7 +174,7 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT (const std::string & path, size_t read_until_position) -> std::unique_ptr { return std::make_unique( - client.get(), + clients.get()->client, bucket, path, version_id, @@ -224,7 +224,7 @@ std::unique_ptr S3ObjectStorage::readObject( /// NOLINT { auto settings_ptr = s3_settings.get(); return std::make_unique( - client.get(), + clients.get()->client, bucket, object.remote_path, version_id, @@ -249,8 +249,10 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN if (write_settings.s3_allow_parallel_part_upload) scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "VFSWrite"); + auto clients_ = clients.get(); return std::make_unique( - client.get(), + clients_->client, + clients_->client_with_long_timeout, bucket, object.remote_path, buf_size, @@ -264,12 +266,15 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN ObjectStorageIteratorPtr S3ObjectStorage::iterate(const std::string & path_prefix) const { auto settings_ptr = s3_settings.get(); - return std::make_shared(bucket, path_prefix, client.get(), settings_ptr->list_object_keys_size); + auto client_ptr = clients.get()->client; + + return std::make_shared(bucket, path_prefix, client_ptr, settings_ptr->list_object_keys_size); } void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const { auto settings_ptr = s3_settings.get(); + auto client_ptr = clients.get()->client; S3::ListObjectsV2Request request; request.SetBucket(bucket); @@ -284,7 +289,7 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet { ProfileEvents::increment(ProfileEvents::S3ListObjects); ProfileEvents::increment(ProfileEvents::DiskS3ListObjects); - outcome = client.get()->ListObjectsV2(request); + outcome = client_ptr->ListObjectsV2(request); throwIfError(outcome); auto result = outcome.GetResult(); @@ -315,12 +320,14 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exists) { + auto client_ptr = clients.get()->client; + ProfileEvents::increment(ProfileEvents::S3DeleteObjects); ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); S3::DeleteObjectRequest request; request.SetBucket(bucket); request.SetKey(object.remote_path); - auto outcome = client.get()->DeleteObject(request); + auto outcome = client_ptr->DeleteObject(request); throwIfUnexpectedError(outcome, if_exists); @@ -339,6 +346,7 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e } else { + auto client_ptr = clients.get()->client; auto settings_ptr = s3_settings.get(); size_t chunk_size_limit = settings_ptr->objects_chunk_size_to_delete; @@ -367,7 +375,7 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e S3::DeleteObjectsRequest request; request.SetBucket(bucket); request.SetDelete(delkeys); - auto outcome = client.get()->DeleteObjects(request); + auto outcome = client_ptr->DeleteObjects(request); throwIfUnexpectedError(outcome, if_exists); @@ -399,7 +407,7 @@ void S3ObjectStorage::removeObjectsIfExist(const StoredObjects & objects) std::optional S3ObjectStorage::tryGetObjectMetadata(const std::string & path) const { auto settings_ptr = s3_settings.get(); - auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true, /* throw_on_error= */ false); + auto object_info = S3::getObjectInfo(*clients.get()->client, bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true, /* throw_on_error= */ false); if (object_info.size == 0 && object_info.last_modification_time == 0 && object_info.metadata.empty()) return {}; @@ -415,7 +423,7 @@ std::optional S3ObjectStorage::tryGetObjectMetadata(const std::s ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path) const { auto settings_ptr = s3_settings.get(); - auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true); + auto object_info = S3::getObjectInfo(*clients.get()->client, bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true); ObjectMetadata result; result.size_bytes = object_info.size; @@ -436,12 +444,12 @@ void S3ObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT /// Shortcut for S3 if (auto * dest_s3 = dynamic_cast(&object_storage_to); dest_s3 != nullptr) { - auto client_ = client.get(); + auto clients_ = clients.get(); auto settings_ptr = s3_settings.get(); - auto size = S3::getObjectSize(*client_, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + auto size = S3::getObjectSize(*clients_->client, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); auto scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "S3ObjStor_copy"); - copyS3File( - client.get(), + copyS3File(clients_->client, + clients_->client_with_long_timeout, bucket, object_from.remote_path, 0, @@ -465,11 +473,12 @@ void S3ObjectStorage::copyObject( // NOLINT const WriteSettings &, std::optional object_to_attributes) { - auto client_ = client.get(); + auto clients_ = clients.get(); auto settings_ptr = s3_settings.get(); - auto size = S3::getObjectSize(*client_, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + auto size = S3::getObjectSize(*clients_->client, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); auto scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "S3ObjStor_copy"); - copyS3File(client_, + copyS3File(clients_->client, + clients_->client_with_long_timeout, bucket, object_from.remote_path, 0, @@ -490,25 +499,31 @@ void S3ObjectStorage::setNewSettings(std::unique_ptr && void S3ObjectStorage::shutdown() { + auto clients_ptr = clients.get(); /// This call stops any next retry attempts for ongoing S3 requests. /// If S3 request is failed and the method below is executed S3 client immediately returns the last failed S3 request outcome. /// If S3 is healthy nothing wrong will be happened and S3 requests will be processed in a regular way without errors. /// This should significantly speed up shutdown process if S3 is unhealthy. - const_cast(*client.get()).DisableRequestProcessing(); + const_cast(*clients_ptr->client).DisableRequestProcessing(); + const_cast(*clients_ptr->client_with_long_timeout).DisableRequestProcessing(); } void S3ObjectStorage::startup() { + auto clients_ptr = clients.get(); + /// Need to be enabled if it was disabled during shutdown() call. - const_cast(*client.get()).EnableRequestProcessing(); + const_cast(*clients_ptr->client).EnableRequestProcessing(); + const_cast(*clients_ptr->client_with_long_timeout).EnableRequestProcessing(); } void S3ObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) { auto new_s3_settings = getSettings(config, config_prefix, context); auto new_client = getClient(config, config_prefix, context, *new_s3_settings); + auto new_clients = std::make_unique(std::move(new_client), *new_s3_settings); s3_settings.set(std::move(new_s3_settings)); - client.set(std::move(new_client)); + clients.set(std::move(new_clients)); } std::unique_ptr S3ObjectStorage::cloneObjectStorage( @@ -523,6 +538,9 @@ std::unique_ptr S3ObjectStorage::cloneObjectStorage( endpoint, object_key_prefix); } +S3ObjectStorage::Clients::Clients(std::shared_ptr client_, const S3ObjectStorageSettings & settings) + : client(std::move(client_)), client_with_long_timeout(client->clone(std::nullopt, settings.request_settings.long_request_timeout_ms)) {} + ObjectStorageKey S3ObjectStorage::generateObjectKeyForPath(const std::string &) const { /// Path to store the new S3 object. diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index 7d14482311f..b1b3fb22366 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -39,6 +39,16 @@ struct S3ObjectStorageSettings class S3ObjectStorage : public IObjectStorage { +public: + struct Clients + { + std::shared_ptr client; + std::shared_ptr client_with_long_timeout; + + Clients() = default; + Clients(std::shared_ptr client, const S3ObjectStorageSettings & settings); + }; + private: friend class S3PlainObjectStorage; @@ -53,7 +63,7 @@ private: String object_key_prefix_) : bucket(std::move(bucket_)) , object_key_prefix(std::move(object_key_prefix_)) - , client(std::move(client_)) + , clients(std::make_unique(std::move(client_), *s3_settings_)) , s3_settings(std::move(s3_settings_)) , s3_capabilities(s3_capabilities_) , version_id(std::move(version_id_)) @@ -174,8 +184,7 @@ private: std::string bucket; String object_key_prefix; - - MultiVersion client; + MultiVersion clients; MultiVersion s3_settings; S3Capabilities s3_capabilities; diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 0232a6eb070..de88c876922 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -60,15 +60,13 @@ std::unique_ptr getClient( uri.uri.getScheme()); client_configuration.connectTimeoutMs = config.getUInt(config_prefix + ".connect_timeout_ms", 1000); - client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 30000); + client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 3000); client_configuration.maxConnections = config.getUInt(config_prefix + ".max_connections", 100); client_configuration.endpointOverride = uri.endpoint; - client_configuration.http_keep_alive_timeout_ms = config.getUInt( - config_prefix + ".http_keep_alive_timeout_ms", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT * 1000); + client_configuration.http_keep_alive_timeout_ms + = config.getUInt(config_prefix + ".http_keep_alive_timeout_ms", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT * 1000); client_configuration.http_connection_pool_size = config.getUInt(config_prefix + ".http_connection_pool_size", 1000); client_configuration.wait_on_pool_size_limit = false; - client_configuration.s3_use_adaptive_timeouts = config.getBool( - config_prefix + ".use_adaptive_timeouts", client_configuration.s3_use_adaptive_timeouts); /* * Override proxy configuration for backwards compatibility with old configuration format. diff --git a/src/IO/ConnectionTimeouts.cpp b/src/IO/ConnectionTimeouts.cpp index 970afc75ec3..01fbaa4f817 100644 --- a/src/IO/ConnectionTimeouts.cpp +++ b/src/IO/ConnectionTimeouts.cpp @@ -133,86 +133,4 @@ ConnectionTimeouts ConnectionTimeouts::getHTTPTimeouts(const Settings & settings settings.http_receive_timeout); } -class SendReceiveTimeoutsForFirstAttempt -{ -private: - static constexpr size_t known_methods_count = 6; - using KnownMethodsArray = std::array; - static const KnownMethodsArray known_methods; - - /// HTTP_POST is used for CompleteMultipartUpload requests. Its latency could be high. - /// These requests need longer timeout, especially when minio is used. - /// The same assumption are made for HTTP_DELETE, HTTP_PATCH - /// That requests are more heavy that HTTP_GET, HTTP_HEAD, HTTP_PUT - - static constexpr Poco::Timestamp::TimeDiff first_byte_ms[known_methods_count][2] = - { - /* GET */ {200, 200}, - /* POST */ {200, 200}, - /* DELETE */ {200, 200}, - /* PUT */ {200, 200}, - /* HEAD */ {200, 200}, - /* PATCH */ {200, 200}, - }; - - static constexpr Poco::Timestamp::TimeDiff rest_bytes_ms[known_methods_count][2] = - { - /* GET */ {500, 500}, - /* POST */ {1000, 30000}, - /* DELETE */ {1000, 10000}, - /* PUT */ {1000, 3000}, - /* HEAD */ {500, 500}, - /* PATCH */ {1000, 10000}, - }; - - static_assert(sizeof(first_byte_ms) == sizeof(rest_bytes_ms)); - static_assert(sizeof(first_byte_ms) == known_methods_count * sizeof(Poco::Timestamp::TimeDiff) * 2); - - static size_t getMethodIndex(const String & method) - { - KnownMethodsArray::const_iterator it = std::find(known_methods.begin(), known_methods.end(), method); - chassert(it != known_methods.end()); - if (it == known_methods.end()) - return 0; - return std::distance(known_methods.begin(), it); - } - -public: - static std::pair getSendReceiveTimeout(const String & method, bool first_byte) - { - auto idx = getMethodIndex(method); - - if (first_byte) - return std::make_pair( - Poco::Timespan(first_byte_ms[idx][0] * 1000), - Poco::Timespan(first_byte_ms[idx][1] * 1000) - ); - - return std::make_pair( - Poco::Timespan(rest_bytes_ms[idx][0] * 1000), - Poco::Timespan(rest_bytes_ms[idx][1] * 1000) - ); - } -}; - -const SendReceiveTimeoutsForFirstAttempt::KnownMethodsArray SendReceiveTimeoutsForFirstAttempt::known_methods = -{ - "GET", "POST", "DELETE", "PUT", "HEAD", "PATCH" -}; - - -ConnectionTimeouts ConnectionTimeouts::getAdaptiveTimeouts(const String & method, bool first_attempt, bool first_byte) const -{ - if (!first_attempt) - return *this; - - auto [send, recv] = SendReceiveTimeoutsForFirstAttempt::getSendReceiveTimeout(method, first_byte); - - auto aggressive = *this; - aggressive.send_timeout = saturate(send, send_timeout); - aggressive.receive_timeout = saturate(recv, receive_timeout); - - return aggressive; -} - } diff --git a/src/IO/ConnectionTimeouts.h b/src/IO/ConnectionTimeouts.h index aabebdb836d..684af42827f 100644 --- a/src/IO/ConnectionTimeouts.h +++ b/src/IO/ConnectionTimeouts.h @@ -67,8 +67,6 @@ struct ConnectionTimeouts /// Timeouts for the case when we will try many addresses in a loop. static ConnectionTimeouts getTCPTimeoutsWithFailover(const Settings & settings); static ConnectionTimeouts getHTTPTimeouts(const Settings & settings, Poco::Timespan http_keep_alive_timeout); - - ConnectionTimeouts getAdaptiveTimeouts(const String & method, bool first_attempt, bool first_byte) const; }; } diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index cce394c67c9..65ffa51a466 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -50,6 +50,12 @@ namespace ErrorCodes namespace { + void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts) + { + session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout); + session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); + } + Poco::Net::HTTPClientSession::ProxyConfig proxyConfigurationToPocoProxyConfig(const ProxyConfiguration & proxy_configuration) { Poco::Net::HTTPClientSession::ProxyConfig poco_proxy_config; @@ -353,12 +359,6 @@ namespace }; } -void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts) -{ - session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout); - session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); -} - void setResponseDefaultHeaders(HTTPServerResponse & response, size_t keep_alive_timeout) { if (!response.getKeepAlive()) diff --git a/src/IO/HTTPCommon.h b/src/IO/HTTPCommon.h index c9968fc6915..de62b5d5c16 100644 --- a/src/IO/HTTPCommon.h +++ b/src/IO/HTTPCommon.h @@ -113,6 +113,4 @@ std::istream * receiveResponse( void assertResponseIsOk( const Poco::Net::HTTPRequest & request, Poco::Net::HTTPResponse & response, std::istream & istr, bool allow_redirects = false); - -void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts); } diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index c9c9319c44c..f19978ccb47 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -167,9 +167,9 @@ bool ReadBufferFromS3::nextImpl() } size_t sleep_time_with_backoff_milliseconds = 100; - for (size_t attempt = 1; !next_result; ++attempt) + for (size_t attempt = 0; !next_result; ++attempt) { - bool last_attempt = attempt >= request_settings.max_single_read_retries; + bool last_attempt = attempt + 1 >= request_settings.max_single_read_retries; ProfileEventTimeIncrement watch(ProfileEvents::ReadBufferFromS3Microseconds); @@ -177,7 +177,7 @@ bool ReadBufferFromS3::nextImpl() { if (!impl) { - impl = initialize(attempt); + impl = initialize(); if (use_external_buffer) { @@ -232,9 +232,9 @@ size_t ReadBufferFromS3::readBigAt(char * to, size_t n, size_t range_begin, cons { size_t initial_n = n; size_t sleep_time_with_backoff_milliseconds = 100; - for (size_t attempt = 1; n > 0; ++attempt) + for (size_t attempt = 0; n > 0; ++attempt) { - bool last_attempt = attempt >= request_settings.max_single_read_retries; + bool last_attempt = attempt + 1 >= request_settings.max_single_read_retries; size_t bytes_copied = 0; ProfileEventTimeIncrement watch(ProfileEvents::ReadBufferFromS3Microseconds); @@ -266,7 +266,7 @@ size_t ReadBufferFromS3::readBigAt(char * to, size_t n, size_t range_begin, cons try { - result = sendRequest(attempt, range_begin, range_begin + n - 1); + result = sendRequest(range_begin, range_begin + n - 1); std::istream & istr = result->GetBody(); copyFromIStreamWithProgressCallback(istr, to, n, progress_callback, &bytes_copied); @@ -304,8 +304,8 @@ bool ReadBufferFromS3::processException(Poco::Exception & e, size_t read_offset, LOG_DEBUG( log, "Caught exception while reading S3 object. Bucket: {}, Key: {}, Version: {}, Offset: {}, " - "Attempt: {}/{}, Message: {}", - bucket, key, version_id.empty() ? "Latest" : version_id, read_offset, attempt, request_settings.max_single_read_retries, e.message()); + "Attempt: {}, Message: {}", + bucket, key, version_id.empty() ? "Latest" : version_id, read_offset, attempt, e.message()); if (auto * s3_exception = dynamic_cast(&e)) @@ -463,7 +463,7 @@ ReadBufferFromS3::~ReadBufferFromS3() } } -std::unique_ptr ReadBufferFromS3::initialize(size_t attempt) +std::unique_ptr ReadBufferFromS3::initialize() { resetSessionIfNeeded(readAllRangeSuccessfully(), read_result); read_all_range_successfully = false; @@ -475,13 +475,13 @@ std::unique_ptr ReadBufferFromS3::initialize(size_t attempt) if (read_until_position && offset >= read_until_position) throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read beyond right offset ({} > {})", offset, read_until_position - 1); - read_result = sendRequest(attempt, offset, read_until_position ? std::make_optional(read_until_position - 1) : std::nullopt); + read_result = sendRequest(offset, read_until_position ? std::make_optional(read_until_position - 1) : std::nullopt); size_t buffer_size = use_external_buffer ? 0 : read_settings.remote_fs_buffer_size; return std::make_unique(read_result->GetBody(), buffer_size); } -Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t attempt, size_t range_begin, std::optional range_end_incl) const +Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t range_begin, std::optional range_end_incl) const { S3::GetObjectRequest req; req.SetBucket(bucket); @@ -489,8 +489,6 @@ Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t attempt, si if (!version_id.empty()) req.SetVersionId(version_id); - req.SetAdditionalCustomHeaderValue("clickhouse-request", fmt::format("attempt={}", attempt)); - if (range_end_incl) { req.SetRange(fmt::format("bytes={}-{}", range_begin, *range_end_incl)); diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h index 101e25f8b43..0835e52a5b2 100644 --- a/src/IO/ReadBufferFromS3.h +++ b/src/IO/ReadBufferFromS3.h @@ -79,7 +79,7 @@ public: bool supportsReadAt() override { return true; } private: - std::unique_ptr initialize(size_t attempt); + std::unique_ptr initialize(); /// If true, if we destroy impl now, no work was wasted. Just for metrics. bool atEndOfRequestedRangeGuess(); @@ -88,7 +88,7 @@ private: /// Returns true if the error looks retriable. bool processException(Poco::Exception & e, size_t read_offset, size_t attempt) const; - Aws::S3::Model::GetObjectResult sendRequest(size_t attempt, size_t range_begin, std::optional range_end_incl) const; + Aws::S3::Model::GetObjectResult sendRequest(size_t range_begin, std::optional range_end_incl) const; bool readAllRangeSuccessfully() const; diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 4630e68fbb6..ceb7d275299 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -118,9 +118,16 @@ std::unique_ptr Client::create( new Client(max_redirects_, std::move(sse_kms_config_), credentials_provider, client_configuration, sign_payloads, use_virtual_addressing)); } -std::unique_ptr Client::clone() const +std::unique_ptr Client::clone( + std::optional> override_retry_strategy, + std::optional override_request_timeout_ms) const { - return std::unique_ptr(new Client(*this, client_configuration)); + PocoHTTPClientConfiguration new_configuration = client_configuration; + if (override_retry_strategy.has_value()) + new_configuration.retryStrategy = *override_retry_strategy; + if (override_request_timeout_ms.has_value()) + new_configuration.requestTimeoutMs = *override_request_timeout_ms; + return std::unique_ptr(new Client(*this, new_configuration)); } namespace @@ -898,7 +905,6 @@ PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT s3_retry_attempts, enable_s3_requests_logging, for_disk_s3, - context->getGlobalContext()->getSettingsRef().s3_use_adaptive_timeouts, get_request_throttler, put_request_throttler, error_report); diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index 5ad57a9d827..48310bc21af 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -118,7 +118,15 @@ public: Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy sign_payloads, bool use_virtual_addressing); - std::unique_ptr clone() const; + /// Create a client with adjusted settings: + /// * override_retry_strategy can be used to disable retries to avoid nested retries when we have + /// a retry loop outside of S3 client. Specifically, for read and write buffers. Currently not + /// actually used. + /// * override_request_timeout_ms is used to increase timeout for CompleteMultipartUploadRequest + /// because it often sits idle for 10 seconds: https://github.com/ClickHouse/ClickHouse/pull/42321 + std::unique_ptr clone( + std::optional> override_retry_strategy = std::nullopt, + std::optional override_request_timeout_ms = std::nullopt) const; Client & operator=(const Client &) = delete; diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 4a1b6def133..d0f248f48a6 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -99,7 +99,6 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( unsigned int s3_retry_attempts_, bool enable_s3_requests_logging_, bool for_disk_s3_, - bool s3_use_adaptive_timeouts_, const ThrottlerPtr & get_request_throttler_, const ThrottlerPtr & put_request_throttler_, std::function error_report_) @@ -112,7 +111,6 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( , for_disk_s3(for_disk_s3_) , get_request_throttler(get_request_throttler_) , put_request_throttler(put_request_throttler_) - , s3_use_adaptive_timeouts(s3_use_adaptive_timeouts_) , error_report(error_report_) { } @@ -159,7 +157,6 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & client_config Poco::Timespan(client_configuration.http_keep_alive_timeout_ms * 1000))) /// flag indicating whether keep-alive is enabled is set to each session upon creation , remote_host_filter(client_configuration.remote_host_filter) , s3_max_redirects(client_configuration.s3_max_redirects) - , s3_use_adaptive_timeouts(client_configuration.s3_use_adaptive_timeouts) , enable_s3_requests_logging(client_configuration.enable_s3_requests_logging) , for_disk_s3(client_configuration.for_disk_s3) , get_request_throttler(client_configuration.get_request_throttler) @@ -271,38 +268,6 @@ void PocoHTTPClient::addMetric(const Aws::Http::HttpRequest & request, S3MetricT ProfileEvents::increment(disk_s3_events_map[static_cast(type)][static_cast(kind)], amount); } -String extractAttemptFromInfo(const Aws::String & request_info) -{ - static auto key = Aws::String("attempt="); - - auto key_begin = request_info.find(key, 0); - if (key_begin == Aws::String::npos) - return "1"; - - auto val_begin = key_begin + key.size(); - auto val_end = request_info.find(';', val_begin); - if (val_end == Aws::String::npos) - val_end = request_info.size(); - - return request_info.substr(val_begin, val_end-val_begin); -} - -String getOrEmpty(const Aws::Http::HeaderValueCollection & map, const String & key) -{ - auto it = map.find(key); - if (it == map.end()) - return {}; - return it->second; -} - -ConnectionTimeouts PocoHTTPClient::getTimeouts(const String & method, bool first_attempt, bool first_byte) const -{ - if (!s3_use_adaptive_timeouts) - return timeouts; - - return timeouts.getAdaptiveTimeouts(method, first_attempt, first_byte); -} - void PocoHTTPClient::makeRequestInternal( Aws::Http::HttpRequest & request, std::shared_ptr & response, @@ -317,25 +282,6 @@ void PocoHTTPClient::makeRequestInternal( makeRequestInternalImpl(request, request_configuration, response, readLimiter, writeLimiter); } -String getMethod(const Aws::Http::HttpRequest & request) -{ - switch (request.GetMethod()) - { - case Aws::Http::HttpMethod::HTTP_GET: - return Poco::Net::HTTPRequest::HTTP_GET; - case Aws::Http::HttpMethod::HTTP_POST: - return Poco::Net::HTTPRequest::HTTP_POST; - case Aws::Http::HttpMethod::HTTP_DELETE: - return Poco::Net::HTTPRequest::HTTP_DELETE; - case Aws::Http::HttpMethod::HTTP_PUT: - return Poco::Net::HTTPRequest::HTTP_PUT; - case Aws::Http::HttpMethod::HTTP_HEAD: - return Poco::Net::HTTPRequest::HTTP_HEAD; - case Aws::Http::HttpMethod::HTTP_PATCH: - return Poco::Net::HTTPRequest::HTTP_PATCH; - } -} - template void PocoHTTPClient::makeRequestInternalImpl( Aws::Http::HttpRequest & request, @@ -349,14 +295,9 @@ void PocoHTTPClient::makeRequestInternalImpl( Poco::Logger * log = &Poco::Logger::get("AWSClient"); auto uri = request.GetUri().GetURIString(); - auto method = getMethod(request); - - auto sdk_attempt = extractAttemptFromInfo(getOrEmpty(request.GetHeaders(), Aws::Http::SDK_REQUEST_HEADER)); - auto ch_attempt = extractAttemptFromInfo(getOrEmpty(request.GetHeaders(), "clickhouse-request")); - bool first_attempt = ch_attempt == "1" && sdk_attempt == "1"; if (enable_s3_requests_logging) - LOG_TEST(log, "Make request to: {}, aws sdk attempt: {}, clickhouse attempt: {}", uri, sdk_attempt, ch_attempt); + LOG_TEST(log, "Make request to: {}", uri); switch (request.GetMethod()) { @@ -407,29 +348,17 @@ void PocoHTTPClient::makeRequestInternalImpl( /// This can lead to request signature difference on S3 side. if constexpr (pooled) session = makePooledHTTPSession( - target_uri, - getTimeouts(method, first_attempt, /*first_byte*/ true), - http_connection_pool_size, - wait_on_pool_size_limit, - proxy_configuration); + target_uri, timeouts, http_connection_pool_size, wait_on_pool_size_limit, proxy_configuration); else - session = makeHTTPSession( - target_uri, - getTimeouts(method, first_attempt, /*first_byte*/ true), - proxy_configuration); + session = makeHTTPSession(target_uri, timeouts, proxy_configuration); } else { if constexpr (pooled) session = makePooledHTTPSession( - target_uri, - getTimeouts(method, first_attempt, /*first_byte*/ true), - http_connection_pool_size, - wait_on_pool_size_limit); + target_uri, timeouts, http_connection_pool_size, wait_on_pool_size_limit); else - session = makeHTTPSession( - target_uri, - getTimeouts(method, first_attempt, /*first_byte*/ true)); + session = makeHTTPSession(target_uri, timeouts); } /// In case of error this address will be written to logs @@ -463,7 +392,28 @@ void PocoHTTPClient::makeRequestInternalImpl( path_and_query = "/"; poco_request.setURI(path_and_query); - poco_request.setMethod(method); + + switch (request.GetMethod()) + { + case Aws::Http::HttpMethod::HTTP_GET: + poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_GET); + break; + case Aws::Http::HttpMethod::HTTP_POST: + poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_POST); + break; + case Aws::Http::HttpMethod::HTTP_DELETE: + poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_DELETE); + break; + case Aws::Http::HttpMethod::HTTP_PUT: + poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_PUT); + break; + case Aws::Http::HttpMethod::HTTP_HEAD: + poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_HEAD); + break; + case Aws::Http::HttpMethod::HTTP_PATCH: + poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_PATCH); + break; + } /// Headers coming from SDK are lower-cased. for (const auto & [header_name, header_value] : request.GetHeaders()) @@ -488,7 +438,6 @@ void PocoHTTPClient::makeRequestInternalImpl( request.GetContentBody()->clear(); request.GetContentBody()->seekg(0); - setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); auto size = Poco::StreamCopier::copyStream(*request.GetContentBody(), request_body_stream); if (enable_s3_requests_logging) LOG_TEST(log, "Written {} bytes to request body", size); @@ -498,8 +447,6 @@ void PocoHTTPClient::makeRequestInternalImpl( LOG_TEST(log, "Receiving response..."); auto & response_body_stream = session->receiveResponse(poco_response); - setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); - watch.stop(); addMetric(request, S3MetricType::Microseconds, watch.elapsedMicroseconds()); @@ -551,7 +498,6 @@ void PocoHTTPClient::makeRequestInternalImpl( /// Request is successful but for some special requests we can have actual error message in body if (status_code >= SUCCESS_RESPONSE_MIN && status_code <= SUCCESS_RESPONSE_MAX && checkRequestCanReturn2xxAndErrorInBody(request)) { - /// reading the full response std::string response_string((std::istreambuf_iterator(response_body_stream)), std::istreambuf_iterator()); @@ -566,6 +512,7 @@ void PocoHTTPClient::makeRequestInternalImpl( addMetric(request, S3MetricType::Errors); if (error_report) error_report(proxy_configuration); + } /// Set response from string @@ -584,8 +531,6 @@ void PocoHTTPClient::makeRequestInternalImpl( if (status_code >= 500 && error_report) error_report(proxy_configuration); } - - /// expose stream, after that client reads data from that stream without built-in retries response->SetResponseBody(response_body_stream, session); } diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 5178d75e7b6..2a449458360 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -55,7 +55,6 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration size_t http_connection_pool_size = 0; /// See PoolBase::BehaviourOnLimit bool wait_on_pool_size_limit = true; - bool s3_use_adaptive_timeouts = true; std::function error_report; @@ -70,7 +69,6 @@ private: unsigned int s3_retry_attempts, bool enable_s3_requests_logging_, bool for_disk_s3_, - bool s3_use_adaptive_timeouts_, const ThrottlerPtr & get_request_throttler_, const ThrottlerPtr & put_request_throttler_, std::function error_report_ @@ -171,8 +169,6 @@ private: Aws::Utils::RateLimits::RateLimiterInterface * readLimiter, Aws::Utils::RateLimits::RateLimiterInterface * writeLimiter) const; - ConnectionTimeouts getTimeouts(const String & method, bool first_attempt, bool first_byte) const; - protected: static S3MetricKind getMetricKind(const Aws::Http::HttpRequest & request); void addMetric(const Aws::Http::HttpRequest & request, S3MetricType type, ProfileEvents::Count amount = 1) const; @@ -182,7 +178,6 @@ protected: ConnectionTimeouts timeouts; const RemoteHostFilter & remote_host_filter; unsigned int s3_max_redirects; - bool s3_use_adaptive_timeouts = true; bool enable_s3_requests_logging; bool for_disk_s3; diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index 30da1c580c1..a16a1a41505 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -53,6 +53,7 @@ namespace public: UploadHelper( const std::shared_ptr & client_ptr_, + const std::shared_ptr & client_with_long_timeout_ptr_, const String & dest_bucket_, const String & dest_key_, const S3Settings::RequestSettings & request_settings_, @@ -61,6 +62,7 @@ namespace bool for_disk_s3_, const Poco::Logger * log_) : client_ptr(client_ptr_) + , client_with_long_timeout_ptr(client_with_long_timeout_ptr_) , dest_bucket(dest_bucket_) , dest_key(dest_key_) , request_settings(request_settings_) @@ -76,6 +78,7 @@ namespace protected: std::shared_ptr client_ptr; + std::shared_ptr client_with_long_timeout_ptr; const String & dest_bucket; const String & dest_key; const S3Settings::RequestSettings & request_settings; @@ -176,7 +179,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); - auto outcome = client_ptr->CompleteMultipartUpload(request); + auto outcome = client_with_long_timeout_ptr->CompleteMultipartUpload(request); if (outcome.IsSuccess()) { @@ -430,13 +433,14 @@ namespace size_t offset_, size_t size_, const std::shared_ptr & client_ptr_, + const std::shared_ptr & client_with_long_timeout_ptr_, const String & dest_bucket_, const String & dest_key_, const S3Settings::RequestSettings & request_settings_, const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, bool for_disk_s3_) - : UploadHelper(client_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyDataToS3File")) + : UploadHelper(client_ptr_, client_with_long_timeout_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyDataToS3File")) , create_read_buffer(create_read_buffer_) , offset(offset_) , size(size_) @@ -598,6 +602,7 @@ namespace public: CopyFileHelper( const std::shared_ptr & client_ptr_, + const std::shared_ptr & client_with_long_timeout_ptr_, const String & src_bucket_, const String & src_key_, size_t src_offset_, @@ -609,7 +614,7 @@ namespace const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, bool for_disk_s3_) - : UploadHelper(client_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyS3File")) + : UploadHelper(client_ptr_, client_with_long_timeout_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyS3File")) , src_bucket(src_bucket_) , src_key(src_key_) , offset(src_offset_) @@ -672,7 +677,7 @@ namespace /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 request.SetContentType("binary/octet-stream"); - client_ptr->setKMSHeaders(request); + client_with_long_timeout_ptr->setKMSHeaders(request); } void processCopyRequest(const S3::CopyObjectRequest & request) @@ -684,7 +689,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3CopyObject); - auto outcome = client_ptr->CopyObject(request); + auto outcome = client_with_long_timeout_ptr->CopyObject(request); if (outcome.IsSuccess()) { LOG_TRACE( @@ -709,6 +714,7 @@ namespace offset, size, client_ptr, + client_with_long_timeout_ptr, dest_bucket, dest_key, request_settings, @@ -782,7 +788,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3UploadPartCopy); - auto outcome = client_ptr->UploadPartCopy(req); + auto outcome = client_with_long_timeout_ptr->UploadPartCopy(req); if (!outcome.IsSuccess()) { abortMultipartUpload(); @@ -800,6 +806,7 @@ void copyDataToS3File( size_t offset, size_t size, const std::shared_ptr & dest_s3_client, + const std::shared_ptr & dest_s3_client_with_long_timeout, const String & dest_bucket, const String & dest_key, const S3Settings::RequestSettings & settings, @@ -807,13 +814,14 @@ void copyDataToS3File( ThreadPoolCallbackRunner schedule, bool for_disk_s3) { - CopyDataToFileHelper helper{create_read_buffer, offset, size, dest_s3_client, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3}; + CopyDataToFileHelper helper{create_read_buffer, offset, size, dest_s3_client, dest_s3_client_with_long_timeout, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3}; helper.performCopy(); } void copyS3File( const std::shared_ptr & s3_client, + const std::shared_ptr & s3_client_with_long_timeout, const String & src_bucket, const String & src_key, size_t src_offset, @@ -828,7 +836,7 @@ void copyS3File( { if (settings.allow_native_copy) { - CopyFileHelper helper{s3_client, src_bucket, src_key, src_offset, src_size, dest_bucket, dest_key, settings, read_settings, object_metadata, schedule, for_disk_s3}; + CopyFileHelper helper{s3_client, s3_client_with_long_timeout, src_bucket, src_key, src_offset, src_size, dest_bucket, dest_key, settings, read_settings, object_metadata, schedule, for_disk_s3}; helper.performCopy(); } else @@ -837,7 +845,7 @@ void copyS3File( { return std::make_unique(s3_client, src_bucket, src_key, "", settings, read_settings); }; - copyDataToS3File(create_read_buffer, src_offset, src_size, s3_client, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3); + copyDataToS3File(create_read_buffer, src_offset, src_size, s3_client, s3_client_with_long_timeout, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3); } } diff --git a/src/IO/S3/copyS3File.h b/src/IO/S3/copyS3File.h index 33e22fdfba2..1bcbfd7735e 100644 --- a/src/IO/S3/copyS3File.h +++ b/src/IO/S3/copyS3File.h @@ -27,9 +27,15 @@ using CreateReadBuffer = std::function()>; /// because it is a known issue, it is fallbacks to read-write copy /// (copyDataToS3File()). /// +/// s3_client_with_long_timeout (may be equal to s3_client) is used for native copy and +/// CompleteMultipartUpload requests. These requests need longer timeout because S3 servers often +/// block on them for multiple seconds without sending or receiving data from us (maybe the servers +/// are copying data internally, or maybe throttling, idk). +/// /// read_settings - is used for throttling in case of native copy is not possible void copyS3File( const std::shared_ptr & s3_client, + const std::shared_ptr & s3_client_with_long_timeout, const String & src_bucket, const String & src_key, size_t src_offset, @@ -52,6 +58,7 @@ void copyDataToS3File( size_t offset, size_t size, const std::shared_ptr & dest_s3_client, + const std::shared_ptr & dest_s3_client_with_long_timeout, const String & dest_bucket, const String & dest_key, const S3Settings::RequestSettings & settings, diff --git a/src/IO/S3/tests/gtest_aws_s3_client.cpp b/src/IO/S3/tests/gtest_aws_s3_client.cpp index bff9ca6fa7b..c42f14e9a53 100644 --- a/src/IO/S3/tests/gtest_aws_s3_client.cpp +++ b/src/IO/S3/tests/gtest_aws_s3_client.cpp @@ -91,6 +91,7 @@ void doWriteRequest(std::shared_ptr client, const DB::S3:: DB::S3Settings::RequestSettings request_settings; request_settings.max_unexpected_write_error_retries = max_unexpected_write_error_retries; DB::WriteBufferFromS3 write_buffer( + client, client, uri.bucket, uri.key, @@ -170,7 +171,6 @@ TEST(IOTestAwsS3Client, AppendExtraSSECHeadersRead) "authorization: ... SignedHeaders=" "amz-sdk-invocation-id;" "amz-sdk-request;" - "clickhouse-request;" "content-type;" "host;" "x-amz-api-version;" @@ -216,7 +216,6 @@ TEST(IOTestAwsS3Client, AppendExtraSSEKMSHeadersRead) "authorization: ... SignedHeaders=" "amz-sdk-invocation-id;" "amz-sdk-request;" - "clickhouse-request;" "content-type;" "host;" "x-amz-api-version;" diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 62d0c80f1f2..e1b9c17efe9 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -77,6 +77,7 @@ struct WriteBufferFromS3::PartData WriteBufferFromS3::WriteBufferFromS3( std::shared_ptr client_ptr_, + std::shared_ptr client_with_long_timeout_ptr_, const String & bucket_, const String & key_, size_t buf_size_, @@ -91,6 +92,7 @@ WriteBufferFromS3::WriteBufferFromS3( , upload_settings(request_settings.getUploadSettings()) , write_settings(write_settings_) , client_ptr(std::move(client_ptr_)) + , client_with_long_timeout_ptr(std::move(client_with_long_timeout_ptr_)) , object_metadata(std::move(object_metadata_)) , buffer_allocation_policy(ChooseBufferPolicy(upload_settings)) , task_tracker( @@ -564,7 +566,7 @@ void WriteBufferFromS3::completeMultipartUpload() ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Stopwatch watch; - auto outcome = client_ptr->CompleteMultipartUpload(req); + auto outcome = client_with_long_timeout_ptr->CompleteMultipartUpload(req); watch.stop(); ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Microseconds, watch.elapsedMicroseconds()); diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 590342cc997..95148c49779 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -30,6 +30,8 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase public: WriteBufferFromS3( std::shared_ptr client_ptr_, + /// for CompleteMultipartUploadRequest, because it blocks on recv() for a few seconds on big uploads + std::shared_ptr client_with_long_timeout_ptr_, const String & bucket_, const String & key_, size_t buf_size_, @@ -88,6 +90,7 @@ private: const S3Settings::RequestSettings::PartUploadSettings & upload_settings; const WriteSettings write_settings; const std::shared_ptr client_ptr; + const std::shared_ptr client_with_long_timeout_ptr; const std::optional> object_metadata; Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3"); LogSeriesLimiterPtr limitedLog = std::make_shared(log, 1, 5); diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp index c82f97f8b20..21bdd9a6f26 100644 --- a/src/IO/tests/gtest_writebuffer_s3.cpp +++ b/src/IO/tests/gtest_writebuffer_s3.cpp @@ -549,6 +549,7 @@ public: getAsyncPolicy().setAutoExecute(false); return std::make_unique( + client, client, bucket, file_name, diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index bdbba5abd96..80ee1e9339d 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -825,6 +825,7 @@ public: write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique( configuration_.client, + configuration_.client_with_long_timeout, bucket, key, DBMS_DEFAULT_BUFFER_SIZE, @@ -1329,6 +1330,8 @@ void StorageS3::Configuration::connect(ContextPtr context) context->getConfigRef().getUInt64("s3.expiration_window_seconds", S3::DEFAULT_EXPIRATION_WINDOW_SECONDS)), auth_settings.no_sign_request.value_or(context->getConfigRef().getBool("s3.no_sign_request", false)), }); + + client_with_long_timeout = client->clone(std::nullopt, request_settings.long_request_timeout_ms); } void StorageS3::processNamedCollectionResult(StorageS3::Configuration & configuration, const NamedCollection & collection) diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 3f35c578e19..3330ac6c210 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -311,6 +311,7 @@ public: HTTPHeaderEntries headers_from_ast; std::shared_ptr client; + std::shared_ptr client_with_long_timeout; std::vector keys; }; diff --git a/src/Storages/StorageS3Settings.h b/src/Storages/StorageS3Settings.h index 728972c948c..e3d577ca0b3 100644 --- a/src/Storages/StorageS3Settings.h +++ b/src/Storages/StorageS3Settings.h @@ -69,7 +69,8 @@ struct S3Settings ThrottlerPtr get_request_throttler; ThrottlerPtr put_request_throttler; size_t retry_attempts = 10; - size_t request_timeout_ms = 30000; + size_t request_timeout_ms = 3000; + size_t long_request_timeout_ms = 30000; // TODO: Take this from config like request_timeout_ms bool allow_native_copy = true; bool throw_on_zero_files_match = false; diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml index 4210c13b727..206eb4f2bad 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml @@ -4,7 +4,6 @@ 1000000 - 1 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml index 95a313ea4f2..556bf60d385 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml @@ -4,7 +4,6 @@ 5 - 0 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml index 7b1f503ed55..b77e72d808b 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml @@ -7,18 +7,11 @@ - - s3 - http://minio1:9001/root/data/ - minio - minio123 - s3 http://resolver:8083/root/data/ minio minio123 - 1 @@ -30,16 +23,9 @@ - - -
- s3 -
- - - s3 + broken_s3 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py index b000ccabcf4..d6bcb3fb8f4 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/test.py +++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py @@ -64,8 +64,6 @@ def test_upload_after_check_works(cluster, broken_s3): data String ) ENGINE=MergeTree() ORDER BY id - SETTINGS - storage_policy='broken_s3' """ ) @@ -80,7 +78,7 @@ def test_upload_after_check_works(cluster, broken_s3): assert "suddenly disappeared" in error, error -def get_multipart_counters(node, query_id, log_type="ExceptionWhileProcessing"): +def get_counters(node, query_id, log_type="ExceptionWhileProcessing"): node.query("SYSTEM FLUSH LOGS") return [ int(x) @@ -89,25 +87,7 @@ def get_multipart_counters(node, query_id, log_type="ExceptionWhileProcessing"): SELECT ProfileEvents['S3CreateMultipartUpload'], ProfileEvents['S3UploadPart'], - ProfileEvents['S3WriteRequestsErrors'], - FROM system.query_log - WHERE query_id='{query_id}' - AND type='{log_type}' - """ - ).split() - if x - ] - - -def get_put_counters(node, query_id, log_type="ExceptionWhileProcessing"): - node.query("SYSTEM FLUSH LOGS") - return [ - int(x) - for x in node.query( - f""" - SELECT - ProfileEvents['S3PutObject'], - ProfileEvents['S3WriteRequestsErrors'], + ProfileEvents['S3WriteRequestsErrors'] FROM system.query_log WHERE query_id='{query_id}' AND type='{log_type}' @@ -149,12 +129,12 @@ def test_upload_s3_fail_create_multi_part_upload(cluster, broken_s3, compression assert "Code: 499" in error, error assert "mock s3 injected error" in error, error - create_multipart, upload_parts, s3_errors = get_multipart_counters( + count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( node, insert_query_id ) - assert create_multipart == 1 - assert upload_parts == 0 - assert s3_errors == 1 + assert count_create_multi_part_uploads == 1 + assert count_upload_parts == 0 + assert count_s3_errors == 1 # Add "lz4" compression method in the list after https://github.com/ClickHouse/ClickHouse/issues/50975 is fixed @@ -192,12 +172,12 @@ def test_upload_s3_fail_upload_part_when_multi_part_upload( assert "Code: 499" in error, error assert "mock s3 injected error" in error, error - create_multipart, upload_parts, s3_errors = get_multipart_counters( + count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( node, insert_query_id ) - assert create_multipart == 1 - assert upload_parts >= 2 - assert s3_errors >= 2 + assert count_create_multi_part_uploads == 1 + assert count_upload_parts >= 2 + assert count_s3_errors >= 2 def test_when_s3_connection_refused_is_retried(cluster, broken_s3): @@ -227,12 +207,12 @@ def test_when_s3_connection_refused_is_retried(cluster, broken_s3): query_id=insert_query_id, ) - create_multipart, upload_parts, s3_errors = get_multipart_counters( + count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( node, insert_query_id, log_type="QueryFinish" ) - assert create_multipart == 1 - assert upload_parts == 39 - assert s3_errors == 3 + assert count_create_multi_part_uploads == 1 + assert count_upload_parts == 39 + assert count_s3_errors == 3 broken_s3.setup_at_part_upload(count=1000, after=2, action="connection_refused") insert_query_id = f"INSERT_INTO_TABLE_FUNCTION_CONNECTION_REFUSED_RETRIED_1" @@ -299,13 +279,13 @@ def test_when_s3_connection_reset_by_peer_at_upload_is_retried( query_id=insert_query_id, ) - create_multipart, upload_parts, s3_errors = get_multipart_counters( + count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( node, insert_query_id, log_type="QueryFinish" ) - assert create_multipart == 1 - assert upload_parts == 39 - assert s3_errors == 3 + assert count_create_multi_part_uploads == 1 + assert count_upload_parts == 39 + assert count_s3_errors == 3 broken_s3.setup_at_part_upload( count=1000, @@ -381,13 +361,13 @@ def test_when_s3_connection_reset_by_peer_at_create_mpu_retried( query_id=insert_query_id, ) - create_multipart, upload_parts, s3_errors = get_multipart_counters( + count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( node, insert_query_id, log_type="QueryFinish" ) - assert create_multipart == 1 - assert upload_parts == 39 - assert s3_errors == 3 + assert count_create_multi_part_uploads == 1 + assert count_upload_parts == 39 + assert count_s3_errors == 3 broken_s3.setup_at_create_multi_part_upload( count=1000, @@ -458,13 +438,13 @@ def test_when_s3_broken_pipe_at_upload_is_retried(cluster, broken_s3): query_id=insert_query_id, ) - create_multipart, upload_parts, s3_errors = get_multipart_counters( + count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( node, insert_query_id, log_type="QueryFinish" ) - assert create_multipart == 1 - assert upload_parts == 7 - assert s3_errors == 3 + assert count_create_multi_part_uploads == 1 + assert count_upload_parts == 7 + assert count_s3_errors == 3 broken_s3.setup_at_part_upload( count=1000, @@ -553,60 +533,3 @@ def test_query_is_canceled_with_inf_retries(cluster, broken_s3): retry_count=120, sleep_time=1, ) - - -@pytest.mark.parametrize("node_name", ["node", "node_with_inf_s3_retries"]) -def test_adaptive_timeouts(cluster, broken_s3, node_name): - node = cluster.instances[node_name] - - broken_s3.setup_fake_puts(part_length=1) - broken_s3.setup_slow_answers( - timeout=5, - count=1000000, - ) - - insert_query_id = f"TEST_ADAPTIVE_TIMEOUTS_{node_name}" - node.query( - f""" - INSERT INTO - TABLE FUNCTION s3( - 'http://resolver:8083/root/data/adaptive_timeouts', - 'minio', 'minio123', - 'CSV', auto, 'none' - ) - SELECT - * - FROM system.numbers - LIMIT 1 - SETTINGS - s3_request_timeout_ms=30000, - s3_check_objects_after_upload=0 - """, - query_id=insert_query_id, - ) - - broken_s3.reset() - - put_objects, s3_errors = get_put_counters( - node, insert_query_id, log_type="QueryFinish" - ) - - assert put_objects == 1 - - s3_use_adaptive_timeouts = node.query( - f""" - SELECT - value - FROM system.settings - WHERE - name='s3_use_adaptive_timeouts' - """ - ).strip() - - if node_name == "node_with_inf_s3_retries": - # first 2 attempts failed - assert s3_use_adaptive_timeouts == "1" - assert s3_errors == 1 - else: - assert s3_use_adaptive_timeouts == "0" - assert s3_errors == 0 diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml index 6303e9273fc..235b9a7b7a1 100644 --- a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml @@ -11,7 +11,6 @@ true 0 - 0 20000 @@ -34,7 +33,6 @@ true 1 - 0 1 20000 diff --git a/tests/integration/test_storage_s3/configs/defaultS3.xml b/tests/integration/test_storage_s3/configs/defaultS3.xml index 7dac6d9fbb5..37454ef6781 100644 --- a/tests/integration/test_storage_s3/configs/defaultS3.xml +++ b/tests/integration/test_storage_s3/configs/defaultS3.xml @@ -1,4 +1,9 @@ + + + 5 + + http://resolver:8080 diff --git a/tests/integration/test_storage_s3/configs/s3_retry.xml b/tests/integration/test_storage_s3/configs/s3_retry.xml index 3171da051d0..727e23273cf 100644 --- a/tests/integration/test_storage_s3/configs/s3_retry.xml +++ b/tests/integration/test_storage_s3/configs/s3_retry.xml @@ -1,9 +1,7 @@ - 1 - 10 - 5 + 5 diff --git a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py index 5ef781bdc9e..103dd30340c 100644 --- a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py +++ b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py @@ -4,7 +4,6 @@ import re import socket import struct import sys -import time def gen_n_digit_number(n): @@ -40,14 +39,14 @@ random.seed("Unstable server/1.0") # Generating some "random" data and append a line which contains sum of numbers in column 4. lines = ( - b"".join([gen_line() for _ in range(500000)]) + b"".join((gen_line() for _ in range(500000))) + f"0,0,0,{-sum_in_4_column}\n".encode() ) class RequestHandler(http.server.BaseHTTPRequestHandler): def do_HEAD(self): - if self.path == "/root/test.csv" or self.path == "/root/slow_send_test.csv": + if self.path == "/root/test.csv": self.from_bytes = 0 self.end_bytes = len(lines) self.size = self.end_bytes @@ -102,18 +101,6 @@ class RequestHandler(http.server.BaseHTTPRequestHandler): print("Dropping connection") break - if self.path == "/root/slow_send_test.csv": - self.send_block_size = 81920 - - for c, i in enumerate( - range(self.from_bytes, self.end_bytes, self.send_block_size) - ): - self.wfile.write( - lines[i : min(i + self.send_block_size, self.end_bytes)] - ) - self.wfile.flush() - time.sleep(1) - elif self.path == "/": self.wfile.write(b"OK") diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 835c8b908f0..3dd3c9e39d0 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -818,15 +818,6 @@ def test_storage_s3_get_unstable(started_cluster): assert result.splitlines() == ["500001,500000,0"] -def test_storage_s3_get_slow(started_cluster): - bucket = started_cluster.minio_bucket - instance = started_cluster.instances["dummy"] - table_format = "column1 Int64, column2 Int64, column3 Int64, column4 Int64" - get_query = f"SELECT count(), sum(column3), sum(column4) FROM s3('http://resolver:8081/{started_cluster.minio_bucket}/slow_send_test.csv', 'CSV', '{table_format}') FORMAT CSV" - result = run_query(instance, get_query) - assert result.splitlines() == ["500001,500000,0"] - - def test_storage_s3_put_uncompressed(started_cluster): bucket = started_cluster.minio_bucket instance = started_cluster.instances["dummy"] From f999337daee891499a78cadec1cc562cf29ebcaa Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:53:22 +0100 Subject: [PATCH 191/192] Revert "Revert "s3 adaptive timeouts"" --- base/poco/Net/src/HTTPServerSession.cpp | 1 - base/poco/Net/src/HTTPSession.cpp | 31 ++++- docs/en/operations/settings/settings.md | 7 + src/Backups/BackupIO_S3.cpp | 9 +- src/Coordination/KeeperSnapshotManagerS3.cpp | 1 - src/Core/Settings.h | 3 +- .../ObjectStorages/S3/S3ObjectStorage.cpp | 58 +++----- src/Disks/ObjectStorages/S3/S3ObjectStorage.h | 15 +- src/Disks/ObjectStorages/S3/diskSettings.cpp | 8 +- src/IO/ConnectionTimeouts.cpp | 82 +++++++++++ src/IO/ConnectionTimeouts.h | 2 + src/IO/HTTPCommon.cpp | 12 +- src/IO/HTTPCommon.h | 2 + src/IO/ReadBufferFromS3.cpp | 24 ++-- src/IO/ReadBufferFromS3.h | 4 +- src/IO/S3/Client.cpp | 12 +- src/IO/S3/Client.h | 10 +- src/IO/S3/PocoHTTPClient.cpp | 111 +++++++++++---- src/IO/S3/PocoHTTPClient.h | 5 + src/IO/S3/copyS3File.cpp | 26 ++-- src/IO/S3/copyS3File.h | 7 - src/IO/S3/tests/gtest_aws_s3_client.cpp | 3 +- src/IO/WriteBufferFromS3.cpp | 4 +- src/IO/WriteBufferFromS3.h | 3 - src/IO/tests/gtest_writebuffer_s3.cpp | 1 - src/Storages/StorageS3.cpp | 3 - src/Storages/StorageS3.h | 1 - src/Storages/StorageS3Settings.h | 3 +- .../configs/inf_s3_retries.xml | 1 + .../configs/s3_retries.xml | 1 + .../configs/storage_conf.xml | 16 ++- .../test_checking_s3_blobs_paranoid/test.py | 129 ++++++++++++++---- .../configs/config.d/storage_conf.xml | 2 + .../test_storage_s3/configs/defaultS3.xml | 5 - .../test_storage_s3/configs/s3_retry.xml | 4 +- .../s3_mocks/unstable_server.py | 17 ++- tests/integration/test_storage_s3/test.py | 9 ++ 37 files changed, 430 insertions(+), 202 deletions(-) diff --git a/base/poco/Net/src/HTTPServerSession.cpp b/base/poco/Net/src/HTTPServerSession.cpp index f6d3c4e5b92..d4f2b24879e 100644 --- a/base/poco/Net/src/HTTPServerSession.cpp +++ b/base/poco/Net/src/HTTPServerSession.cpp @@ -26,7 +26,6 @@ HTTPServerSession::HTTPServerSession(const StreamSocket& socket, HTTPServerParam _maxKeepAliveRequests(pParams->getMaxKeepAliveRequests()) { setTimeout(pParams->getTimeout()); - this->socket().setReceiveTimeout(pParams->getTimeout()); } diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index d2663baaf9f..8f951b3102c 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -93,9 +93,34 @@ void HTTPSession::setTimeout(const Poco::Timespan& timeout) void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco::Timespan& sendTimeout, const Poco::Timespan& receiveTimeout) { - _connectionTimeout = connectionTimeout; - _sendTimeout = sendTimeout; - _receiveTimeout = receiveTimeout; + try + { + _connectionTimeout = connectionTimeout; + + if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) { + _sendTimeout = sendTimeout; + + if (connected()) + _socket.setSendTimeout(_sendTimeout); + } + + if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) { + _receiveTimeout = receiveTimeout; + + if (connected()) + _socket.setReceiveTimeout(_receiveTimeout); + } + } + catch (NetException &) + { +#ifndef NDEBUG + throw; +#else + // mute exceptions in release + // just in case when changing settings on socket is not allowed + // however it should be OK for timeouts +#endif + } } diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index e61934d2168..edc1c9bdfd7 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -4826,3 +4826,10 @@ When set to `true` the metadata files are written with `VERSION_FULL_OBJECT_KEY` When set to `false` the metadata files are written with the previous format version, `VERSION_INLINE_DATA`. With that format only suffixes of object storage key names are are written to the metadata files. The prefix for all of object storage key names is set in configurations files at `storage_configuration.disks` section. Default value: `false`. + +## s3_use_adaptive_timeouts {#s3_use_adaptive_timeouts} + +When set to `true` than for all s3 requests first two attempts are made with low send and receive timeouts. +When set to `false` than all attempts are made with identical timeouts. + +Default value: `true`. diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index eb9dcf6b45a..ea3f57c27ff 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -55,7 +55,9 @@ namespace static_cast(context->getGlobalContext()->getSettingsRef().s3_max_redirects), static_cast(context->getGlobalContext()->getSettingsRef().s3_retry_attempts), context->getGlobalContext()->getSettingsRef().enable_s3_requests_logging, - /* for_disk_s3 = */ false, request_settings.get_request_throttler, request_settings.put_request_throttler, + /* for_disk_s3 = */ false, + request_settings.get_request_throttler, + request_settings.put_request_throttler, s3_uri.uri.getScheme()); client_configuration.endpointOverride = s3_uri.endpoint; @@ -167,7 +169,6 @@ void BackupReaderS3::copyFileToDisk(const String & path_in_backup, size_t file_s blob_path.size(), mode); copyS3File( - client, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, @@ -229,7 +230,6 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src { LOG_TRACE(log, "Copying file {} from disk {} to S3", src_path, src_disk->getName()); copyS3File( - client, client, /* src_bucket */ blob_path[1], /* src_key= */ blob_path[0], @@ -268,7 +268,7 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, void BackupWriterS3::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) { - copyDataToS3File(create_read_buffer, start_pos, length, client, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {}, + copyDataToS3File(create_read_buffer, start_pos, length, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {}, threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterS3")); } @@ -298,7 +298,6 @@ std::unique_ptr BackupWriterS3::writeFile(const String & file_name) { return std::make_unique( client, - client, // already has long timeout s3_uri.bucket, fs::path(s3_uri.key) / file_name, DBMS_DEFAULT_BUFFER_SIZE, diff --git a/src/Coordination/KeeperSnapshotManagerS3.cpp b/src/Coordination/KeeperSnapshotManagerS3.cpp index 302e05c8418..bedde0d7b39 100644 --- a/src/Coordination/KeeperSnapshotManagerS3.cpp +++ b/src/Coordination/KeeperSnapshotManagerS3.cpp @@ -148,7 +148,6 @@ void KeeperSnapshotManagerS3::uploadSnapshotImpl(const SnapshotFileInfo & snapsh const auto create_writer = [&](const auto & key) { return WriteBufferFromS3( - s3_client->client, s3_client->client, s3_client->uri.bucket, key, diff --git a/src/Core/Settings.h b/src/Core/Settings.h index ac4c6b6c17f..bb5e4322485 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -94,6 +94,7 @@ class IColumn; M(UInt64, s3_max_put_rps, 0, "Limit on S3 PUT request per second rate before throttling. Zero means unlimited.", 0) \ M(UInt64, s3_max_put_burst, 0, "Max number of requests that can be issued simultaneously before hitting request per second limit. By default (0) equals to `s3_max_put_rps`", 0) \ M(UInt64, s3_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ + M(Bool, s3_use_adaptive_timeouts, true, "When adaptive timeouts are enabled first two attempts are made with low receive and send timeout", 0) \ M(UInt64, azure_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ M(Bool, s3_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables.", 0) \ M(Bool, azure_truncate_on_insert, false, "Enables or disables truncate before insert in azure engine tables.", 0) \ @@ -104,7 +105,7 @@ class IColumn; M(Bool, s3_allow_parallel_part_upload, true, "Use multiple threads for s3 multipart upload. It may lead to slightly higher memory usage", 0) \ M(Bool, s3_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \ M(UInt64, s3_retry_attempts, 100, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries", 0) \ - M(UInt64, s3_request_timeout_ms, 3000, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \ + M(UInt64, s3_request_timeout_ms, 30000, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \ M(UInt64, s3_http_connection_pool_size, 1000, "How many reusable open connections to keep per S3 endpoint. Only applies to the S3 table engine and table function, not to S3 disks (for disks, use disk config instead). Global setting, can only be set in config, overriding it per session or per query has no effect.", 0) \ M(Bool, enable_s3_requests_logging, false, "Enable very explicit logging of S3 requests. Makes sense for debug only.", 0) \ M(String, s3queue_default_zookeeper_path, "/clickhouse/s3queue/", "Default zookeeper path prefix for S3Queue engine", 0) \ diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 3af316bf0cf..308db389ee1 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -155,7 +155,7 @@ private: bool S3ObjectStorage::exists(const StoredObject & object) const { auto settings_ptr = s3_settings.get(); - return S3::objectExists(*clients.get()->client, bucket, object.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + return S3::objectExists(*client.get(), bucket, object.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); } std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT @@ -174,7 +174,7 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT (const std::string & path, size_t read_until_position) -> std::unique_ptr { return std::make_unique( - clients.get()->client, + client.get(), bucket, path, version_id, @@ -224,7 +224,7 @@ std::unique_ptr S3ObjectStorage::readObject( /// NOLINT { auto settings_ptr = s3_settings.get(); return std::make_unique( - clients.get()->client, + client.get(), bucket, object.remote_path, version_id, @@ -249,10 +249,8 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN if (write_settings.s3_allow_parallel_part_upload) scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "VFSWrite"); - auto clients_ = clients.get(); return std::make_unique( - clients_->client, - clients_->client_with_long_timeout, + client.get(), bucket, object.remote_path, buf_size, @@ -266,15 +264,12 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN ObjectStorageIteratorPtr S3ObjectStorage::iterate(const std::string & path_prefix) const { auto settings_ptr = s3_settings.get(); - auto client_ptr = clients.get()->client; - - return std::make_shared(bucket, path_prefix, client_ptr, settings_ptr->list_object_keys_size); + return std::make_shared(bucket, path_prefix, client.get(), settings_ptr->list_object_keys_size); } void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const { auto settings_ptr = s3_settings.get(); - auto client_ptr = clients.get()->client; S3::ListObjectsV2Request request; request.SetBucket(bucket); @@ -289,7 +284,7 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet { ProfileEvents::increment(ProfileEvents::S3ListObjects); ProfileEvents::increment(ProfileEvents::DiskS3ListObjects); - outcome = client_ptr->ListObjectsV2(request); + outcome = client.get()->ListObjectsV2(request); throwIfError(outcome); auto result = outcome.GetResult(); @@ -320,14 +315,12 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exists) { - auto client_ptr = clients.get()->client; - ProfileEvents::increment(ProfileEvents::S3DeleteObjects); ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); S3::DeleteObjectRequest request; request.SetBucket(bucket); request.SetKey(object.remote_path); - auto outcome = client_ptr->DeleteObject(request); + auto outcome = client.get()->DeleteObject(request); throwIfUnexpectedError(outcome, if_exists); @@ -346,7 +339,6 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e } else { - auto client_ptr = clients.get()->client; auto settings_ptr = s3_settings.get(); size_t chunk_size_limit = settings_ptr->objects_chunk_size_to_delete; @@ -375,7 +367,7 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e S3::DeleteObjectsRequest request; request.SetBucket(bucket); request.SetDelete(delkeys); - auto outcome = client_ptr->DeleteObjects(request); + auto outcome = client.get()->DeleteObjects(request); throwIfUnexpectedError(outcome, if_exists); @@ -407,7 +399,7 @@ void S3ObjectStorage::removeObjectsIfExist(const StoredObjects & objects) std::optional S3ObjectStorage::tryGetObjectMetadata(const std::string & path) const { auto settings_ptr = s3_settings.get(); - auto object_info = S3::getObjectInfo(*clients.get()->client, bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true, /* throw_on_error= */ false); + auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true, /* throw_on_error= */ false); if (object_info.size == 0 && object_info.last_modification_time == 0 && object_info.metadata.empty()) return {}; @@ -423,7 +415,7 @@ std::optional S3ObjectStorage::tryGetObjectMetadata(const std::s ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path) const { auto settings_ptr = s3_settings.get(); - auto object_info = S3::getObjectInfo(*clients.get()->client, bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true); + auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true); ObjectMetadata result; result.size_bytes = object_info.size; @@ -444,12 +436,12 @@ void S3ObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT /// Shortcut for S3 if (auto * dest_s3 = dynamic_cast(&object_storage_to); dest_s3 != nullptr) { - auto clients_ = clients.get(); + auto client_ = client.get(); auto settings_ptr = s3_settings.get(); - auto size = S3::getObjectSize(*clients_->client, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + auto size = S3::getObjectSize(*client_, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); auto scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "S3ObjStor_copy"); - copyS3File(clients_->client, - clients_->client_with_long_timeout, + copyS3File( + client.get(), bucket, object_from.remote_path, 0, @@ -473,12 +465,11 @@ void S3ObjectStorage::copyObject( // NOLINT const WriteSettings &, std::optional object_to_attributes) { - auto clients_ = clients.get(); + auto client_ = client.get(); auto settings_ptr = s3_settings.get(); - auto size = S3::getObjectSize(*clients_->client, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + auto size = S3::getObjectSize(*client_, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); auto scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "S3ObjStor_copy"); - copyS3File(clients_->client, - clients_->client_with_long_timeout, + copyS3File(client_, bucket, object_from.remote_path, 0, @@ -499,31 +490,25 @@ void S3ObjectStorage::setNewSettings(std::unique_ptr && void S3ObjectStorage::shutdown() { - auto clients_ptr = clients.get(); /// This call stops any next retry attempts for ongoing S3 requests. /// If S3 request is failed and the method below is executed S3 client immediately returns the last failed S3 request outcome. /// If S3 is healthy nothing wrong will be happened and S3 requests will be processed in a regular way without errors. /// This should significantly speed up shutdown process if S3 is unhealthy. - const_cast(*clients_ptr->client).DisableRequestProcessing(); - const_cast(*clients_ptr->client_with_long_timeout).DisableRequestProcessing(); + const_cast(*client.get()).DisableRequestProcessing(); } void S3ObjectStorage::startup() { - auto clients_ptr = clients.get(); - /// Need to be enabled if it was disabled during shutdown() call. - const_cast(*clients_ptr->client).EnableRequestProcessing(); - const_cast(*clients_ptr->client_with_long_timeout).EnableRequestProcessing(); + const_cast(*client.get()).EnableRequestProcessing(); } void S3ObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) { auto new_s3_settings = getSettings(config, config_prefix, context); auto new_client = getClient(config, config_prefix, context, *new_s3_settings); - auto new_clients = std::make_unique(std::move(new_client), *new_s3_settings); s3_settings.set(std::move(new_s3_settings)); - clients.set(std::move(new_clients)); + client.set(std::move(new_client)); } std::unique_ptr S3ObjectStorage::cloneObjectStorage( @@ -538,9 +523,6 @@ std::unique_ptr S3ObjectStorage::cloneObjectStorage( endpoint, object_key_prefix); } -S3ObjectStorage::Clients::Clients(std::shared_ptr client_, const S3ObjectStorageSettings & settings) - : client(std::move(client_)), client_with_long_timeout(client->clone(std::nullopt, settings.request_settings.long_request_timeout_ms)) {} - ObjectStorageKey S3ObjectStorage::generateObjectKeyForPath(const std::string &) const { /// Path to store the new S3 object. diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index b1b3fb22366..7d14482311f 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -39,16 +39,6 @@ struct S3ObjectStorageSettings class S3ObjectStorage : public IObjectStorage { -public: - struct Clients - { - std::shared_ptr client; - std::shared_ptr client_with_long_timeout; - - Clients() = default; - Clients(std::shared_ptr client, const S3ObjectStorageSettings & settings); - }; - private: friend class S3PlainObjectStorage; @@ -63,7 +53,7 @@ private: String object_key_prefix_) : bucket(std::move(bucket_)) , object_key_prefix(std::move(object_key_prefix_)) - , clients(std::make_unique(std::move(client_), *s3_settings_)) + , client(std::move(client_)) , s3_settings(std::move(s3_settings_)) , s3_capabilities(s3_capabilities_) , version_id(std::move(version_id_)) @@ -184,7 +174,8 @@ private: std::string bucket; String object_key_prefix; - MultiVersion clients; + + MultiVersion client; MultiVersion s3_settings; S3Capabilities s3_capabilities; diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index de88c876922..0232a6eb070 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -60,13 +60,15 @@ std::unique_ptr getClient( uri.uri.getScheme()); client_configuration.connectTimeoutMs = config.getUInt(config_prefix + ".connect_timeout_ms", 1000); - client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 3000); + client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 30000); client_configuration.maxConnections = config.getUInt(config_prefix + ".max_connections", 100); client_configuration.endpointOverride = uri.endpoint; - client_configuration.http_keep_alive_timeout_ms - = config.getUInt(config_prefix + ".http_keep_alive_timeout_ms", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT * 1000); + client_configuration.http_keep_alive_timeout_ms = config.getUInt( + config_prefix + ".http_keep_alive_timeout_ms", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT * 1000); client_configuration.http_connection_pool_size = config.getUInt(config_prefix + ".http_connection_pool_size", 1000); client_configuration.wait_on_pool_size_limit = false; + client_configuration.s3_use_adaptive_timeouts = config.getBool( + config_prefix + ".use_adaptive_timeouts", client_configuration.s3_use_adaptive_timeouts); /* * Override proxy configuration for backwards compatibility with old configuration format. diff --git a/src/IO/ConnectionTimeouts.cpp b/src/IO/ConnectionTimeouts.cpp index 01fbaa4f817..970afc75ec3 100644 --- a/src/IO/ConnectionTimeouts.cpp +++ b/src/IO/ConnectionTimeouts.cpp @@ -133,4 +133,86 @@ ConnectionTimeouts ConnectionTimeouts::getHTTPTimeouts(const Settings & settings settings.http_receive_timeout); } +class SendReceiveTimeoutsForFirstAttempt +{ +private: + static constexpr size_t known_methods_count = 6; + using KnownMethodsArray = std::array; + static const KnownMethodsArray known_methods; + + /// HTTP_POST is used for CompleteMultipartUpload requests. Its latency could be high. + /// These requests need longer timeout, especially when minio is used. + /// The same assumption are made for HTTP_DELETE, HTTP_PATCH + /// That requests are more heavy that HTTP_GET, HTTP_HEAD, HTTP_PUT + + static constexpr Poco::Timestamp::TimeDiff first_byte_ms[known_methods_count][2] = + { + /* GET */ {200, 200}, + /* POST */ {200, 200}, + /* DELETE */ {200, 200}, + /* PUT */ {200, 200}, + /* HEAD */ {200, 200}, + /* PATCH */ {200, 200}, + }; + + static constexpr Poco::Timestamp::TimeDiff rest_bytes_ms[known_methods_count][2] = + { + /* GET */ {500, 500}, + /* POST */ {1000, 30000}, + /* DELETE */ {1000, 10000}, + /* PUT */ {1000, 3000}, + /* HEAD */ {500, 500}, + /* PATCH */ {1000, 10000}, + }; + + static_assert(sizeof(first_byte_ms) == sizeof(rest_bytes_ms)); + static_assert(sizeof(first_byte_ms) == known_methods_count * sizeof(Poco::Timestamp::TimeDiff) * 2); + + static size_t getMethodIndex(const String & method) + { + KnownMethodsArray::const_iterator it = std::find(known_methods.begin(), known_methods.end(), method); + chassert(it != known_methods.end()); + if (it == known_methods.end()) + return 0; + return std::distance(known_methods.begin(), it); + } + +public: + static std::pair getSendReceiveTimeout(const String & method, bool first_byte) + { + auto idx = getMethodIndex(method); + + if (first_byte) + return std::make_pair( + Poco::Timespan(first_byte_ms[idx][0] * 1000), + Poco::Timespan(first_byte_ms[idx][1] * 1000) + ); + + return std::make_pair( + Poco::Timespan(rest_bytes_ms[idx][0] * 1000), + Poco::Timespan(rest_bytes_ms[idx][1] * 1000) + ); + } +}; + +const SendReceiveTimeoutsForFirstAttempt::KnownMethodsArray SendReceiveTimeoutsForFirstAttempt::known_methods = +{ + "GET", "POST", "DELETE", "PUT", "HEAD", "PATCH" +}; + + +ConnectionTimeouts ConnectionTimeouts::getAdaptiveTimeouts(const String & method, bool first_attempt, bool first_byte) const +{ + if (!first_attempt) + return *this; + + auto [send, recv] = SendReceiveTimeoutsForFirstAttempt::getSendReceiveTimeout(method, first_byte); + + auto aggressive = *this; + aggressive.send_timeout = saturate(send, send_timeout); + aggressive.receive_timeout = saturate(recv, receive_timeout); + + return aggressive; +} + } diff --git a/src/IO/ConnectionTimeouts.h b/src/IO/ConnectionTimeouts.h index 684af42827f..aabebdb836d 100644 --- a/src/IO/ConnectionTimeouts.h +++ b/src/IO/ConnectionTimeouts.h @@ -67,6 +67,8 @@ struct ConnectionTimeouts /// Timeouts for the case when we will try many addresses in a loop. static ConnectionTimeouts getTCPTimeoutsWithFailover(const Settings & settings); static ConnectionTimeouts getHTTPTimeouts(const Settings & settings, Poco::Timespan http_keep_alive_timeout); + + ConnectionTimeouts getAdaptiveTimeouts(const String & method, bool first_attempt, bool first_byte) const; }; } diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 65ffa51a466..cce394c67c9 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -50,12 +50,6 @@ namespace ErrorCodes namespace { - void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts) - { - session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout); - session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); - } - Poco::Net::HTTPClientSession::ProxyConfig proxyConfigurationToPocoProxyConfig(const ProxyConfiguration & proxy_configuration) { Poco::Net::HTTPClientSession::ProxyConfig poco_proxy_config; @@ -359,6 +353,12 @@ namespace }; } +void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts) +{ + session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout); + session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); +} + void setResponseDefaultHeaders(HTTPServerResponse & response, size_t keep_alive_timeout) { if (!response.getKeepAlive()) diff --git a/src/IO/HTTPCommon.h b/src/IO/HTTPCommon.h index de62b5d5c16..c9968fc6915 100644 --- a/src/IO/HTTPCommon.h +++ b/src/IO/HTTPCommon.h @@ -113,4 +113,6 @@ std::istream * receiveResponse( void assertResponseIsOk( const Poco::Net::HTTPRequest & request, Poco::Net::HTTPResponse & response, std::istream & istr, bool allow_redirects = false); + +void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts); } diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index f19978ccb47..c9c9319c44c 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -167,9 +167,9 @@ bool ReadBufferFromS3::nextImpl() } size_t sleep_time_with_backoff_milliseconds = 100; - for (size_t attempt = 0; !next_result; ++attempt) + for (size_t attempt = 1; !next_result; ++attempt) { - bool last_attempt = attempt + 1 >= request_settings.max_single_read_retries; + bool last_attempt = attempt >= request_settings.max_single_read_retries; ProfileEventTimeIncrement watch(ProfileEvents::ReadBufferFromS3Microseconds); @@ -177,7 +177,7 @@ bool ReadBufferFromS3::nextImpl() { if (!impl) { - impl = initialize(); + impl = initialize(attempt); if (use_external_buffer) { @@ -232,9 +232,9 @@ size_t ReadBufferFromS3::readBigAt(char * to, size_t n, size_t range_begin, cons { size_t initial_n = n; size_t sleep_time_with_backoff_milliseconds = 100; - for (size_t attempt = 0; n > 0; ++attempt) + for (size_t attempt = 1; n > 0; ++attempt) { - bool last_attempt = attempt + 1 >= request_settings.max_single_read_retries; + bool last_attempt = attempt >= request_settings.max_single_read_retries; size_t bytes_copied = 0; ProfileEventTimeIncrement watch(ProfileEvents::ReadBufferFromS3Microseconds); @@ -266,7 +266,7 @@ size_t ReadBufferFromS3::readBigAt(char * to, size_t n, size_t range_begin, cons try { - result = sendRequest(range_begin, range_begin + n - 1); + result = sendRequest(attempt, range_begin, range_begin + n - 1); std::istream & istr = result->GetBody(); copyFromIStreamWithProgressCallback(istr, to, n, progress_callback, &bytes_copied); @@ -304,8 +304,8 @@ bool ReadBufferFromS3::processException(Poco::Exception & e, size_t read_offset, LOG_DEBUG( log, "Caught exception while reading S3 object. Bucket: {}, Key: {}, Version: {}, Offset: {}, " - "Attempt: {}, Message: {}", - bucket, key, version_id.empty() ? "Latest" : version_id, read_offset, attempt, e.message()); + "Attempt: {}/{}, Message: {}", + bucket, key, version_id.empty() ? "Latest" : version_id, read_offset, attempt, request_settings.max_single_read_retries, e.message()); if (auto * s3_exception = dynamic_cast(&e)) @@ -463,7 +463,7 @@ ReadBufferFromS3::~ReadBufferFromS3() } } -std::unique_ptr ReadBufferFromS3::initialize() +std::unique_ptr ReadBufferFromS3::initialize(size_t attempt) { resetSessionIfNeeded(readAllRangeSuccessfully(), read_result); read_all_range_successfully = false; @@ -475,13 +475,13 @@ std::unique_ptr ReadBufferFromS3::initialize() if (read_until_position && offset >= read_until_position) throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read beyond right offset ({} > {})", offset, read_until_position - 1); - read_result = sendRequest(offset, read_until_position ? std::make_optional(read_until_position - 1) : std::nullopt); + read_result = sendRequest(attempt, offset, read_until_position ? std::make_optional(read_until_position - 1) : std::nullopt); size_t buffer_size = use_external_buffer ? 0 : read_settings.remote_fs_buffer_size; return std::make_unique(read_result->GetBody(), buffer_size); } -Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t range_begin, std::optional range_end_incl) const +Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t attempt, size_t range_begin, std::optional range_end_incl) const { S3::GetObjectRequest req; req.SetBucket(bucket); @@ -489,6 +489,8 @@ Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t range_begin if (!version_id.empty()) req.SetVersionId(version_id); + req.SetAdditionalCustomHeaderValue("clickhouse-request", fmt::format("attempt={}", attempt)); + if (range_end_incl) { req.SetRange(fmt::format("bytes={}-{}", range_begin, *range_end_incl)); diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h index 0835e52a5b2..101e25f8b43 100644 --- a/src/IO/ReadBufferFromS3.h +++ b/src/IO/ReadBufferFromS3.h @@ -79,7 +79,7 @@ public: bool supportsReadAt() override { return true; } private: - std::unique_ptr initialize(); + std::unique_ptr initialize(size_t attempt); /// If true, if we destroy impl now, no work was wasted. Just for metrics. bool atEndOfRequestedRangeGuess(); @@ -88,7 +88,7 @@ private: /// Returns true if the error looks retriable. bool processException(Poco::Exception & e, size_t read_offset, size_t attempt) const; - Aws::S3::Model::GetObjectResult sendRequest(size_t range_begin, std::optional range_end_incl) const; + Aws::S3::Model::GetObjectResult sendRequest(size_t attempt, size_t range_begin, std::optional range_end_incl) const; bool readAllRangeSuccessfully() const; diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index ceb7d275299..4630e68fbb6 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -118,16 +118,9 @@ std::unique_ptr Client::create( new Client(max_redirects_, std::move(sse_kms_config_), credentials_provider, client_configuration, sign_payloads, use_virtual_addressing)); } -std::unique_ptr Client::clone( - std::optional> override_retry_strategy, - std::optional override_request_timeout_ms) const +std::unique_ptr Client::clone() const { - PocoHTTPClientConfiguration new_configuration = client_configuration; - if (override_retry_strategy.has_value()) - new_configuration.retryStrategy = *override_retry_strategy; - if (override_request_timeout_ms.has_value()) - new_configuration.requestTimeoutMs = *override_request_timeout_ms; - return std::unique_ptr(new Client(*this, new_configuration)); + return std::unique_ptr(new Client(*this, client_configuration)); } namespace @@ -905,6 +898,7 @@ PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT s3_retry_attempts, enable_s3_requests_logging, for_disk_s3, + context->getGlobalContext()->getSettingsRef().s3_use_adaptive_timeouts, get_request_throttler, put_request_throttler, error_report); diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index 48310bc21af..5ad57a9d827 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -118,15 +118,7 @@ public: Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy sign_payloads, bool use_virtual_addressing); - /// Create a client with adjusted settings: - /// * override_retry_strategy can be used to disable retries to avoid nested retries when we have - /// a retry loop outside of S3 client. Specifically, for read and write buffers. Currently not - /// actually used. - /// * override_request_timeout_ms is used to increase timeout for CompleteMultipartUploadRequest - /// because it often sits idle for 10 seconds: https://github.com/ClickHouse/ClickHouse/pull/42321 - std::unique_ptr clone( - std::optional> override_retry_strategy = std::nullopt, - std::optional override_request_timeout_ms = std::nullopt) const; + std::unique_ptr clone() const; Client & operator=(const Client &) = delete; diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index d0f248f48a6..4a1b6def133 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -99,6 +99,7 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( unsigned int s3_retry_attempts_, bool enable_s3_requests_logging_, bool for_disk_s3_, + bool s3_use_adaptive_timeouts_, const ThrottlerPtr & get_request_throttler_, const ThrottlerPtr & put_request_throttler_, std::function error_report_) @@ -111,6 +112,7 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( , for_disk_s3(for_disk_s3_) , get_request_throttler(get_request_throttler_) , put_request_throttler(put_request_throttler_) + , s3_use_adaptive_timeouts(s3_use_adaptive_timeouts_) , error_report(error_report_) { } @@ -157,6 +159,7 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & client_config Poco::Timespan(client_configuration.http_keep_alive_timeout_ms * 1000))) /// flag indicating whether keep-alive is enabled is set to each session upon creation , remote_host_filter(client_configuration.remote_host_filter) , s3_max_redirects(client_configuration.s3_max_redirects) + , s3_use_adaptive_timeouts(client_configuration.s3_use_adaptive_timeouts) , enable_s3_requests_logging(client_configuration.enable_s3_requests_logging) , for_disk_s3(client_configuration.for_disk_s3) , get_request_throttler(client_configuration.get_request_throttler) @@ -268,6 +271,38 @@ void PocoHTTPClient::addMetric(const Aws::Http::HttpRequest & request, S3MetricT ProfileEvents::increment(disk_s3_events_map[static_cast(type)][static_cast(kind)], amount); } +String extractAttemptFromInfo(const Aws::String & request_info) +{ + static auto key = Aws::String("attempt="); + + auto key_begin = request_info.find(key, 0); + if (key_begin == Aws::String::npos) + return "1"; + + auto val_begin = key_begin + key.size(); + auto val_end = request_info.find(';', val_begin); + if (val_end == Aws::String::npos) + val_end = request_info.size(); + + return request_info.substr(val_begin, val_end-val_begin); +} + +String getOrEmpty(const Aws::Http::HeaderValueCollection & map, const String & key) +{ + auto it = map.find(key); + if (it == map.end()) + return {}; + return it->second; +} + +ConnectionTimeouts PocoHTTPClient::getTimeouts(const String & method, bool first_attempt, bool first_byte) const +{ + if (!s3_use_adaptive_timeouts) + return timeouts; + + return timeouts.getAdaptiveTimeouts(method, first_attempt, first_byte); +} + void PocoHTTPClient::makeRequestInternal( Aws::Http::HttpRequest & request, std::shared_ptr & response, @@ -282,6 +317,25 @@ void PocoHTTPClient::makeRequestInternal( makeRequestInternalImpl(request, request_configuration, response, readLimiter, writeLimiter); } +String getMethod(const Aws::Http::HttpRequest & request) +{ + switch (request.GetMethod()) + { + case Aws::Http::HttpMethod::HTTP_GET: + return Poco::Net::HTTPRequest::HTTP_GET; + case Aws::Http::HttpMethod::HTTP_POST: + return Poco::Net::HTTPRequest::HTTP_POST; + case Aws::Http::HttpMethod::HTTP_DELETE: + return Poco::Net::HTTPRequest::HTTP_DELETE; + case Aws::Http::HttpMethod::HTTP_PUT: + return Poco::Net::HTTPRequest::HTTP_PUT; + case Aws::Http::HttpMethod::HTTP_HEAD: + return Poco::Net::HTTPRequest::HTTP_HEAD; + case Aws::Http::HttpMethod::HTTP_PATCH: + return Poco::Net::HTTPRequest::HTTP_PATCH; + } +} + template void PocoHTTPClient::makeRequestInternalImpl( Aws::Http::HttpRequest & request, @@ -295,9 +349,14 @@ void PocoHTTPClient::makeRequestInternalImpl( Poco::Logger * log = &Poco::Logger::get("AWSClient"); auto uri = request.GetUri().GetURIString(); + auto method = getMethod(request); + + auto sdk_attempt = extractAttemptFromInfo(getOrEmpty(request.GetHeaders(), Aws::Http::SDK_REQUEST_HEADER)); + auto ch_attempt = extractAttemptFromInfo(getOrEmpty(request.GetHeaders(), "clickhouse-request")); + bool first_attempt = ch_attempt == "1" && sdk_attempt == "1"; if (enable_s3_requests_logging) - LOG_TEST(log, "Make request to: {}", uri); + LOG_TEST(log, "Make request to: {}, aws sdk attempt: {}, clickhouse attempt: {}", uri, sdk_attempt, ch_attempt); switch (request.GetMethod()) { @@ -348,17 +407,29 @@ void PocoHTTPClient::makeRequestInternalImpl( /// This can lead to request signature difference on S3 side. if constexpr (pooled) session = makePooledHTTPSession( - target_uri, timeouts, http_connection_pool_size, wait_on_pool_size_limit, proxy_configuration); + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true), + http_connection_pool_size, + wait_on_pool_size_limit, + proxy_configuration); else - session = makeHTTPSession(target_uri, timeouts, proxy_configuration); + session = makeHTTPSession( + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true), + proxy_configuration); } else { if constexpr (pooled) session = makePooledHTTPSession( - target_uri, timeouts, http_connection_pool_size, wait_on_pool_size_limit); + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true), + http_connection_pool_size, + wait_on_pool_size_limit); else - session = makeHTTPSession(target_uri, timeouts); + session = makeHTTPSession( + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true)); } /// In case of error this address will be written to logs @@ -392,28 +463,7 @@ void PocoHTTPClient::makeRequestInternalImpl( path_and_query = "/"; poco_request.setURI(path_and_query); - - switch (request.GetMethod()) - { - case Aws::Http::HttpMethod::HTTP_GET: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_GET); - break; - case Aws::Http::HttpMethod::HTTP_POST: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_POST); - break; - case Aws::Http::HttpMethod::HTTP_DELETE: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_DELETE); - break; - case Aws::Http::HttpMethod::HTTP_PUT: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_PUT); - break; - case Aws::Http::HttpMethod::HTTP_HEAD: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_HEAD); - break; - case Aws::Http::HttpMethod::HTTP_PATCH: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_PATCH); - break; - } + poco_request.setMethod(method); /// Headers coming from SDK are lower-cased. for (const auto & [header_name, header_value] : request.GetHeaders()) @@ -438,6 +488,7 @@ void PocoHTTPClient::makeRequestInternalImpl( request.GetContentBody()->clear(); request.GetContentBody()->seekg(0); + setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); auto size = Poco::StreamCopier::copyStream(*request.GetContentBody(), request_body_stream); if (enable_s3_requests_logging) LOG_TEST(log, "Written {} bytes to request body", size); @@ -447,6 +498,8 @@ void PocoHTTPClient::makeRequestInternalImpl( LOG_TEST(log, "Receiving response..."); auto & response_body_stream = session->receiveResponse(poco_response); + setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); + watch.stop(); addMetric(request, S3MetricType::Microseconds, watch.elapsedMicroseconds()); @@ -498,6 +551,7 @@ void PocoHTTPClient::makeRequestInternalImpl( /// Request is successful but for some special requests we can have actual error message in body if (status_code >= SUCCESS_RESPONSE_MIN && status_code <= SUCCESS_RESPONSE_MAX && checkRequestCanReturn2xxAndErrorInBody(request)) { + /// reading the full response std::string response_string((std::istreambuf_iterator(response_body_stream)), std::istreambuf_iterator()); @@ -512,7 +566,6 @@ void PocoHTTPClient::makeRequestInternalImpl( addMetric(request, S3MetricType::Errors); if (error_report) error_report(proxy_configuration); - } /// Set response from string @@ -531,6 +584,8 @@ void PocoHTTPClient::makeRequestInternalImpl( if (status_code >= 500 && error_report) error_report(proxy_configuration); } + + /// expose stream, after that client reads data from that stream without built-in retries response->SetResponseBody(response_body_stream, session); } diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 2a449458360..5178d75e7b6 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -55,6 +55,7 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration size_t http_connection_pool_size = 0; /// See PoolBase::BehaviourOnLimit bool wait_on_pool_size_limit = true; + bool s3_use_adaptive_timeouts = true; std::function error_report; @@ -69,6 +70,7 @@ private: unsigned int s3_retry_attempts, bool enable_s3_requests_logging_, bool for_disk_s3_, + bool s3_use_adaptive_timeouts_, const ThrottlerPtr & get_request_throttler_, const ThrottlerPtr & put_request_throttler_, std::function error_report_ @@ -169,6 +171,8 @@ private: Aws::Utils::RateLimits::RateLimiterInterface * readLimiter, Aws::Utils::RateLimits::RateLimiterInterface * writeLimiter) const; + ConnectionTimeouts getTimeouts(const String & method, bool first_attempt, bool first_byte) const; + protected: static S3MetricKind getMetricKind(const Aws::Http::HttpRequest & request); void addMetric(const Aws::Http::HttpRequest & request, S3MetricType type, ProfileEvents::Count amount = 1) const; @@ -178,6 +182,7 @@ protected: ConnectionTimeouts timeouts; const RemoteHostFilter & remote_host_filter; unsigned int s3_max_redirects; + bool s3_use_adaptive_timeouts = true; bool enable_s3_requests_logging; bool for_disk_s3; diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index a16a1a41505..30da1c580c1 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -53,7 +53,6 @@ namespace public: UploadHelper( const std::shared_ptr & client_ptr_, - const std::shared_ptr & client_with_long_timeout_ptr_, const String & dest_bucket_, const String & dest_key_, const S3Settings::RequestSettings & request_settings_, @@ -62,7 +61,6 @@ namespace bool for_disk_s3_, const Poco::Logger * log_) : client_ptr(client_ptr_) - , client_with_long_timeout_ptr(client_with_long_timeout_ptr_) , dest_bucket(dest_bucket_) , dest_key(dest_key_) , request_settings(request_settings_) @@ -78,7 +76,6 @@ namespace protected: std::shared_ptr client_ptr; - std::shared_ptr client_with_long_timeout_ptr; const String & dest_bucket; const String & dest_key; const S3Settings::RequestSettings & request_settings; @@ -179,7 +176,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); - auto outcome = client_with_long_timeout_ptr->CompleteMultipartUpload(request); + auto outcome = client_ptr->CompleteMultipartUpload(request); if (outcome.IsSuccess()) { @@ -433,14 +430,13 @@ namespace size_t offset_, size_t size_, const std::shared_ptr & client_ptr_, - const std::shared_ptr & client_with_long_timeout_ptr_, const String & dest_bucket_, const String & dest_key_, const S3Settings::RequestSettings & request_settings_, const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, bool for_disk_s3_) - : UploadHelper(client_ptr_, client_with_long_timeout_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyDataToS3File")) + : UploadHelper(client_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyDataToS3File")) , create_read_buffer(create_read_buffer_) , offset(offset_) , size(size_) @@ -602,7 +598,6 @@ namespace public: CopyFileHelper( const std::shared_ptr & client_ptr_, - const std::shared_ptr & client_with_long_timeout_ptr_, const String & src_bucket_, const String & src_key_, size_t src_offset_, @@ -614,7 +609,7 @@ namespace const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, bool for_disk_s3_) - : UploadHelper(client_ptr_, client_with_long_timeout_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyS3File")) + : UploadHelper(client_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyS3File")) , src_bucket(src_bucket_) , src_key(src_key_) , offset(src_offset_) @@ -677,7 +672,7 @@ namespace /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 request.SetContentType("binary/octet-stream"); - client_with_long_timeout_ptr->setKMSHeaders(request); + client_ptr->setKMSHeaders(request); } void processCopyRequest(const S3::CopyObjectRequest & request) @@ -689,7 +684,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3CopyObject); - auto outcome = client_with_long_timeout_ptr->CopyObject(request); + auto outcome = client_ptr->CopyObject(request); if (outcome.IsSuccess()) { LOG_TRACE( @@ -714,7 +709,6 @@ namespace offset, size, client_ptr, - client_with_long_timeout_ptr, dest_bucket, dest_key, request_settings, @@ -788,7 +782,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3UploadPartCopy); - auto outcome = client_with_long_timeout_ptr->UploadPartCopy(req); + auto outcome = client_ptr->UploadPartCopy(req); if (!outcome.IsSuccess()) { abortMultipartUpload(); @@ -806,7 +800,6 @@ void copyDataToS3File( size_t offset, size_t size, const std::shared_ptr & dest_s3_client, - const std::shared_ptr & dest_s3_client_with_long_timeout, const String & dest_bucket, const String & dest_key, const S3Settings::RequestSettings & settings, @@ -814,14 +807,13 @@ void copyDataToS3File( ThreadPoolCallbackRunner schedule, bool for_disk_s3) { - CopyDataToFileHelper helper{create_read_buffer, offset, size, dest_s3_client, dest_s3_client_with_long_timeout, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3}; + CopyDataToFileHelper helper{create_read_buffer, offset, size, dest_s3_client, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3}; helper.performCopy(); } void copyS3File( const std::shared_ptr & s3_client, - const std::shared_ptr & s3_client_with_long_timeout, const String & src_bucket, const String & src_key, size_t src_offset, @@ -836,7 +828,7 @@ void copyS3File( { if (settings.allow_native_copy) { - CopyFileHelper helper{s3_client, s3_client_with_long_timeout, src_bucket, src_key, src_offset, src_size, dest_bucket, dest_key, settings, read_settings, object_metadata, schedule, for_disk_s3}; + CopyFileHelper helper{s3_client, src_bucket, src_key, src_offset, src_size, dest_bucket, dest_key, settings, read_settings, object_metadata, schedule, for_disk_s3}; helper.performCopy(); } else @@ -845,7 +837,7 @@ void copyS3File( { return std::make_unique(s3_client, src_bucket, src_key, "", settings, read_settings); }; - copyDataToS3File(create_read_buffer, src_offset, src_size, s3_client, s3_client_with_long_timeout, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3); + copyDataToS3File(create_read_buffer, src_offset, src_size, s3_client, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3); } } diff --git a/src/IO/S3/copyS3File.h b/src/IO/S3/copyS3File.h index 1bcbfd7735e..33e22fdfba2 100644 --- a/src/IO/S3/copyS3File.h +++ b/src/IO/S3/copyS3File.h @@ -27,15 +27,9 @@ using CreateReadBuffer = std::function()>; /// because it is a known issue, it is fallbacks to read-write copy /// (copyDataToS3File()). /// -/// s3_client_with_long_timeout (may be equal to s3_client) is used for native copy and -/// CompleteMultipartUpload requests. These requests need longer timeout because S3 servers often -/// block on them for multiple seconds without sending or receiving data from us (maybe the servers -/// are copying data internally, or maybe throttling, idk). -/// /// read_settings - is used for throttling in case of native copy is not possible void copyS3File( const std::shared_ptr & s3_client, - const std::shared_ptr & s3_client_with_long_timeout, const String & src_bucket, const String & src_key, size_t src_offset, @@ -58,7 +52,6 @@ void copyDataToS3File( size_t offset, size_t size, const std::shared_ptr & dest_s3_client, - const std::shared_ptr & dest_s3_client_with_long_timeout, const String & dest_bucket, const String & dest_key, const S3Settings::RequestSettings & settings, diff --git a/src/IO/S3/tests/gtest_aws_s3_client.cpp b/src/IO/S3/tests/gtest_aws_s3_client.cpp index c42f14e9a53..bff9ca6fa7b 100644 --- a/src/IO/S3/tests/gtest_aws_s3_client.cpp +++ b/src/IO/S3/tests/gtest_aws_s3_client.cpp @@ -91,7 +91,6 @@ void doWriteRequest(std::shared_ptr client, const DB::S3:: DB::S3Settings::RequestSettings request_settings; request_settings.max_unexpected_write_error_retries = max_unexpected_write_error_retries; DB::WriteBufferFromS3 write_buffer( - client, client, uri.bucket, uri.key, @@ -171,6 +170,7 @@ TEST(IOTestAwsS3Client, AppendExtraSSECHeadersRead) "authorization: ... SignedHeaders=" "amz-sdk-invocation-id;" "amz-sdk-request;" + "clickhouse-request;" "content-type;" "host;" "x-amz-api-version;" @@ -216,6 +216,7 @@ TEST(IOTestAwsS3Client, AppendExtraSSEKMSHeadersRead) "authorization: ... SignedHeaders=" "amz-sdk-invocation-id;" "amz-sdk-request;" + "clickhouse-request;" "content-type;" "host;" "x-amz-api-version;" diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index e1b9c17efe9..62d0c80f1f2 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -77,7 +77,6 @@ struct WriteBufferFromS3::PartData WriteBufferFromS3::WriteBufferFromS3( std::shared_ptr client_ptr_, - std::shared_ptr client_with_long_timeout_ptr_, const String & bucket_, const String & key_, size_t buf_size_, @@ -92,7 +91,6 @@ WriteBufferFromS3::WriteBufferFromS3( , upload_settings(request_settings.getUploadSettings()) , write_settings(write_settings_) , client_ptr(std::move(client_ptr_)) - , client_with_long_timeout_ptr(std::move(client_with_long_timeout_ptr_)) , object_metadata(std::move(object_metadata_)) , buffer_allocation_policy(ChooseBufferPolicy(upload_settings)) , task_tracker( @@ -566,7 +564,7 @@ void WriteBufferFromS3::completeMultipartUpload() ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Stopwatch watch; - auto outcome = client_with_long_timeout_ptr->CompleteMultipartUpload(req); + auto outcome = client_ptr->CompleteMultipartUpload(req); watch.stop(); ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Microseconds, watch.elapsedMicroseconds()); diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 95148c49779..590342cc997 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -30,8 +30,6 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase public: WriteBufferFromS3( std::shared_ptr client_ptr_, - /// for CompleteMultipartUploadRequest, because it blocks on recv() for a few seconds on big uploads - std::shared_ptr client_with_long_timeout_ptr_, const String & bucket_, const String & key_, size_t buf_size_, @@ -90,7 +88,6 @@ private: const S3Settings::RequestSettings::PartUploadSettings & upload_settings; const WriteSettings write_settings; const std::shared_ptr client_ptr; - const std::shared_ptr client_with_long_timeout_ptr; const std::optional> object_metadata; Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3"); LogSeriesLimiterPtr limitedLog = std::make_shared(log, 1, 5); diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp index 21bdd9a6f26..c82f97f8b20 100644 --- a/src/IO/tests/gtest_writebuffer_s3.cpp +++ b/src/IO/tests/gtest_writebuffer_s3.cpp @@ -549,7 +549,6 @@ public: getAsyncPolicy().setAutoExecute(false); return std::make_unique( - client, client, bucket, file_name, diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 80ee1e9339d..bdbba5abd96 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -825,7 +825,6 @@ public: write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique( configuration_.client, - configuration_.client_with_long_timeout, bucket, key, DBMS_DEFAULT_BUFFER_SIZE, @@ -1330,8 +1329,6 @@ void StorageS3::Configuration::connect(ContextPtr context) context->getConfigRef().getUInt64("s3.expiration_window_seconds", S3::DEFAULT_EXPIRATION_WINDOW_SECONDS)), auth_settings.no_sign_request.value_or(context->getConfigRef().getBool("s3.no_sign_request", false)), }); - - client_with_long_timeout = client->clone(std::nullopt, request_settings.long_request_timeout_ms); } void StorageS3::processNamedCollectionResult(StorageS3::Configuration & configuration, const NamedCollection & collection) diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 3330ac6c210..3f35c578e19 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -311,7 +311,6 @@ public: HTTPHeaderEntries headers_from_ast; std::shared_ptr client; - std::shared_ptr client_with_long_timeout; std::vector keys; }; diff --git a/src/Storages/StorageS3Settings.h b/src/Storages/StorageS3Settings.h index e3d577ca0b3..728972c948c 100644 --- a/src/Storages/StorageS3Settings.h +++ b/src/Storages/StorageS3Settings.h @@ -69,8 +69,7 @@ struct S3Settings ThrottlerPtr get_request_throttler; ThrottlerPtr put_request_throttler; size_t retry_attempts = 10; - size_t request_timeout_ms = 3000; - size_t long_request_timeout_ms = 30000; // TODO: Take this from config like request_timeout_ms + size_t request_timeout_ms = 30000; bool allow_native_copy = true; bool throw_on_zero_files_match = false; diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml index 206eb4f2bad..4210c13b727 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml @@ -4,6 +4,7 @@ 1000000 + 1 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml index 556bf60d385..95a313ea4f2 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml @@ -4,6 +4,7 @@ 5 + 0 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml index b77e72d808b..7b1f503ed55 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml @@ -7,11 +7,18 @@ + + s3 + http://minio1:9001/root/data/ + minio + minio123 + s3 http://resolver:8083/root/data/ minio minio123 + 1 @@ -23,9 +30,16 @@ + + +
+ s3 +
+ + - broken_s3 + s3 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py index d6bcb3fb8f4..b000ccabcf4 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/test.py +++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py @@ -64,6 +64,8 @@ def test_upload_after_check_works(cluster, broken_s3): data String ) ENGINE=MergeTree() ORDER BY id + SETTINGS + storage_policy='broken_s3' """ ) @@ -78,7 +80,7 @@ def test_upload_after_check_works(cluster, broken_s3): assert "suddenly disappeared" in error, error -def get_counters(node, query_id, log_type="ExceptionWhileProcessing"): +def get_multipart_counters(node, query_id, log_type="ExceptionWhileProcessing"): node.query("SYSTEM FLUSH LOGS") return [ int(x) @@ -87,7 +89,25 @@ def get_counters(node, query_id, log_type="ExceptionWhileProcessing"): SELECT ProfileEvents['S3CreateMultipartUpload'], ProfileEvents['S3UploadPart'], - ProfileEvents['S3WriteRequestsErrors'] + ProfileEvents['S3WriteRequestsErrors'], + FROM system.query_log + WHERE query_id='{query_id}' + AND type='{log_type}' + """ + ).split() + if x + ] + + +def get_put_counters(node, query_id, log_type="ExceptionWhileProcessing"): + node.query("SYSTEM FLUSH LOGS") + return [ + int(x) + for x in node.query( + f""" + SELECT + ProfileEvents['S3PutObject'], + ProfileEvents['S3WriteRequestsErrors'], FROM system.query_log WHERE query_id='{query_id}' AND type='{log_type}' @@ -129,12 +149,12 @@ def test_upload_s3_fail_create_multi_part_upload(cluster, broken_s3, compression assert "Code: 499" in error, error assert "mock s3 injected error" in error, error - count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( + create_multipart, upload_parts, s3_errors = get_multipart_counters( node, insert_query_id ) - assert count_create_multi_part_uploads == 1 - assert count_upload_parts == 0 - assert count_s3_errors == 1 + assert create_multipart == 1 + assert upload_parts == 0 + assert s3_errors == 1 # Add "lz4" compression method in the list after https://github.com/ClickHouse/ClickHouse/issues/50975 is fixed @@ -172,12 +192,12 @@ def test_upload_s3_fail_upload_part_when_multi_part_upload( assert "Code: 499" in error, error assert "mock s3 injected error" in error, error - count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( + create_multipart, upload_parts, s3_errors = get_multipart_counters( node, insert_query_id ) - assert count_create_multi_part_uploads == 1 - assert count_upload_parts >= 2 - assert count_s3_errors >= 2 + assert create_multipart == 1 + assert upload_parts >= 2 + assert s3_errors >= 2 def test_when_s3_connection_refused_is_retried(cluster, broken_s3): @@ -207,12 +227,12 @@ def test_when_s3_connection_refused_is_retried(cluster, broken_s3): query_id=insert_query_id, ) - count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( + create_multipart, upload_parts, s3_errors = get_multipart_counters( node, insert_query_id, log_type="QueryFinish" ) - assert count_create_multi_part_uploads == 1 - assert count_upload_parts == 39 - assert count_s3_errors == 3 + assert create_multipart == 1 + assert upload_parts == 39 + assert s3_errors == 3 broken_s3.setup_at_part_upload(count=1000, after=2, action="connection_refused") insert_query_id = f"INSERT_INTO_TABLE_FUNCTION_CONNECTION_REFUSED_RETRIED_1" @@ -279,13 +299,13 @@ def test_when_s3_connection_reset_by_peer_at_upload_is_retried( query_id=insert_query_id, ) - count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( + create_multipart, upload_parts, s3_errors = get_multipart_counters( node, insert_query_id, log_type="QueryFinish" ) - assert count_create_multi_part_uploads == 1 - assert count_upload_parts == 39 - assert count_s3_errors == 3 + assert create_multipart == 1 + assert upload_parts == 39 + assert s3_errors == 3 broken_s3.setup_at_part_upload( count=1000, @@ -361,13 +381,13 @@ def test_when_s3_connection_reset_by_peer_at_create_mpu_retried( query_id=insert_query_id, ) - count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( + create_multipart, upload_parts, s3_errors = get_multipart_counters( node, insert_query_id, log_type="QueryFinish" ) - assert count_create_multi_part_uploads == 1 - assert count_upload_parts == 39 - assert count_s3_errors == 3 + assert create_multipart == 1 + assert upload_parts == 39 + assert s3_errors == 3 broken_s3.setup_at_create_multi_part_upload( count=1000, @@ -438,13 +458,13 @@ def test_when_s3_broken_pipe_at_upload_is_retried(cluster, broken_s3): query_id=insert_query_id, ) - count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( + create_multipart, upload_parts, s3_errors = get_multipart_counters( node, insert_query_id, log_type="QueryFinish" ) - assert count_create_multi_part_uploads == 1 - assert count_upload_parts == 7 - assert count_s3_errors == 3 + assert create_multipart == 1 + assert upload_parts == 7 + assert s3_errors == 3 broken_s3.setup_at_part_upload( count=1000, @@ -533,3 +553,60 @@ def test_query_is_canceled_with_inf_retries(cluster, broken_s3): retry_count=120, sleep_time=1, ) + + +@pytest.mark.parametrize("node_name", ["node", "node_with_inf_s3_retries"]) +def test_adaptive_timeouts(cluster, broken_s3, node_name): + node = cluster.instances[node_name] + + broken_s3.setup_fake_puts(part_length=1) + broken_s3.setup_slow_answers( + timeout=5, + count=1000000, + ) + + insert_query_id = f"TEST_ADAPTIVE_TIMEOUTS_{node_name}" + node.query( + f""" + INSERT INTO + TABLE FUNCTION s3( + 'http://resolver:8083/root/data/adaptive_timeouts', + 'minio', 'minio123', + 'CSV', auto, 'none' + ) + SELECT + * + FROM system.numbers + LIMIT 1 + SETTINGS + s3_request_timeout_ms=30000, + s3_check_objects_after_upload=0 + """, + query_id=insert_query_id, + ) + + broken_s3.reset() + + put_objects, s3_errors = get_put_counters( + node, insert_query_id, log_type="QueryFinish" + ) + + assert put_objects == 1 + + s3_use_adaptive_timeouts = node.query( + f""" + SELECT + value + FROM system.settings + WHERE + name='s3_use_adaptive_timeouts' + """ + ).strip() + + if node_name == "node_with_inf_s3_retries": + # first 2 attempts failed + assert s3_use_adaptive_timeouts == "1" + assert s3_errors == 1 + else: + assert s3_use_adaptive_timeouts == "0" + assert s3_errors == 0 diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml index 235b9a7b7a1..6303e9273fc 100644 --- a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml @@ -11,6 +11,7 @@ true 0 + 0 20000 @@ -33,6 +34,7 @@ true 1 + 0 1 20000 diff --git a/tests/integration/test_storage_s3/configs/defaultS3.xml b/tests/integration/test_storage_s3/configs/defaultS3.xml index 37454ef6781..7dac6d9fbb5 100644 --- a/tests/integration/test_storage_s3/configs/defaultS3.xml +++ b/tests/integration/test_storage_s3/configs/defaultS3.xml @@ -1,9 +1,4 @@ - - - 5 - - http://resolver:8080 diff --git a/tests/integration/test_storage_s3/configs/s3_retry.xml b/tests/integration/test_storage_s3/configs/s3_retry.xml index 727e23273cf..3171da051d0 100644 --- a/tests/integration/test_storage_s3/configs/s3_retry.xml +++ b/tests/integration/test_storage_s3/configs/s3_retry.xml @@ -1,7 +1,9 @@ - 5 + 1 + 10 + 5 diff --git a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py index 103dd30340c..5ef781bdc9e 100644 --- a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py +++ b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py @@ -4,6 +4,7 @@ import re import socket import struct import sys +import time def gen_n_digit_number(n): @@ -39,14 +40,14 @@ random.seed("Unstable server/1.0") # Generating some "random" data and append a line which contains sum of numbers in column 4. lines = ( - b"".join((gen_line() for _ in range(500000))) + b"".join([gen_line() for _ in range(500000)]) + f"0,0,0,{-sum_in_4_column}\n".encode() ) class RequestHandler(http.server.BaseHTTPRequestHandler): def do_HEAD(self): - if self.path == "/root/test.csv": + if self.path == "/root/test.csv" or self.path == "/root/slow_send_test.csv": self.from_bytes = 0 self.end_bytes = len(lines) self.size = self.end_bytes @@ -101,6 +102,18 @@ class RequestHandler(http.server.BaseHTTPRequestHandler): print("Dropping connection") break + if self.path == "/root/slow_send_test.csv": + self.send_block_size = 81920 + + for c, i in enumerate( + range(self.from_bytes, self.end_bytes, self.send_block_size) + ): + self.wfile.write( + lines[i : min(i + self.send_block_size, self.end_bytes)] + ) + self.wfile.flush() + time.sleep(1) + elif self.path == "/": self.wfile.write(b"OK") diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 3dd3c9e39d0..835c8b908f0 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -818,6 +818,15 @@ def test_storage_s3_get_unstable(started_cluster): assert result.splitlines() == ["500001,500000,0"] +def test_storage_s3_get_slow(started_cluster): + bucket = started_cluster.minio_bucket + instance = started_cluster.instances["dummy"] + table_format = "column1 Int64, column2 Int64, column3 Int64, column4 Int64" + get_query = f"SELECT count(), sum(column3), sum(column4) FROM s3('http://resolver:8081/{started_cluster.minio_bucket}/slow_send_test.csv', 'CSV', '{table_format}') FORMAT CSV" + result = run_query(instance, get_query) + assert result.splitlines() == ["500001,500000,0"] + + def test_storage_s3_put_uncompressed(started_cluster): bucket = started_cluster.minio_bucket instance = started_cluster.instances["dummy"] From a7fc8d4b997359da08cc46fbf66f8aad1de42ed9 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Mon, 20 Nov 2023 15:04:14 +0100 Subject: [PATCH 192/192] test_merge_tree_s3 counts errors, turn off s3_use_adaptive_timeouts --- tests/integration/test_merge_tree_s3/configs/config.d/users.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_merge_tree_s3/configs/config.d/users.xml b/tests/integration/test_merge_tree_s3/configs/config.d/users.xml index 3daa6f06a78..79e5091b28a 100644 --- a/tests/integration/test_merge_tree_s3/configs/config.d/users.xml +++ b/tests/integration/test_merge_tree_s3/configs/config.d/users.xml @@ -3,6 +3,7 @@ 1 20 + 0