diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3dcce68ab46..976c69d3c34 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,7 +12,7 @@ tests/ci/cancel_and_rerun_workflow_lambda/app.py - Backward Incompatible Change - Build/Testing/Packaging Improvement - Documentation (changelog entry is not required) -- Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC) +- Critical Bug Fix (crash, data loss, RBAC) - Bug Fix (user-visible misbehavior in an official stable release) - CI Fix or Improvement (changelog entry is not required) - Not for changelog (changelog entry is not required) diff --git a/.gitmodules b/.gitmodules index bbc8fc7d06c..a3b6450032a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -332,7 +332,7 @@ url = https://github.com/ClickHouse/usearch.git [submodule "contrib/SimSIMD"] path = contrib/SimSIMD - url = https://github.com/ashvardanian/SimSIMD.git + url = https://github.com/ClickHouse/SimSIMD.git [submodule "contrib/FP16"] path = contrib/FP16 url = https://github.com/Maratyszcza/FP16.git diff --git a/CHANGELOG.md b/CHANGELOG.md index 90285582b4e..dacee73440f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -488,6 +488,7 @@ * Remove `is_deterministic` field from the `system.functions` table. [#66630](https://github.com/ClickHouse/ClickHouse/pull/66630) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Function `tuple` will now try to construct named tuples in query (controlled by `enable_named_columns_in_function_tuple`). Introduce function `tupleNames` to extract names from tuples. [#54881](https://github.com/ClickHouse/ClickHouse/pull/54881) ([Amos Bird](https://github.com/amosbird)). * Change how deduplication for Materialized Views works. Fixed a lot of cases like: - on destination table: data is split for 2 or more blocks and that blocks is considered as duplicate when that block is inserted in parallel. - on MV destination table: the equal blocks are deduplicated, that happens when MV often produces equal data as a result for different input data due to performing aggregation. - on MV destination table: the equal blocks which comes from different MV are deduplicated. [#61601](https://github.com/ClickHouse/ClickHouse/pull/61601) ([Sema Checherinda](https://github.com/CheSema)). +* Functions `bitShiftLeft` and `bitShitfRight` return an error for out of bounds shift positions [#65838](https://github.com/ClickHouse/ClickHouse/pull/65838) ([Pablo Marcos](https://github.com/pamarcos)). #### New Feature * Add `ASOF JOIN` support for `full_sorting_join` algorithm. [#55051](https://github.com/ClickHouse/ClickHouse/pull/55051) ([vdimir](https://github.com/vdimir)). @@ -599,7 +600,6 @@ * Functions `bitTest`, `bitTestAll`, and `bitTestAny` now return an error if the specified bit index is out-of-bounds [#65818](https://github.com/ClickHouse/ClickHouse/pull/65818) ([Pablo Marcos](https://github.com/pamarcos)). * Setting `join_any_take_last_row` is supported in any query with hash join. [#65820](https://github.com/ClickHouse/ClickHouse/pull/65820) ([vdimir](https://github.com/vdimir)). * Better handling of join conditions involving `IS NULL` checks (for example `ON (a = b AND (a IS NOT NULL) AND (b IS NOT NULL) ) OR ( (a IS NULL) AND (b IS NULL) )` is rewritten to `ON a <=> b`), fix incorrect optimization when condition other then `IS NULL` are present. [#65835](https://github.com/ClickHouse/ClickHouse/pull/65835) ([vdimir](https://github.com/vdimir)). -* Functions `bitShiftLeft` and `bitShitfRight` return an error for out of bounds shift positions [#65838](https://github.com/ClickHouse/ClickHouse/pull/65838) ([Pablo Marcos](https://github.com/pamarcos)). * Fix growing memory usage in S3Queue. [#65839](https://github.com/ClickHouse/ClickHouse/pull/65839) ([Kseniia Sumarokova](https://github.com/kssenii)). * Fix tie handling in `arrayAUC` to match sklearn. [#65840](https://github.com/ClickHouse/ClickHouse/pull/65840) ([gabrielmcg44](https://github.com/gabrielmcg44)). * Fix possible issues with MySQL server protocol TLS connections. [#65917](https://github.com/ClickHouse/ClickHouse/pull/65917) ([Azat Khuzhin](https://github.com/azat)). diff --git a/CMakeLists.txt b/CMakeLists.txt index f0965530739..a165be799c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -88,6 +88,7 @@ string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES) option (ENABLE_FUZZING "Fuzzy testing using libfuzzer" OFF) +option (ENABLE_FUZZER_TEST "Build testing fuzzers in order to test libFuzzer functionality" OFF) if (ENABLE_FUZZING) # Also set WITH_COVERAGE=1 for better fuzzing process diff --git a/README.md b/README.md index dcaeda13acd..abaf27abf11 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ Upcoming meetups * [Dubai Meetup](https://www.meetup.com/clickhouse-dubai-meetup-group/events/303096989/) - November 21 * [Paris Meetup](https://www.meetup.com/clickhouse-france-user-group/events/303096434) - November 26 * [Amsterdam Meetup](https://www.meetup.com/clickhouse-netherlands-user-group/events/303638814) - December 3 +* [Stockholm Meetup](https://www.meetup.com/clickhouse-stockholm-user-group/events/304382411) - December 9 * [New York Meetup](https://www.meetup.com/clickhouse-new-york-user-group/events/304268174) - December 9 * [San Francisco Meetup](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/304286951/) - December 12 diff --git a/SECURITY.md b/SECURITY.md index db302da8ecd..1b0648dc489 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -14,9 +14,10 @@ The following versions of ClickHouse server are currently supported with securit | Version | Supported | |:-|:-| +| 24.10 | ✔️ | | 24.9 | ✔️ | | 24.8 | ✔️ | -| 24.7 | ✔️ | +| 24.7 | ❌ | | 24.6 | ❌ | | 24.5 | ❌ | | 24.4 | ❌ | diff --git a/base/base/StringRef.h b/base/base/StringRef.h index aa2bce71032..74878b50545 100644 --- a/base/base/StringRef.h +++ b/base/base/StringRef.h @@ -86,7 +86,7 @@ using StringRefs = std::vector; * For more information, see hash_map_string_2.cpp */ -inline bool compare8(const char * p1, const char * p2) +inline bool compare16(const char * p1, const char * p2) { return 0xFFFF == _mm_movemask_epi8(_mm_cmpeq_epi8( _mm_loadu_si128(reinterpret_cast(p1)), @@ -115,7 +115,7 @@ inline bool compare64(const char * p1, const char * p2) #elif defined(__aarch64__) && defined(__ARM_NEON) -inline bool compare8(const char * p1, const char * p2) +inline bool compare16(const char * p1, const char * p2) { uint64_t mask = getNibbleMask(vceqq_u8( vld1q_u8(reinterpret_cast(p1)), vld1q_u8(reinterpret_cast(p2)))); @@ -185,13 +185,22 @@ inline bool memequalWide(const char * p1, const char * p2, size_t size) switch (size / 16) // NOLINT(bugprone-switch-missing-default-case) { - case 3: if (!compare8(p1 + 32, p2 + 32)) return false; [[fallthrough]]; - case 2: if (!compare8(p1 + 16, p2 + 16)) return false; [[fallthrough]]; - case 1: if (!compare8(p1, p2)) return false; [[fallthrough]]; + case 3: + if (!compare16(p1 + 32, p2 + 32)) + return false; + [[fallthrough]]; + case 2: + if (!compare16(p1 + 16, p2 + 16)) + return false; + [[fallthrough]]; + case 1: + if (!compare16(p1, p2)) + return false; + [[fallthrough]]; default: ; } - return compare8(p1 + size - 16, p2 + size - 16); + return compare16(p1 + size - 16, p2 + size - 16); } #endif diff --git a/base/base/defines.h b/base/base/defines.h index 5685a6d9833..a0c3c0d1de5 100644 --- a/base/base/defines.h +++ b/base/base/defines.h @@ -145,6 +145,7 @@ #define TSA_TRY_ACQUIRE_SHARED(...) __attribute__((try_acquire_shared_capability(__VA_ARGS__))) /// function tries to acquire a shared capability and returns a boolean value indicating success or failure #define TSA_RELEASE_SHARED(...) __attribute__((release_shared_capability(__VA_ARGS__))) /// function releases the given shared capability #define TSA_SCOPED_LOCKABLE __attribute__((scoped_lockable)) /// object of a class has scoped lockable capability +#define TSA_RETURN_CAPABILITY(...) __attribute__((lock_returned(__VA_ARGS__))) /// to return capabilities in functions /// Macros for suppressing TSA warnings for specific reads/writes (instead of suppressing it for the whole function) /// They use a lambda function to apply function attribute to a single statement. This enable us to suppress warnings locally instead of diff --git a/contrib/SimSIMD b/contrib/SimSIMD index ff51434d90c..fa60f1b8e35 160000 --- a/contrib/SimSIMD +++ b/contrib/SimSIMD @@ -1 +1 @@ -Subproject commit ff51434d90c66f916e94ff05b24530b127aa4cff +Subproject commit fa60f1b8e3582c50978f0ae86c2ebb6c9af957f3 diff --git a/contrib/SimSIMD-cmake/CMakeLists.txt b/contrib/SimSIMD-cmake/CMakeLists.txt index f5dc4d63604..8350417479a 100644 --- a/contrib/SimSIMD-cmake/CMakeLists.txt +++ b/contrib/SimSIMD-cmake/CMakeLists.txt @@ -1,4 +1,8 @@ -set(SIMSIMD_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/SimSIMD") - -add_library(_simsimd INTERFACE) -target_include_directories(_simsimd SYSTEM INTERFACE "${SIMSIMD_PROJECT_DIR}/include") +# See contrib/usearch-cmake/CMakeLists.txt, why only enabled on x86 +if (ARCH_AMD64) + set(SIMSIMD_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/SimSIMD") + set(SIMSIMD_SRCS ${SIMSIMD_PROJECT_DIR}/c/lib.c) + add_library(_simsimd ${SIMSIMD_SRCS}) + target_include_directories(_simsimd SYSTEM PUBLIC "${SIMSIMD_PROJECT_DIR}/include") + target_compile_definitions(_simsimd PUBLIC SIMSIMD_DYNAMIC_DISPATCH) +endif() diff --git a/contrib/krb5 b/contrib/krb5 index 71b06c22760..c5b4b994c18 160000 --- a/contrib/krb5 +++ b/contrib/krb5 @@ -1 +1 @@ -Subproject commit 71b06c2276009ae649c7703019f3b4605f66fd3d +Subproject commit c5b4b994c18db86933255907a97eee5993fd18fe diff --git a/contrib/usearch b/contrib/usearch index 1706420acaf..7efe8b710c9 160000 --- a/contrib/usearch +++ b/contrib/usearch @@ -1 +1 @@ -Subproject commit 1706420acafbd83d852c512dcf343af0a4059e48 +Subproject commit 7efe8b710c9831bfe06573b1df0fad001b04a2b5 diff --git a/contrib/usearch-cmake/CMakeLists.txt b/contrib/usearch-cmake/CMakeLists.txt index 25f6ca82a74..fda061bf467 100644 --- a/contrib/usearch-cmake/CMakeLists.txt +++ b/contrib/usearch-cmake/CMakeLists.txt @@ -6,12 +6,63 @@ target_include_directories(_usearch SYSTEM INTERFACE ${USEARCH_PROJECT_DIR}/incl target_link_libraries(_usearch INTERFACE _fp16) target_compile_definitions(_usearch INTERFACE USEARCH_USE_FP16LIB) -# target_compile_definitions(_usearch INTERFACE USEARCH_USE_SIMSIMD) -# ^^ simsimd is not enabled at the moment. Reasons: -# - Vectorization is important for raw scans but not so much for HNSW. We use usearch only for HNSW. -# - Simsimd does compile-time dispatch (choice of SIMD kernels determined by capabilities of the build machine) or dynamic dispatch (SIMD -# kernels chosen at runtime based on cpuid instruction). Since current builds are limited to SSE 4.2 (x86) and NEON (ARM), the speedup of -# the former would be moderate compared to AVX-512 / SVE. The latter is at the moment too fragile with respect to portability across x86 -# and ARM machines ... certain conbinations of quantizations / distance functions / SIMD instructions are not implemented at the moment. +# Only x86 for now. On ARM, the linker goes down in flames. To make SimSIMD compile, I had to remove a macro checks in SimSIMD +# for AVX512 (x86, worked nicely) and __ARM_BF16_FORMAT_ALTERNATIVE. It is probably because of that. +if (ARCH_AMD64) + target_link_libraries(_usearch INTERFACE _simsimd) + target_compile_definitions(_usearch INTERFACE USEARCH_USE_SIMSIMD) + + target_compile_definitions(_usearch INTERFACE USEARCH_CAN_COMPILE_FLOAT16) + target_compile_definitions(_usearch INTERFACE USEARCH_CAN_COMPILE_BF16) +endif () add_library(ch_contrib::usearch ALIAS _usearch) + + +# Cf. https://github.com/llvm/llvm-project/issues/107810 (though it is not 100% the same stack) +# +# LLVM ERROR: Cannot select: 0x7996e7a73150: f32,ch = load<(load (s16) from %ir.22, !tbaa !54231), anyext from bf16> 0x79961cb737c0, 0x7996e7a1a500, undef:i64, ./contrib/SimSIMD/include/simsimd/dot.h:215:1 +# 0x7996e7a1a500: i64 = add 0x79961e770d00, Constant:i64<-16>, ./contrib/SimSIMD/include/simsimd/dot.h:215:1 +# 0x79961e770d00: i64,ch = CopyFromReg 0x79961cb737c0, Register:i64 %4, ./contrib/SimSIMD/include/simsimd/dot.h:215:1 +# 0x7996e7a1ae10: i64 = Register %4 +# 0x7996e7a1b5f0: i64 = Constant<-16> +# 0x7996e7a1a730: i64 = undef +# In function: _ZL23simsimd_dot_bf16_serialPKu6__bf16S0_yPd +# PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace. +# Stack dump: +# 0. Running pass 'Function Pass Manager' on module 'src/libdbms.a(MergeTreeIndexVectorSimilarity.cpp.o at 2312737440)'. +# 1. Running pass 'AArch64 Instruction Selection' on function '@_ZL23simsimd_dot_bf16_serialPKu6__bf16S0_yPd' +# #0 0x00007999e83a63bf llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xda63bf) +# #1 0x00007999e83a44f9 llvm::sys::RunSignalHandlers() (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xda44f9) +# #2 0x00007999e83a6b00 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xda6b00) +# #3 0x00007999e6e45320 (/lib/x86_64-linux-gnu/libc.so.6+0x45320) +# #4 0x00007999e6e9eb1c pthread_kill (/lib/x86_64-linux-gnu/libc.so.6+0x9eb1c) +# #5 0x00007999e6e4526e raise (/lib/x86_64-linux-gnu/libc.so.6+0x4526e) +# #6 0x00007999e6e288ff abort (/lib/x86_64-linux-gnu/libc.so.6+0x288ff) +# #7 0x00007999e82fe0c2 llvm::report_fatal_error(llvm::Twine const&, bool) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xcfe0c2) +# #8 0x00007999e8c2f8e3 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x162f8e3) +# #9 0x00007999e8c2ed76 llvm::SelectionDAGISel::SelectCodeCommon(llvm::SDNode*, unsigned char const*, unsigned int) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x162ed76) +# #10 0x00007999ea1adbcb (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x2badbcb) +# #11 0x00007999e8c2611f llvm::SelectionDAGISel::DoInstructionSelection() (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x162611f) +# #12 0x00007999e8c25790 llvm::SelectionDAGISel::CodeGenAndEmitDAG() (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x1625790) +# #13 0x00007999e8c248de llvm::SelectionDAGISel::SelectAllBasicBlocks(llvm::Function const&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x16248de) +# #14 0x00007999e8c22934 llvm::SelectionDAGISel::runOnMachineFunction(llvm::MachineFunction&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x1622934) +# #15 0x00007999e87826b9 llvm::MachineFunctionPass::runOnFunction(llvm::Function&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x11826b9) +# #16 0x00007999e84f7772 llvm::FPPassManager::runOnFunction(llvm::Function&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xef7772) +# #17 0x00007999e84fd2f4 llvm::FPPassManager::runOnModule(llvm::Module&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xefd2f4) +# #18 0x00007999e84f7e9f llvm::legacy::PassManagerImpl::run(llvm::Module&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xef7e9f) +# #19 0x00007999e99f7d61 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f7d61) +# #20 0x00007999e99f8c91 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f8c91) +# #21 0x00007999e99f8b10 llvm::lto::thinBackend(llvm::lto::Config const&, unsigned int, std::function>> (unsigned int, llvm::Twine const&)>, llvm::Module&, llvm::ModuleSummaryIndex const&, llvm::DenseMap, std::equal_to, std::allocator>, llvm::DenseMapInfo, llvm::detail::DenseMapPair, std::equal_to, std::allocator>>> const&, llvm::DenseMap, llvm::detail::DenseMapPair> const&, llvm::MapVector, llvm::detail::DenseMapPair>, llvm::SmallVector, 0u>>*, std::vector> const&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f8b10) +# #22 0x00007999e99f248d (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f248d) +# #23 0x00007999e99f1cd6 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f1cd6) +# #24 0x00007999e82c9beb (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xcc9beb) +# #25 0x00007999e834ebe3 llvm::ThreadPool::processTasks(llvm::ThreadPoolTaskGroup*) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xd4ebe3) +# #26 0x00007999e834f704 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xd4f704) +# #27 0x00007999e6e9ca94 (/lib/x86_64-linux-gnu/libc.so.6+0x9ca94) +# #28 0x00007999e6f29c3c (/lib/x86_64-linux-gnu/libc.so.6+0x129c3c) +# clang++-18: error: unable to execute command: Aborted (core dumped) +# clang++-18: error: linker command failed due to signal (use -v to see invocation) +# ^[[A^Cninja: build stopped: interrupted by user. diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index dfe6a420260..4ecc087afb4 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -1,7 +1,7 @@ # The Dockerfile.ubuntu exists for the tests/ci/docker_server.py script # If the image is built from Dockerfile.alpine, then the `-alpine` suffix is added automatically, # so the only purpose of Dockerfile.ubuntu is to push `latest`, `head` and so on w/o suffixes -FROM ubuntu:20.04 AS glibc-donor +FROM ubuntu:22.04 AS glibc-donor ARG TARGETARCH RUN arch=${TARGETARCH:-amd64} \ @@ -9,7 +9,11 @@ RUN arch=${TARGETARCH:-amd64} \ amd64) rarch=x86_64 ;; \ arm64) rarch=aarch64 ;; \ esac \ - && ln -s "${rarch}-linux-gnu" /lib/linux-gnu + && ln -s "${rarch}-linux-gnu" /lib/linux-gnu \ + && case $arch in \ + amd64) ln /lib/linux-gnu/ld-linux-x86-64.so.2 /lib/linux-gnu/ld-2.35.so ;; \ + arm64) ln /lib/linux-gnu/ld-linux-aarch64.so.1 /lib/linux-gnu/ld-2.35.so ;; \ + esac FROM alpine @@ -20,7 +24,7 @@ ENV LANG=en_US.UTF-8 \ TZ=UTC \ CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.xml -COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.31.so /lib/ +COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.35.so /lib/ COPY --from=glibc-donor /etc/nsswitch.conf /etc/ COPY entrypoint.sh /entrypoint.sh @@ -34,7 +38,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="24.9.2.42" +ARG VERSION="24.10.1.2812" ARG PACKAGES="clickhouse-keeper" ARG DIRECT_DOWNLOAD_URLS="" diff --git a/docker/keeper/entrypoint.sh b/docker/keeper/entrypoint.sh index 68bd0ef9d87..934605b0b6f 100644 --- a/docker/keeper/entrypoint.sh +++ b/docker/keeper/entrypoint.sh @@ -1,21 +1,31 @@ #!/bin/bash -set +x set -eo pipefail shopt -s nullglob DO_CHOWN=1 -if [ "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]; then +if [[ "${CLICKHOUSE_RUN_AS_ROOT:=0}" = "1" || "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]]; then DO_CHOWN=0 fi -CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}" -CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}" +# CLICKHOUSE_UID and CLICKHOUSE_GID are kept for backward compatibility, but deprecated +# One must use either "docker run --user" or CLICKHOUSE_RUN_AS_ROOT=1 to run the process as +# FIXME: Remove ALL CLICKHOUSE_UID CLICKHOUSE_GID before 25.3 +if [[ "${CLICKHOUSE_UID:-}" || "${CLICKHOUSE_GID:-}" ]]; then + echo 'WARNING: Support for CLICKHOUSE_UID/CLICKHOUSE_GID will be removed in a couple of releases.' >&2 + echo 'WARNING: Either use a proper "docker run --user=xxx:xxxx" argument instead of CLICKHOUSE_UID/CLICKHOUSE_GID' >&2 + echo 'WARNING: or set "CLICKHOUSE_RUN_AS_ROOT=1" ENV to run the clickhouse-server as root:root' >&2 +fi -# support --user -if [ "$(id -u)" = "0" ]; then - USER=$CLICKHOUSE_UID - GROUP=$CLICKHOUSE_GID +# support `docker run --user=xxx:xxxx` +if [[ "$(id -u)" = "0" ]]; then + if [[ "$CLICKHOUSE_RUN_AS_ROOT" = 1 ]]; then + USER=0 + GROUP=0 + else + USER="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}" + GROUP="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}" + fi if command -v gosu &> /dev/null; then gosu="gosu $USER:$GROUP" elif command -v su-exec &> /dev/null; then @@ -82,11 +92,11 @@ if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then # There is a config file. It is already tested with gosu (if it is readably by keeper user) if [ -f "$KEEPER_CONFIG" ]; then - exec $gosu /usr/bin/clickhouse-keeper --config-file="$KEEPER_CONFIG" "$@" + exec $gosu clickhouse-keeper --config-file="$KEEPER_CONFIG" "$@" fi # There is no config file. Will use embedded one - exec $gosu /usr/bin/clickhouse-keeper --log-file="$LOG_PATH" --errorlog-file="$ERROR_LOG_PATH" "$@" + exec $gosu clickhouse-keeper --log-file="$LOG_PATH" --errorlog-file="$ERROR_LOG_PATH" "$@" fi # Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 991c25ad142..93acf1a5773 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -35,7 +35,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="24.9.2.42" +ARG VERSION="24.10.1.2812" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" ARG DIRECT_DOWNLOAD_URLS="" diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 5dc88b49e31..e6bde845c4e 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 # see https://github.com/moby/moby/issues/4032#issuecomment-192327844 # It could be removed after we move on a version 23:04+ @@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="24.9.2.42" +ARG VERSION="24.10.1.2812" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" #docker-official-library:off @@ -88,34 +88,34 @@ RUN if [ -n "${single_binary_location_url}" ]; then \ #docker-official-library:on # A fallback to installation from ClickHouse repository -RUN if ! clickhouse local -q "SELECT ''" > /dev/null 2>&1; then \ - apt-get update \ - && apt-get install --yes --no-install-recommends \ - apt-transport-https \ - dirmngr \ - gnupg2 \ - && mkdir -p /etc/apt/sources.list.d \ - && GNUPGHOME=$(mktemp -d) \ - && GNUPGHOME="$GNUPGHOME" gpg --batch --no-default-keyring \ - --keyring /usr/share/keyrings/clickhouse-keyring.gpg \ - --keyserver hkp://keyserver.ubuntu.com:80 \ - --recv-keys 3a9ea1193a97b548be1457d48919f6bd2b48d754 \ - && rm -rf "$GNUPGHOME" \ - && chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \ - && echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \ - && echo "installing from repository: ${REPOSITORY}" \ - && apt-get update \ - && for package in ${PACKAGES}; do \ - packages="${packages} ${package}=${VERSION}" \ - ; done \ - && apt-get install --allow-unauthenticated --yes --no-install-recommends ${packages} || exit 1 \ - && rm -rf \ - /var/lib/apt/lists/* \ - /var/cache/debconf \ - /tmp/* \ - && apt-get autoremove --purge -yq libksba8 \ - && apt-get autoremove -yq \ - ; fi +# It works unless the clickhouse binary already exists +RUN clickhouse local -q 'SELECT 1' >/dev/null 2>&1 && exit 0 || : \ + ; apt-get update \ + && apt-get install --yes --no-install-recommends \ + dirmngr \ + gnupg2 \ + && mkdir -p /etc/apt/sources.list.d \ + && GNUPGHOME=$(mktemp -d) \ + && GNUPGHOME="$GNUPGHOME" gpg --batch --no-default-keyring \ + --keyring /usr/share/keyrings/clickhouse-keyring.gpg \ + --keyserver hkp://keyserver.ubuntu.com:80 \ + --recv-keys 3a9ea1193a97b548be1457d48919f6bd2b48d754 \ + && rm -rf "$GNUPGHOME" \ + && chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \ + && echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \ + && echo "installing from repository: ${REPOSITORY}" \ + && apt-get update \ + && for package in ${PACKAGES}; do \ + packages="${packages} ${package}=${VERSION}" \ + ; done \ + && apt-get install --yes --no-install-recommends ${packages} || exit 1 \ + && rm -rf \ + /var/lib/apt/lists/* \ + /var/cache/debconf \ + /tmp/* \ + && apt-get autoremove --purge -yq dirmngr gnupg2 \ + && chmod ugo+Xrw -R /etc/clickhouse-server /etc/clickhouse-client +# The last chmod is here to make the next one is No-op in docker official library Dockerfile # post install # we need to allow "others" access to clickhouse folder, because docker container @@ -126,8 +126,6 @@ RUN clickhouse-local -q 'SELECT * FROM system.build_options' \ RUN locale-gen en_US.UTF-8 ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 ENV TZ UTC RUN mkdir /docker-entrypoint-initdb.d diff --git a/docker/server/README.md b/docker/server/README.md index 65239126790..5f6144d0633 100644 --- a/docker/server/README.md +++ b/docker/server/README.md @@ -1,3 +1,11 @@ + + # ClickHouse Server Docker Image ## What is ClickHouse? @@ -8,6 +16,7 @@ ClickHouse works 100-1000x faster than traditional database management systems, For more information and documentation see https://clickhouse.com/. + ## Versions - The `latest` tag points to the latest release of the latest stable branch. @@ -16,10 +25,12 @@ For more information and documentation see https://clickhouse.com/. - The tag `head` is built from the latest commit to the default branch. - Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`. + ### Compatibility - The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3. - The arm64 image requires support for the [ARMv8.2-A architecture](https://en.wikipedia.org/wiki/AArch64#ARMv8.2-A) and additionally the Load-Acquire RCpc register. The register is optional in version ARMv8.2-A and mandatory in [ARMv8.3-A](https://en.wikipedia.org/wiki/AArch64#ARMv8.3-A). Supported in Graviton >=2, Azure and GCP instances. Examples for unsupported devices are Raspberry Pi 4 (ARMv8.0-A) and Jetson AGX Xavier/Orin (ARMv8.2-A). +- Since the Clickhouse 24.11 Ubuntu images started using `ubuntu:22.04` as its base image. It requires docker version >= `20.10.10` containing [patch](https://github.com/moby/moby/commit/977283509f75303bc6612665a04abf76ff1d2468). As a workaround you could use `docker run --security-opt seccomp=unconfined` instead, however that has security implications. ## How to use this image @@ -29,7 +40,7 @@ For more information and documentation see https://clickhouse.com/. docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server ``` -By default, ClickHouse will be accessible only via the Docker network. See the [networking section below](#networking). +By default, ClickHouse will be accessible only via the Docker network. See the **networking** section below. By default, starting above server instance will be run as the `default` user without password. @@ -46,7 +57,7 @@ More information about the [ClickHouse client](https://clickhouse.com/docs/en/in ### connect to it using curl ```bash -echo "SELECT 'Hello, ClickHouse!'" | docker run -i --rm --link some-clickhouse-server:clickhouse-server curlimages/curl 'http://clickhouse-server:8123/?query=' -s --data-binary @- +echo "SELECT 'Hello, ClickHouse!'" | docker run -i --rm --link some-clickhouse-server:clickhouse-server buildpack-deps:curl curl 'http://clickhouse-server:8123/?query=' -s --data-binary @- ``` More information about the [ClickHouse HTTP Interface](https://clickhouse.com/docs/en/interfaces/http/). @@ -69,7 +80,7 @@ echo 'SELECT version()' | curl 'http://localhost:18123/' --data-binary @- `22.6.3.35` -or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows achieving better network performance): +Or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows achieving better network performance): ```bash docker run -d --network=host --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server @@ -87,8 +98,8 @@ Typically you may want to mount the following folders inside your container to a ```bash docker run -d \ - -v $(realpath ./ch_data):/var/lib/clickhouse/ \ - -v $(realpath ./ch_logs):/var/log/clickhouse-server/ \ + -v "$PWD/ch_data:/var/lib/clickhouse/" \ + -v "$PWD/ch_logs:/var/log/clickhouse-server/" \ --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server ``` @@ -110,6 +121,8 @@ docker run -d \ --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server ``` +Read more in [knowledge base](https://clickhouse.com/docs/knowledgebase/configure_cap_ipc_lock_and_cap_sys_nice_in_docker). + ## Configuration The container exposes port 8123 for the [HTTP interface](https://clickhouse.com/docs/en/interfaces/http_interface/) and port 9000 for the [native client](https://clickhouse.com/docs/en/interfaces/tcp/). @@ -125,8 +138,8 @@ docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /pa ### Start server as custom user ```bash -# $(pwd)/data/clickhouse should exist and be owned by current user -docker run --rm --user ${UID}:${GID} --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server +# $PWD/data/clickhouse should exist and be owned by current user +docker run --rm --user "${UID}:${GID}" --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server ``` When you use the image with local directories mounted, you probably want to specify the user to maintain the proper file ownership. Use the `--user` argument and mount `/var/lib/clickhouse` and `/var/log/clickhouse-server` inside the container. Otherwise, the image will complain and not start. @@ -134,7 +147,7 @@ When you use the image with local directories mounted, you probably want to spec ### Start server from root (useful in case of enabled user namespace) ```bash -docker run --rm -e CLICKHOUSE_UID=0 -e CLICKHOUSE_GID=0 --name clickhouse-server-userns -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server +docker run --rm -e CLICKHOUSE_RUN_AS_ROOT=1 --name clickhouse-server-userns -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server ``` ### How to create default database and user on starting diff --git a/docker/server/README.sh b/docker/server/README.sh new file mode 100755 index 00000000000..42fa72404d1 --- /dev/null +++ b/docker/server/README.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -ueo pipefail + +# A script to generate README.sh close to as it done in https://github.com/docker-library/docs + +WORKDIR=$(dirname "$0") +SCRIPT_NAME=$(basename "$0") +CONTENT=README.src/content.md +LICENSE=README.src/license.md +cd "$WORKDIR" + +R=README.md + +cat > "$R" < + +EOD + +cat "$CONTENT" >> "$R" + +cat >> "$R" < +## Versions + +- The `latest` tag points to the latest release of the latest stable branch. +- Branch tags like `22.2` point to the latest release of the corresponding branch. +- Full version tags like `22.2.3.5` point to the corresponding release. +- The tag `head` is built from the latest commit to the default branch. +- Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`. + + +### Compatibility + +- The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3. +- The arm64 image requires support for the [ARMv8.2-A architecture](https://en.wikipedia.org/wiki/AArch64#ARMv8.2-A) and additionally the Load-Acquire RCpc register. The register is optional in version ARMv8.2-A and mandatory in [ARMv8.3-A](https://en.wikipedia.org/wiki/AArch64#ARMv8.3-A). Supported in Graviton >=2, Azure and GCP instances. Examples for unsupported devices are Raspberry Pi 4 (ARMv8.0-A) and Jetson AGX Xavier/Orin (ARMv8.2-A). +- Since the Clickhouse 24.11 Ubuntu images started using `ubuntu:22.04` as its base image. It requires docker version >= `20.10.10` containing [patch](https://github.com/moby/moby/commit/977283509f75303bc6612665a04abf76ff1d2468). As a workaround you could use `docker run --security-opt seccomp=unconfined` instead, however that has security implications. + +## How to use this image + +### start server instance + +```bash +docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%% +``` + +By default, ClickHouse will be accessible only via the Docker network. See the **networking** section below. + +By default, starting above server instance will be run as the `default` user without password. + +### connect to it from a native client + +```bash +docker run -it --rm --link some-clickhouse-server:clickhouse-server --entrypoint clickhouse-client %%IMAGE%% --host clickhouse-server +# OR +docker exec -it some-clickhouse-server clickhouse-client +``` + +More information about the [ClickHouse client](https://clickhouse.com/docs/en/interfaces/cli/). + +### connect to it using curl + +```bash +echo "SELECT 'Hello, ClickHouse!'" | docker run -i --rm --link some-clickhouse-server:clickhouse-server buildpack-deps:curl curl 'http://clickhouse-server:8123/?query=' -s --data-binary @- +``` + +More information about the [ClickHouse HTTP Interface](https://clickhouse.com/docs/en/interfaces/http/). + +### stopping / removing the container + +```bash +docker stop some-clickhouse-server +docker rm some-clickhouse-server +``` + +### networking + +You can expose your ClickHouse running in docker by [mapping a particular port](https://docs.docker.com/config/containers/container-networking/) from inside the container using host ports: + +```bash +docker run -d -p 18123:8123 -p19000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%% +echo 'SELECT version()' | curl 'http://localhost:18123/' --data-binary @- +``` + +`22.6.3.35` + +Or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows achieving better network performance): + +```bash +docker run -d --network=host --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%% +echo 'SELECT version()' | curl 'http://localhost:8123/' --data-binary @- +``` + +`22.6.3.35` + +### Volumes + +Typically you may want to mount the following folders inside your container to achieve persistency: + +- `/var/lib/clickhouse/` - main folder where ClickHouse stores the data +- `/var/log/clickhouse-server/` - logs + +```bash +docker run -d \ + -v "$PWD/ch_data:/var/lib/clickhouse/" \ + -v "$PWD/ch_logs:/var/log/clickhouse-server/" \ + --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%% +``` + +You may also want to mount: + +- `/etc/clickhouse-server/config.d/*.xml` - files with server configuration adjustments +- `/etc/clickhouse-server/users.d/*.xml` - files with user settings adjustments +- `/docker-entrypoint-initdb.d/` - folder with database initialization scripts (see below). + +### Linux capabilities + +ClickHouse has some advanced functionality, which requires enabling several [Linux capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html). + +They are optional and can be enabled using the following [docker command-line arguments](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities): + +```bash +docker run -d \ + --cap-add=SYS_NICE --cap-add=NET_ADMIN --cap-add=IPC_LOCK \ + --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%% +``` + +Read more in [knowledge base](https://clickhouse.com/docs/knowledgebase/configure_cap_ipc_lock_and_cap_sys_nice_in_docker). + +## Configuration + +The container exposes port 8123 for the [HTTP interface](https://clickhouse.com/docs/en/interfaces/http_interface/) and port 9000 for the [native client](https://clickhouse.com/docs/en/interfaces/tcp/). + +ClickHouse configuration is represented with a file "config.xml" ([documentation](https://clickhouse.com/docs/en/operations/configuration_files/)) + +### Start server instance with custom configuration + +```bash +docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /path/to/your/config.xml:/etc/clickhouse-server/config.xml %%IMAGE%% +``` + +### Start server as custom user + +```bash +# $PWD/data/clickhouse should exist and be owned by current user +docker run --rm --user "${UID}:${GID}" --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" %%IMAGE%% +``` + +When you use the image with local directories mounted, you probably want to specify the user to maintain the proper file ownership. Use the `--user` argument and mount `/var/lib/clickhouse` and `/var/log/clickhouse-server` inside the container. Otherwise, the image will complain and not start. + +### Start server from root (useful in case of enabled user namespace) + +```bash +docker run --rm -e CLICKHOUSE_RUN_AS_ROOT=1 --name clickhouse-server-userns -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" %%IMAGE%% +``` + +### How to create default database and user on starting + +Sometimes you may want to create a user (user named `default` is used by default) and database on a container start. You can do it using environment variables `CLICKHOUSE_DB`, `CLICKHOUSE_USER`, `CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT` and `CLICKHOUSE_PASSWORD`: + +```bash +docker run --rm -e CLICKHOUSE_DB=my_database -e CLICKHOUSE_USER=username -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -e CLICKHOUSE_PASSWORD=password -p 9000:9000/tcp %%IMAGE%% +``` + +## How to extend this image + +To perform additional initialization in an image derived from this one, add one or more `*.sql`, `*.sql.gz`, or `*.sh` scripts under `/docker-entrypoint-initdb.d`. After the entrypoint calls `initdb`, it will run any `*.sql` files, run any executable `*.sh` scripts, and source any non-executable `*.sh` scripts found in that directory to do further initialization before starting the service. +Also, you can provide environment variables `CLICKHOUSE_USER` & `CLICKHOUSE_PASSWORD` that will be used for clickhouse-client during initialization. + +For example, to add an additional user and database, add the following to `/docker-entrypoint-initdb.d/init-db.sh`: + +```bash +#!/bin/bash +set -e + +clickhouse client -n <<-EOSQL + CREATE DATABASE docker; + CREATE TABLE docker.docker (x Int32) ENGINE = Log; +EOSQL +``` diff --git a/docker/server/README.src/github-repo b/docker/server/README.src/github-repo new file mode 100644 index 00000000000..70a009ec958 --- /dev/null +++ b/docker/server/README.src/github-repo @@ -0,0 +1 @@ +https://github.com/ClickHouse/ClickHouse diff --git a/docker/server/README.src/license.md b/docker/server/README.src/license.md new file mode 100644 index 00000000000..6be024edcde --- /dev/null +++ b/docker/server/README.src/license.md @@ -0,0 +1 @@ +View [license information](https://github.com/ClickHouse/ClickHouse/blob/master/LICENSE) for the software contained in this image. diff --git a/docker/server/README.src/logo.svg b/docker/server/README.src/logo.svg new file mode 100644 index 00000000000..a50dd81a164 --- /dev/null +++ b/docker/server/README.src/logo.svg @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docker/server/README.src/maintainer.md b/docker/server/README.src/maintainer.md new file mode 100644 index 00000000000..26c7db1a293 --- /dev/null +++ b/docker/server/README.src/maintainer.md @@ -0,0 +1 @@ +[ClickHouse Inc.](%%GITHUB-REPO%%) diff --git a/docker/server/README.src/metadata.json b/docker/server/README.src/metadata.json new file mode 100644 index 00000000000..3d3937b21fb --- /dev/null +++ b/docker/server/README.src/metadata.json @@ -0,0 +1,7 @@ +{ + "hub": { + "categories": [ + "databases-and-storage" + ] + } +} diff --git a/docker/server/entrypoint.sh b/docker/server/entrypoint.sh index 3102ab8297c..947244dd97f 100755 --- a/docker/server/entrypoint.sh +++ b/docker/server/entrypoint.sh @@ -4,17 +4,28 @@ set -eo pipefail shopt -s nullglob DO_CHOWN=1 -if [ "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]; then +if [[ "${CLICKHOUSE_RUN_AS_ROOT:=0}" = "1" || "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]]; then DO_CHOWN=0 fi -CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}" -CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}" +# CLICKHOUSE_UID and CLICKHOUSE_GID are kept for backward compatibility, but deprecated +# One must use either "docker run --user" or CLICKHOUSE_RUN_AS_ROOT=1 to run the process as +# FIXME: Remove ALL CLICKHOUSE_UID CLICKHOUSE_GID before 25.3 +if [[ "${CLICKHOUSE_UID:-}" || "${CLICKHOUSE_GID:-}" ]]; then + echo 'WARNING: Support for CLICKHOUSE_UID/CLICKHOUSE_GID will be removed in a couple of releases.' >&2 + echo 'WARNING: Either use a proper "docker run --user=xxx:xxxx" argument instead of CLICKHOUSE_UID/CLICKHOUSE_GID' >&2 + echo 'WARNING: or set "CLICKHOUSE_RUN_AS_ROOT=1" ENV to run the clickhouse-server as root:root' >&2 +fi -# support --user -if [ "$(id -u)" = "0" ]; then - USER=$CLICKHOUSE_UID - GROUP=$CLICKHOUSE_GID +# support `docker run --user=xxx:xxxx` +if [[ "$(id -u)" = "0" ]]; then + if [[ "$CLICKHOUSE_RUN_AS_ROOT" = 1 ]]; then + USER=0 + GROUP=0 + else + USER="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}" + GROUP="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}" + fi else USER="$(id -u)" GROUP="$(id -g)" @@ -55,14 +66,14 @@ function create_directory_and_do_chown() { [ -z "$dir" ] && return # ensure directories exist if [ "$DO_CHOWN" = "1" ]; then - mkdir="mkdir" + mkdir=( mkdir ) else # if DO_CHOWN=0 it means that the system does not map root user to "admin" permissions # it mainly happens on NFS mounts where root==nobody for security reasons # thus mkdir MUST run with user id/gid and not from nobody that has zero permissions - mkdir="/usr/bin/clickhouse su "${USER}:${GROUP}" mkdir" + mkdir=( clickhouse su "${USER}:${GROUP}" mkdir ) fi - if ! $mkdir -p "$dir"; then + if ! "${mkdir[@]}" -p "$dir"; then echo "Couldn't create necessary directory: $dir" exit 1 fi @@ -143,7 +154,7 @@ if [ -n "${RUN_INITDB_SCRIPTS}" ]; then fi # Listen only on localhost until the initialization is done - /usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" -- --listen_host=127.0.0.1 & + clickhouse su "${USER}:${GROUP}" clickhouse-server --config-file="$CLICKHOUSE_CONFIG" -- --listen_host=127.0.0.1 & pid="$!" # check if clickhouse is ready to accept connections @@ -151,7 +162,7 @@ if [ -n "${RUN_INITDB_SCRIPTS}" ]; then tries=${CLICKHOUSE_INIT_TIMEOUT:-1000} while ! wget --spider --no-check-certificate -T 1 -q "$URL" 2>/dev/null; do if [ "$tries" -le "0" ]; then - echo >&2 'ClickHouse init process failed.' + echo >&2 'ClickHouse init process timeout.' exit 1 fi tries=$(( tries-1 )) @@ -203,18 +214,8 @@ if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then CLICKHOUSE_WATCHDOG_ENABLE=${CLICKHOUSE_WATCHDOG_ENABLE:-0} export CLICKHOUSE_WATCHDOG_ENABLE - # An option for easy restarting and replacing clickhouse-server in a container, especially in Kubernetes. - # For example, you can replace the clickhouse-server binary to another and restart it while keeping the container running. - if [[ "${CLICKHOUSE_DOCKER_RESTART_ON_EXIT:-0}" -eq "1" ]]; then - while true; do - # This runs the server as a child process of the shell script: - /usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@" ||: - echo >&2 'ClickHouse Server exited, and the environment variable CLICKHOUSE_DOCKER_RESTART_ON_EXIT is set to 1. Restarting the server.' - done - else - # This replaces the shell script with the server: - exec /usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@" - fi + # This replaces the shell script with the server: + exec clickhouse su "${USER}:${GROUP}" clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@" fi # Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image diff --git a/docker/test/libfuzzer/Dockerfile b/docker/test/libfuzzer/Dockerfile index 3ffae0cd921..46e305c90ab 100644 --- a/docker/test/libfuzzer/Dockerfile +++ b/docker/test/libfuzzer/Dockerfile @@ -33,8 +33,6 @@ RUN apt-get update \ COPY requirements.txt / RUN pip3 install --no-cache-dir -r /requirements.txt -ENV FUZZER_ARGS="-max_total_time=60" - SHELL ["/bin/bash", "-c"] # docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/libfuzzer diff --git a/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile b/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile deleted file mode 100644 index a9802f6f1da..00000000000 --- a/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# Since right now we can't set volumes to the docker during build, we split building container in stages: -# 1. build base container -# 2. run base conatiner with mounted volumes -# 3. commit container as image -FROM ubuntu:20.04 as clickhouse-test-runner-base - -# A volume where directory with clickhouse packages to be mounted, -# for later installing. -VOLUME /packages - -CMD apt-get update ;\ - DEBIAN_FRONTEND=noninteractive \ - apt install -y /packages/clickhouse-common-static_*.deb \ - /packages/clickhouse-client_*.deb \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* diff --git a/docs/changelogs/v24.10.1.2812-stable.md b/docs/changelogs/v24.10.1.2812-stable.md new file mode 100644 index 00000000000..c26bbf706ff --- /dev/null +++ b/docs/changelogs/v24.10.1.2812-stable.md @@ -0,0 +1,412 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.10.1.2812-stable (9cd0a3738d5) FIXME as compared to v24.10.1.1-new (b12a3677418) + +#### Backward Incompatible Change +* Allow to write `SETTINGS` before `FORMAT` in a chain of queries with `UNION` when subqueries are inside parentheses. This closes [#39712](https://github.com/ClickHouse/ClickHouse/issues/39712). Change the behavior when a query has the SETTINGS clause specified twice in a sequence. The closest SETTINGS clause will have a preference for the corresponding subquery. In the previous versions, the outermost SETTINGS clause could take a preference over the inner one. [#68614](https://github.com/ClickHouse/ClickHouse/pull/68614) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Reordering of filter conditions from `[PRE]WHERE` clause is now allowed by default. It could be disabled by setting `allow_reorder_prewhere_conditions` to `false`. [#70657](https://github.com/ClickHouse/ClickHouse/pull/70657) ([Nikita Taranov](https://github.com/nickitat)). +* Fix `optimize_functions_to_subcolumns` optimization (previously could lead to `Invalid column type for ColumnUnique::insertRangeFrom. Expected String, got LowCardinality(String)` error), by preserving `LowCardinality` type in `mapKeys`/`mapValues`. [#70716](https://github.com/ClickHouse/ClickHouse/pull/70716) ([Azat Khuzhin](https://github.com/azat)). +* Remove the `idxd-config` library, which has an incompatible license. This also removes the experimental Intel DeflateQPL codec. [#70987](https://github.com/ClickHouse/ClickHouse/pull/70987) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### New Feature +* MongoDB integration refactored: migration to new driver mongocxx from deprecated Poco::MongoDB, remove support for deprecated old protocol, support for connection by URI, support for all MongoDB types, support for WHERE and ORDER BY statements on MongoDB side, restriction for expression unsupported by MongoDB. [#63279](https://github.com/ClickHouse/ClickHouse/pull/63279) ([Kirill Nikiforov](https://github.com/allmazz)). +* A new `--progress-table` option in clickhouse-client prints a table with metrics changing during query execution; a new `--enable-progress-table-toggle` is associated with the `--progress-table` option, and toggles the rendering of the progress table by pressing the control key (Space). [#63689](https://github.com/ClickHouse/ClickHouse/pull/63689) ([Maria Khristenko](https://github.com/mariaKhr)). +* This allows to grant access to the wildcard prefixes. `GRANT SELECT ON db.table_pefix_* TO user`. [#65311](https://github.com/ClickHouse/ClickHouse/pull/65311) ([pufit](https://github.com/pufit)). +* Add system.query_metric_log which contains history of memory and metric values from table system.events for individual queries, periodically flushed to disk. [#66532](https://github.com/ClickHouse/ClickHouse/pull/66532) ([Pablo Marcos](https://github.com/pamarcos)). +* A simple SELECT query can be written with implicit SELECT to enable calculator-style expressions, e.g., `ch "1 + 2"`. This is controlled by a new setting, `implicit_select`. [#68502](https://github.com/ClickHouse/ClickHouse/pull/68502) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support --copy mode for clickhouse local as a shortcut for format conversion [#68503](https://github.com/ClickHouse/ClickHouse/issues/68503). [#68583](https://github.com/ClickHouse/ClickHouse/pull/68583) ([Denis Hananein](https://github.com/denis-hananein)). +* Add support for `arrayUnion` function. [#68989](https://github.com/ClickHouse/ClickHouse/pull/68989) ([Peter Nguyen](https://github.com/petern48)). +* Support aggreate function `quantileExactWeightedInterpolated`, which is a interpolated version based on quantileExactWeighted. Some people may wonder why we need a new `quantileExactWeightedInterpolated` since we already have `quantileExactInterpolatedWeighted`. The reason is the new one is more accurate than the old one. BTW, it is for spark compatiability in Apache Gluten. [#69619](https://github.com/ClickHouse/ClickHouse/pull/69619) ([李扬](https://github.com/taiyang-li)). +* Support function arrayElementOrNull. It returns null if array index is out of range or map key not found. [#69646](https://github.com/ClickHouse/ClickHouse/pull/69646) ([李扬](https://github.com/taiyang-li)). +* Allows users to specify regular expressions through new `message_regexp` and `message_regexp_negative` fields in the `config.xml` file to filter out logging. The logging is applied to the formatted un-colored text for the most intuitive developer experience. [#69657](https://github.com/ClickHouse/ClickHouse/pull/69657) ([Peter Nguyen](https://github.com/petern48)). +* Support Dynamic type in most functions by executing them on internal types inside Dynamic. [#69691](https://github.com/ClickHouse/ClickHouse/pull/69691) ([Pavel Kruglov](https://github.com/Avogar)). +* Re-added `RIPEMD160` function, which computes the RIPEMD-160 cryptographic hash of a string. Example: `SELECT HEX(RIPEMD160('The quick brown fox jumps over the lazy dog'))` returns `37F332F68DB77BD9D7EDD4969571AD671CF9DD3B`. [#70087](https://github.com/ClickHouse/ClickHouse/pull/70087) ([Dergousov Maxim](https://github.com/m7kss1)). +* Allow to cache read files for object storage table engines and data lakes using hash from ETag + file path as cache key. [#70135](https://github.com/ClickHouse/ClickHouse/pull/70135) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Support reading Iceberg tables on HDFS. [#70268](https://github.com/ClickHouse/ClickHouse/pull/70268) ([flynn](https://github.com/ucasfl)). +* Allow to read/write JSON type as binary string in RowBinary format under settings `input_format_binary_read_json_as_string/output_format_binary_write_json_as_string`. [#70288](https://github.com/ClickHouse/ClickHouse/pull/70288) ([Pavel Kruglov](https://github.com/Avogar)). +* Allow to serialize/deserialize JSON column as single String column in Native format. For output use setting `output_format_native_write_json_as_string`. For input, use serialization version `1` before the column data. [#70312](https://github.com/ClickHouse/ClickHouse/pull/70312) ([Pavel Kruglov](https://github.com/Avogar)). +* Supports standard CTE, `with insert`, as previously only supports `insert ... with ...`. [#70593](https://github.com/ClickHouse/ClickHouse/pull/70593) ([Shichao Jin](https://github.com/jsc0218)). + +#### Performance Improvement +* Support minmax index for `pointInPolygon`. [#62085](https://github.com/ClickHouse/ClickHouse/pull/62085) ([JackyWoo](https://github.com/JackyWoo)). +* Add support for parquet bloom filters. [#62966](https://github.com/ClickHouse/ClickHouse/pull/62966) ([Arthur Passos](https://github.com/arthurpassos)). +* Lock-free parts rename to avoid INSERT affect SELECT (due to parts lock) (under normal circumstances with `fsync_part_directory`, QPS of SELECT with INSERT in parallel, increased 2x, under heavy load the effect is even bigger). Note, this only includes `ReplicatedMergeTree` for now. [#64955](https://github.com/ClickHouse/ClickHouse/pull/64955) ([Azat Khuzhin](https://github.com/azat)). +* Respect `ttl_only_drop_parts` on `materialize ttl`; only read necessary columns to recalculate TTL and drop parts by replacing them with an empty one. [#65488](https://github.com/ClickHouse/ClickHouse/pull/65488) ([Andrey Zvonov](https://github.com/zvonand)). +* Refactor `IDisk` and `IObjectStorage` for better performance. Tables from `plain` and `plain_rewritable` object storages will initialize faster. [#68146](https://github.com/ClickHouse/ClickHouse/pull/68146) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Optimized thread creation in the ThreadPool to minimize lock contention. Thread creation is now performed outside of the critical section to avoid delays in job scheduling and thread management under high load conditions. This leads to a much more responsive ClickHouse under heavy concurrent load. [#68694](https://github.com/ClickHouse/ClickHouse/pull/68694) ([filimonov](https://github.com/filimonov)). +* Enable reading LowCardinality string columns from ORC. [#69481](https://github.com/ClickHouse/ClickHouse/pull/69481) ([李扬](https://github.com/taiyang-li)). +* Added an ability to parse data directly into sparse columns. [#69828](https://github.com/ClickHouse/ClickHouse/pull/69828) ([Anton Popov](https://github.com/CurtizJ)). +* Supports parallel reading of parquet row groups and prefetching of row groups in single-threaded mode. [#69862](https://github.com/ClickHouse/ClickHouse/pull/69862) ([LiuNeng](https://github.com/liuneng1994)). +* Improved performance of parsing formats with high number of missed values (e.g. `JSONEachRow`). [#69875](https://github.com/ClickHouse/ClickHouse/pull/69875) ([Anton Popov](https://github.com/CurtizJ)). +* Use `LowCardinality` for `ProfileEvents` in system logs such as `part_log`, `query_views_log`, `filesystem_cache_log`. [#70152](https://github.com/ClickHouse/ClickHouse/pull/70152) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve performance of FromUnixTimestamp/ToUnixTimestamp functions. [#71042](https://github.com/ClickHouse/ClickHouse/pull/71042) ([kevinyhzou](https://github.com/KevinyhZou)). + +#### Improvement +* Allow parametrised SQL aliases. [#50665](https://github.com/ClickHouse/ClickHouse/pull/50665) ([Anton Kozlov](https://github.com/tonickkozlov)). +* Fixed [#57616](https://github.com/ClickHouse/ClickHouse/issues/57616) this problem occurs because all positive number arguments are automatically identified as `uint64` type, leading to an inability to match int type data in `summapfiltered`. the issue of non-matching is indeed confusing, as the `uint64` parameters are not specified by the user. additionally, if the arguments are `[1,2,3,toint8(-3)]`, due to the `getleastsupertype()`, these parameters will be uniformly treated as `int` type, causing `'1,2,3'` to also fail in matching the `uint` type data in `summapfiltered`. [#58408](https://github.com/ClickHouse/ClickHouse/pull/58408) ([Chen768959](https://github.com/Chen768959)). +* `ALTER TABLE .. REPLACE PARTITION` doesn't wait anymore for mutations/merges that happen in other partitions. [#59138](https://github.com/ClickHouse/ClickHouse/pull/59138) ([Vasily Nemkov](https://github.com/Enmk)). +* Refreshable materialized views are now supported in Replicated databases. [#60669](https://github.com/ClickHouse/ClickHouse/pull/60669) ([Michael Kolupaev](https://github.com/al13n321)). +* Symbolic links for tables in the `data/database_name/` directory are created for the actual paths to the table's data, depending on the storage policy, instead of the `store/...` directory on the default disk. [#61777](https://github.com/ClickHouse/ClickHouse/pull/61777) ([Kirill](https://github.com/kirillgarbar)). +* Apply configuration updates in global context object. It fixes issues like [#62308](https://github.com/ClickHouse/ClickHouse/issues/62308). [#62944](https://github.com/ClickHouse/ClickHouse/pull/62944) ([Amos Bird](https://github.com/amosbird)). +* Reworked settings that control the behavior of parallel replicas algorithms. A quick recap: ClickHouse has four different algorithms for parallel reading involving multiple replicas, which is reflected in the setting `parallel_replicas_mode`, the default value for it is `read_tasks` Additionally, the toggle-switch setting `enable_parallel_replicas` has been added. [#63151](https://github.com/ClickHouse/ClickHouse/pull/63151) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `ReadSettings` not using user set values, because defaults were only used. [#65625](https://github.com/ClickHouse/ClickHouse/pull/65625) ([Kseniia Sumarokova](https://github.com/kssenii)). +* While parsing an Enum field from JSON, a string containing an integer will be interpreted as the corresponding Enum element. This closes [#65119](https://github.com/ClickHouse/ClickHouse/issues/65119). [#66801](https://github.com/ClickHouse/ClickHouse/pull/66801) ([scanhex12](https://github.com/scanhex12)). +* Allow `TRIM` -ing `LEADING` or `TRAILING` empty string as a no-op. Closes [#67792](https://github.com/ClickHouse/ClickHouse/issues/67792). [#68455](https://github.com/ClickHouse/ClickHouse/pull/68455) ([Peter Nguyen](https://github.com/petern48)). +* Support creating a table with a query: `CREATE TABLE ... CLONE AS ...`. It clones the source table's schema and then attaches all partitions to the newly created table. This feature is only supported with tables of the `MergeTree` family Closes [#65015](https://github.com/ClickHouse/ClickHouse/issues/65015). [#69091](https://github.com/ClickHouse/ClickHouse/pull/69091) ([tuanpach](https://github.com/tuanpach)). +* In Gluten ClickHouse, Spark's timestamp type is mapped to ClickHouse's datetime64(6) type. When casting timestamp '2012-01-01 00:11:22' as a string, Spark returns '2012-01-01 00:11:22', while Gluten ClickHouse returns '2012-01-01 00:11:22.000000'. [#69179](https://github.com/ClickHouse/ClickHouse/pull/69179) ([Wenzheng Liu](https://github.com/lwz9103)). +* Always use the new analyzer to calculate constant expressions when `enable_analyzer` is set to `true`. Support calculation of `executable()` table function arguments without using `SELECT` query for constant expression. [#69292](https://github.com/ClickHouse/ClickHouse/pull/69292) ([Dmitry Novik](https://github.com/novikd)). +* Add `enable_secure_identifiers` to disallow insecure identifiers. [#69411](https://github.com/ClickHouse/ClickHouse/pull/69411) ([tuanpach](https://github.com/tuanpach)). +* Add `show_create_query_identifier_quoting_rule` to define identifier quoting behavior of the show create query result. Possible values: - `user_display`: When the identifiers is a keyword. - `when_necessary`: When the identifiers is one of `{"distinct", "all", "table"}`, or it can cause ambiguity: column names, dictionary attribute names. - `always`: Always quote identifiers. [#69448](https://github.com/ClickHouse/ClickHouse/pull/69448) ([tuanpach](https://github.com/tuanpach)). +* Follow-up to https://github.com/ClickHouse/ClickHouse/pull/69346 Point 4 described there will work now as well:. [#69563](https://github.com/ClickHouse/ClickHouse/pull/69563) ([Vitaly Baranov](https://github.com/vitlibar)). +* Implement generic SerDe between Avro Union and ClickHouse Variant type. Resolves [#69713](https://github.com/ClickHouse/ClickHouse/issues/69713). [#69712](https://github.com/ClickHouse/ClickHouse/pull/69712) ([Jiří Kozlovský](https://github.com/jirislav)). +* 1. CREATE TABLE AS will copy PRIMARY KEY, ORDER BY, and similar clauses. Now it is supported only for the MergeTree family of table engines. 2. For example, the follow SQL statements will trigger exception in the past, but this PR fixes it: if the destination table do not provide an `ORDER BY` or `PRIMARY KEY` expression in the table definition, we will copy that from source table. [#69739](https://github.com/ClickHouse/ClickHouse/pull/69739) ([sakulali](https://github.com/sakulali)). +* Added user-level settings `min_free_disk_bytes_to_throw_insert` and `min_free_disk_ratio_to_throw_insert` to prevent insertions on disks that are almost full. [#69755](https://github.com/ClickHouse/ClickHouse/pull/69755) ([Marco Vilas Boas](https://github.com/marco-vb)). +* If you run `clickhouse-client` or other CLI application and it starts up slowly due to an overloaded server, and you start typing your query, such as `SELECT`, the previous versions will display the remaining of the terminal echo contents before printing the greetings message, such as `SELECTClickHouse local version 24.10.1.1.` instead of `ClickHouse local version 24.10.1.1.`. Now it is fixed. This closes [#31696](https://github.com/ClickHouse/ClickHouse/issues/31696). [#69856](https://github.com/ClickHouse/ClickHouse/pull/69856) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add new column readonly_duration to the system.replicas table. Needed to be able to distinguish actual readonly replicas from sentinel ones in alerts. [#69871](https://github.com/ClickHouse/ClickHouse/pull/69871) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)). +* Change the join to sort settings type to unsigned int. [#69886](https://github.com/ClickHouse/ClickHouse/pull/69886) ([kevinyhzou](https://github.com/KevinyhZou)). +* Support 64-bit XID in Keeper. It can be enabled with `use_xid_64` config. [#69908](https://github.com/ClickHouse/ClickHouse/pull/69908) ([Antonio Andelic](https://github.com/antonio2368)). +* New function getSettingOrDefault() added to return the default value and avoid exception if a custom setting is not found in the current profile. [#69917](https://github.com/ClickHouse/ClickHouse/pull/69917) ([Shankar](https://github.com/shiyer7474)). +* Allow empty needle in function replace, the same behavior with PostgreSQL. [#69918](https://github.com/ClickHouse/ClickHouse/pull/69918) ([zhanglistar](https://github.com/zhanglistar)). +* Enhance OpenTelemetry span logging to include query settings. [#70011](https://github.com/ClickHouse/ClickHouse/pull/70011) ([sharathks118](https://github.com/sharathks118)). +* Allow empty needle in functions replaceRegexp*, like https://github.com/ClickHouse/ClickHouse/pull/69918. [#70053](https://github.com/ClickHouse/ClickHouse/pull/70053) ([zhanglistar](https://github.com/zhanglistar)). +* Add info to higher-order array functions if lambda result type is unexpected. [#70093](https://github.com/ClickHouse/ClickHouse/pull/70093) ([ttanay](https://github.com/ttanay)). +* Keeper improvement: less blocking during cluster changes. [#70275](https://github.com/ClickHouse/ClickHouse/pull/70275) ([Antonio Andelic](https://github.com/antonio2368)). +* Embedded documentation for settings will be strictly more detailed and complete than the documentation on the website. This is the first step before making the website documentation always auto-generated from the source code. This has long-standing implications: - it will be guaranteed to have every setting; - there is no chance of having default values obsolete; - we can generate this documentation for each ClickHouse version; - the documentation can be displayed by the server itself even without Internet access. Generate the docs on the website from the source code. [#70289](https://github.com/ClickHouse/ClickHouse/pull/70289) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `WITH IMPLICIT` and `FINAL` keywords to the `SHOW GRANTS` command. Fix a minor bug with implicit grants: [#70094](https://github.com/ClickHouse/ClickHouse/issues/70094). [#70293](https://github.com/ClickHouse/ClickHouse/pull/70293) ([pufit](https://github.com/pufit)). +* Don't disable nonblocking read from page cache for the entire server when reading from a blocking I/O. [#70299](https://github.com/ClickHouse/ClickHouse/pull/70299) ([Antonio Andelic](https://github.com/antonio2368)). +* Respect `compatibility` for MergeTree settings. The `compatibility` value is taken from the `default` profile on server startup, and default MergeTree settings are changed accordingly. Further changes of the `compatibility` setting do not affect MergeTree settings. [#70322](https://github.com/ClickHouse/ClickHouse/pull/70322) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Clickhouse-client realtime metrics follow-up: restore cursor when ctrl-c cancels query; immediately stop intercepting keystrokes when the query is canceled; display the metrics table if `--progress-table` is on, and toggling is disabled. [#70423](https://github.com/ClickHouse/ClickHouse/pull/70423) ([Julia Kartseva](https://github.com/jkartseva)). +* Command-line arguments for Bool settings are set to true when no value is provided for the argument (e.g. `clickhouse-client --optimize_aggregation_in_order --query "SELECT 1"`). [#70459](https://github.com/ClickHouse/ClickHouse/pull/70459) ([davidtsuk](https://github.com/davidtsuk)). +* Avoid spamming the logs with large HTTP response bodies in case of errors during inter-server communication. [#70487](https://github.com/ClickHouse/ClickHouse/pull/70487) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Added a new setting `max_parts_to_move` to control the maximum number of parts that can be moved at once. [#70520](https://github.com/ClickHouse/ClickHouse/pull/70520) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Limit the frequency of certain log messages. [#70601](https://github.com/ClickHouse/ClickHouse/pull/70601) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Don't do validation when synchronizing user_directories from keeper. [#70644](https://github.com/ClickHouse/ClickHouse/pull/70644) ([Raúl Marín](https://github.com/Algunenano)). +* Introduced a special (experimental) mode of a merge selector for MergeTree tables which makes it more aggressive for the partitions that are close to the limit by the number of parts. It is controlled by the `merge_selector_use_blurry_base` MergeTree-level setting. [#70645](https://github.com/ClickHouse/ClickHouse/pull/70645) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* `CHECK TABLE` with `PART` qualifier was incorrectly formatted in the client. [#70660](https://github.com/ClickHouse/ClickHouse/pull/70660) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support write column index and offset index using parquet native writer. [#70669](https://github.com/ClickHouse/ClickHouse/pull/70669) ([LiuNeng](https://github.com/liuneng1994)). +* Support parse `DateTime64` for microseond and timezone in joda syntax. [#70737](https://github.com/ClickHouse/ClickHouse/pull/70737) ([kevinyhzou](https://github.com/KevinyhZou)). +* Changed an approach to figure out if a cloud storage supports [batch delete](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html) or not. [#70786](https://github.com/ClickHouse/ClickHouse/pull/70786) ([Vitaly Baranov](https://github.com/vitlibar)). +* Support for Parquet page V2 on native reader. [#70807](https://github.com/ClickHouse/ClickHouse/pull/70807) ([Arthur Passos](https://github.com/arthurpassos)). +* Add an HTML page for visualizing merges. [#70821](https://github.com/ClickHouse/ClickHouse/pull/70821) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#71234](https://github.com/ClickHouse/ClickHouse/issues/71234): Do not call the object storage API when listing directories, as this may be cost-inefficient. Instead, store the list of filenames in the memory. The trade-offs are increased initial load time and memory required to store filenames. [#70823](https://github.com/ClickHouse/ClickHouse/pull/70823) ([Julia Kartseva](https://github.com/jkartseva)). +* A check if table has both `storage_policy` and `disk` set after alter query is added. A check if a new storage policy is compatible with an old one when using `disk` setting is added. [#70839](https://github.com/ClickHouse/ClickHouse/pull/70839) ([Kirill](https://github.com/kirillgarbar)). +* Add system.s3_queue_settings and system.azure_queue_settings. [#70841](https://github.com/ClickHouse/ClickHouse/pull/70841) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Functions `base58Encode` and `base58Decode` now accept arguments of type `FixedString`. Example: `SELECT base58Encode(toFixedString('plaintext', 9));`. [#70846](https://github.com/ClickHouse/ClickHouse/pull/70846) ([Faizan Patel](https://github.com/faizan2786)). +* Add the `partition` column to every entry type of the part log. Previously, it was set only for some entries. This closes [#70819](https://github.com/ClickHouse/ClickHouse/issues/70819). [#70848](https://github.com/ClickHouse/ClickHouse/pull/70848) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add merge start and mutate start events into `system.part_log` which helps with merges analysis and visualization. [#70850](https://github.com/ClickHouse/ClickHouse/pull/70850) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not call the LIST object storage API when determining if a file or directory exists on the plain rewritable disk, as it can be cost-inefficient. [#70852](https://github.com/ClickHouse/ClickHouse/pull/70852) ([Julia Kartseva](https://github.com/jkartseva)). +* Add a profile event about the number of merged source parts. It allows the monitoring of the fanout of the merge tree in production. [#70908](https://github.com/ClickHouse/ClickHouse/pull/70908) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Reduce the number of object storage HEAD API requests in the plain_rewritable disk. [#70915](https://github.com/ClickHouse/ClickHouse/pull/70915) ([Julia Kartseva](https://github.com/jkartseva)). +* Background downloads to filesystem cache was enabled back. [#70929](https://github.com/ClickHouse/ClickHouse/pull/70929) ([Nikita Taranov](https://github.com/nickitat)). +* Add a new merge selector algorithm, named `Trivial`, for professional usage only. It is worse than the `Simple` merge selector. [#70969](https://github.com/ClickHouse/ClickHouse/pull/70969) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix (user-visible misbehavior in an official stable release) +* Fix toHour-like conversion functions' monotonicity when optional time zone argument is passed. [#60264](https://github.com/ClickHouse/ClickHouse/pull/60264) ([Amos Bird](https://github.com/amosbird)). +* Relax `supportsPrewhere` check for StorageMerge. This fixes [#61064](https://github.com/ClickHouse/ClickHouse/issues/61064). It was hardened unnecessarily in [#60082](https://github.com/ClickHouse/ClickHouse/issues/60082). [#61091](https://github.com/ClickHouse/ClickHouse/pull/61091) ([Amos Bird](https://github.com/amosbird)). +* Fix `use_concurrency_control` setting handling for proper `concurrent_threads_soft_limit_num` limit enforcing. This enables concurrency control by default because previously it was broken. [#61473](https://github.com/ClickHouse/ClickHouse/pull/61473) ([Sergei Trifonov](https://github.com/serxa)). +* Fix incorrect JOIN ON section optimization in case of `IS NULL` check under any other function (like `NOT`) that may lead to wrong results. Closes [#67915](https://github.com/ClickHouse/ClickHouse/issues/67915). [#68049](https://github.com/ClickHouse/ClickHouse/pull/68049) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Prevent `ALTER` queries that would make the `CREATE` query of tables invalid. [#68574](https://github.com/ClickHouse/ClickHouse/pull/68574) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Fix inconsistent AST formatting for `negate` (`-`) and `NOT` functions with tuples and arrays. [#68600](https://github.com/ClickHouse/ClickHouse/pull/68600) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Fix insertion of incomplete type into Dynamic during deserialization. It could lead to `Parameter out of bound` errors. [#69291](https://github.com/ClickHouse/ClickHouse/pull/69291) ([Pavel Kruglov](https://github.com/Avogar)). +* Fix inf loop after `restore replica` in the replicated merge tree with zero copy. [#69293](https://github.com/ClickHouse/ClickHouse/pull/69293) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Return back default value of `processing_threads_num` as number of cpu cores in storage `S3Queue`. [#69384](https://github.com/ClickHouse/ClickHouse/pull/69384) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Bypass try/catch flow when de/serializing nested repeated protobuf to nested columns ( fixes [#41971](https://github.com/ClickHouse/ClickHouse/issues/41971) ). [#69556](https://github.com/ClickHouse/ClickHouse/pull/69556) ([Eliot Hautefeuille](https://github.com/hileef)). +* Fix vrash during insertion into FixedString column in PostgreSQL engine. [#69584](https://github.com/ClickHouse/ClickHouse/pull/69584) ([Pavel Kruglov](https://github.com/Avogar)). +* Fix crash when executing `create view t as (with recursive 42 as ttt select ttt);`. [#69676](https://github.com/ClickHouse/ClickHouse/pull/69676) ([Han Fei](https://github.com/hanfei1991)). +* Added `strict_once` mode to aggregate function `windowFunnel` to avoid counting one event several times in case it matches multiple conditions, close [#21835](https://github.com/ClickHouse/ClickHouse/issues/21835). [#69738](https://github.com/ClickHouse/ClickHouse/pull/69738) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Fixed `maxMapState` throwing 'Bad get' if value type is DateTime64. [#69787](https://github.com/ClickHouse/ClickHouse/pull/69787) ([Michael Kolupaev](https://github.com/al13n321)). +* Fix `getSubcolumn` with `LowCardinality` columns by overriding `useDefaultImplementationForLowCardinalityColumns` to return `true`. [#69831](https://github.com/ClickHouse/ClickHouse/pull/69831) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)). +* Fix permanent blocked distributed sends if DROP of distributed table fails. [#69843](https://github.com/ClickHouse/ClickHouse/pull/69843) ([Azat Khuzhin](https://github.com/azat)). +* Fix non-cancellable queries containing WITH FILL with NaN keys. This closes [#69261](https://github.com/ClickHouse/ClickHouse/issues/69261). [#69845](https://github.com/ClickHouse/ClickHouse/pull/69845) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix analyzer default with old compatibility value. [#69895](https://github.com/ClickHouse/ClickHouse/pull/69895) ([Raúl Marín](https://github.com/Algunenano)). +* Don't check dependencies during CREATE OR REPLACE VIEW during DROP of old table. Previously CREATE OR REPLACE query failed when there are dependent tables of the recreated view. [#69907](https://github.com/ClickHouse/ClickHouse/pull/69907) ([Pavel Kruglov](https://github.com/Avogar)). +* Implement missing decimal cases for `zeroField`. Fixes [#69730](https://github.com/ClickHouse/ClickHouse/issues/69730). [#69978](https://github.com/ClickHouse/ClickHouse/pull/69978) ([Arthur Passos](https://github.com/arthurpassos)). +* Now SQL security will work with parameterized views correctly. [#69984](https://github.com/ClickHouse/ClickHouse/pull/69984) ([pufit](https://github.com/pufit)). +* Closes [#69752](https://github.com/ClickHouse/ClickHouse/issues/69752). [#69985](https://github.com/ClickHouse/ClickHouse/pull/69985) ([pufit](https://github.com/pufit)). +* Fixed a bug when the timezone could change the result of the query with a `Date` or `Date32` arguments. [#70036](https://github.com/ClickHouse/ClickHouse/pull/70036) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Fixes `Block structure mismatch` for queries with nested views and `WHERE` condition. Fixes [#66209](https://github.com/ClickHouse/ClickHouse/issues/66209). [#70054](https://github.com/ClickHouse/ClickHouse/pull/70054) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Avoid reusing columns among different named tuples when evaluating `tuple` functions. This fixes [#70022](https://github.com/ClickHouse/ClickHouse/issues/70022). [#70103](https://github.com/ClickHouse/ClickHouse/pull/70103) ([Amos Bird](https://github.com/amosbird)). +* Fix wrong LOGICAL_ERROR when replacing literals in ranges. [#70122](https://github.com/ClickHouse/ClickHouse/pull/70122) ([Pablo Marcos](https://github.com/pamarcos)). +* Check for Nullable(Nothing) type during ALTER TABLE MODIFY COLUMN/QUERY to prevent tables with such data type. [#70123](https://github.com/ClickHouse/ClickHouse/pull/70123) ([Pavel Kruglov](https://github.com/Avogar)). +* Proper error message for illegal query `JOIN ... ON *` , close [#68650](https://github.com/ClickHouse/ClickHouse/issues/68650). [#70124](https://github.com/ClickHouse/ClickHouse/pull/70124) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Fix wrong result with skipping index. [#70127](https://github.com/ClickHouse/ClickHouse/pull/70127) ([Raúl Marín](https://github.com/Algunenano)). +* Fix data race in ColumnObject/ColumnTuple decompress method that could lead to heap use after free. [#70137](https://github.com/ClickHouse/ClickHouse/pull/70137) ([Pavel Kruglov](https://github.com/Avogar)). +* Fix possible hung in ALTER COLUMN with Dynamic type. [#70144](https://github.com/ClickHouse/ClickHouse/pull/70144) ([Pavel Kruglov](https://github.com/Avogar)). +* Now ClickHouse will consider more errors as retriable and will not mark data parts as broken in case of such errors. [#70145](https://github.com/ClickHouse/ClickHouse/pull/70145) ([alesapin](https://github.com/alesapin)). +* Use correct `max_types` parameter during Dynamic type creation for JSON subcolumn. [#70147](https://github.com/ClickHouse/ClickHouse/pull/70147) ([Pavel Kruglov](https://github.com/Avogar)). +* Fix the password being displayed in `system.query_log` for users with bcrypt password authentication method. [#70148](https://github.com/ClickHouse/ClickHouse/pull/70148) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix event counter for native interface (InterfaceNativeSendBytes). [#70153](https://github.com/ClickHouse/ClickHouse/pull/70153) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix possible crash in JSON column. [#70172](https://github.com/ClickHouse/ClickHouse/pull/70172) ([Pavel Kruglov](https://github.com/Avogar)). +* Fix multiple issues with arrayMin and arrayMax. [#70207](https://github.com/ClickHouse/ClickHouse/pull/70207) ([Raúl Marín](https://github.com/Algunenano)). +* Respect setting allow_simdjson in JSON type parser. [#70218](https://github.com/ClickHouse/ClickHouse/pull/70218) ([Pavel Kruglov](https://github.com/Avogar)). +* Fix server segfault on creating a materialized view with two selects and an `INTERSECT`, e.g. `CREATE MATERIALIZED VIEW v0 AS (SELECT 1) INTERSECT (SELECT 1);`. [#70264](https://github.com/ClickHouse/ClickHouse/pull/70264) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Don't modify global settings with startup scripts. Previously, changing a setting in a startup script would change it globally. [#70310](https://github.com/ClickHouse/ClickHouse/pull/70310) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix ALTER of Dynamic type with reducing max_types parameter that could lead to server crash. [#70328](https://github.com/ClickHouse/ClickHouse/pull/70328) ([Pavel Kruglov](https://github.com/Avogar)). +* Fix crash when using WITH FILL incorrectly. [#70338](https://github.com/ClickHouse/ClickHouse/pull/70338) ([Raúl Marín](https://github.com/Algunenano)). +* Fix possible use-after-free in `SYSTEM DROP FORMAT SCHEMA CACHE FOR Protobuf`. [#70358](https://github.com/ClickHouse/ClickHouse/pull/70358) ([Azat Khuzhin](https://github.com/azat)). +* Fix crash during GROUP BY JSON sub-object subcolumn. [#70374](https://github.com/ClickHouse/ClickHouse/pull/70374) ([Pavel Kruglov](https://github.com/Avogar)). +* Don't prefetch parts for vertical merges if part has no rows. [#70452](https://github.com/ClickHouse/ClickHouse/pull/70452) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix crash in WHERE with lambda functions. [#70464](https://github.com/ClickHouse/ClickHouse/pull/70464) ([Raúl Marín](https://github.com/Algunenano)). +* Fix table creation with `CREATE ... AS table_function()` with database `Replicated` and unavailable table function source on secondary replica. [#70511](https://github.com/ClickHouse/ClickHouse/pull/70511) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Ignore all output on async insert with `wait_for_async_insert=1`. Closes [#62644](https://github.com/ClickHouse/ClickHouse/issues/62644). [#70530](https://github.com/ClickHouse/ClickHouse/pull/70530) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Ignore frozen_metadata.txt while traversing shadow directory from system.remote_data_paths. [#70590](https://github.com/ClickHouse/ClickHouse/pull/70590) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Fix creation of stateful window functions on misaligned memory. [#70631](https://github.com/ClickHouse/ClickHouse/pull/70631) ([Raúl Marín](https://github.com/Algunenano)). +* Fixed rare crashes in `SELECT`-s and merges after adding a column of `Array` type with non-empty default expression. [#70695](https://github.com/ClickHouse/ClickHouse/pull/70695) ([Anton Popov](https://github.com/CurtizJ)). +* Insert into table function s3 respect query settings. [#70696](https://github.com/ClickHouse/ClickHouse/pull/70696) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Fix infinite recursion when infering a proto schema with skip unsupported fields enabled. [#70697](https://github.com/ClickHouse/ClickHouse/pull/70697) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#71122](https://github.com/ClickHouse/ClickHouse/issues/71122): `GroupArraySortedData` uses a PODArray with non-POD elements, manually calling constructors and destructors for the elements as needed. But it wasn't careful enough: in two places it forgot to call destructor, in one place it left elements uninitialized if an exception is thrown when deserializing previous elements. Then `GroupArraySortedData`'s destructor called destructors on uninitialized elements and crashed: ``` 2024.10.17 22:58:23.523790 [ 5233 ] {} BaseDaemon: ########## Short fault info ############ 2024.10.17 22:58:23.523834 [ 5233 ] {} BaseDaemon: (version 24.6.1.4609 (official build), build id: 5423339A6571004018D55BBE05D464AFA35E6718, git hash: fa6cdfda8a94890eb19bc7f22f8b0b56292f7a26) (from thread 682) Received signal 11 2024.10.17 22:58:23.523862 [ 5233 ] {} BaseDaemon: Signal description: Segmentation fault 2024.10.17 22:58:23.523883 [ 5233 ] {} BaseDaemon: Address: 0x8f. Access: . Address not mapped to object. 2024.10.17 22:58:23.523908 [ 5233 ] {} BaseDaemon: Stack trace: 0x0000aaaac4b78308 0x0000ffffb7701850 0x0000aaaac0104855 0x0000aaaac01048a0 0x0000aaaac501e84c 0x0000aaaac7c510d0 0x0000aaaac7c4ba20 0x0000aaaac968bbfc 0x0000aaaac968fab0 0x0000aaaac969bf50 0x0000aaaac9b7520c 0x0000aaaac9b74c74 0x0000aaaac9b8a150 0x0000aaaac9b809f0 0x0000aaaac9b80574 0x0000aaaac9b8e364 0x0000aaaac9b8e4fc 0x0000aaaac94f4328 0x0000aaaac94f428c 0x0000aaaac94f7df0 0x0000aaaac98b5a3c 0x0000aaaac950b234 0x0000aaaac49ae264 0x0000aaaac49b1dd0 0x0000aaaac49b0a80 0x0000ffffb755d5c8 0x0000ffffb75c5edc 2024.10.17 22:58:23.523936 [ 5233 ] {} BaseDaemon: ######################################## 2024.10.17 22:58:23.523959 [ 5233 ] {} BaseDaemon: (version 24.6.1.4609 (official build), build id: 5423339A6571004018D55BBE05D464AFA35E6718, git hash: fa6cdfda8a94890eb19bc7f22f8b0b56292f7a26) (from thread 682) (query_id: 6c8a33a2-f45a-4a3b-bd71-ded6a1c9ccd3::202410_534066_534078_2) (query: ) Received signal Segmentation fault (11) 2024.10.17 22:58:23.523977 [ 5233 ] {} BaseDaemon: Address: 0x8f. Access: . Address not mapped to object. 2024.10.17 22:58:23.523993 [ 5233 ] {} BaseDaemon: Stack trace: 0x0000aaaac4b78308 0x0000ffffb7701850 0x0000aaaac0104855 0x0000aaaac01048a0 0x0000aaaac501e84c 0x0000aaaac7c510d0 0x0000aaaac7c4ba20 0x0000aaaac968bbfc 0x0000aaaac968fab0 0x0000aaaac969bf50 0x0000aaaac9b7520c 0x0000aaaac9b74c74 0x0000aaaac9b8a150 0x0000aaaac9b809f0 0x0000aaaac9b80574 0x0000aaaac9b8e364 0x0000aaaac9b8e4fc 0x0000aaaac94f4328 0x0000aaaac94f428c 0x0000aaaac94f7df0 0x0000aaaac98b5a3c 0x0000aaaac950b234 0x0000aaaac49ae264 0x0000aaaac49b1dd0 0x0000aaaac49b0a80 0x0000ffffb755d5c8 0x0000ffffb75c5edc 2024.10.17 22:58:23.524817 [ 5233 ] {} BaseDaemon: 0. signalHandler(int, siginfo_t*, void*) @ 0x000000000c6f8308 2024.10.17 22:58:23.524917 [ 5233 ] {} BaseDaemon: 1. ? @ 0x0000ffffb7701850 2024.10.17 22:58:23.524962 [ 5233 ] {} BaseDaemon: 2. DB::Field::~Field() @ 0x0000000007c84855 2024.10.17 22:58:23.525012 [ 5233 ] {} BaseDaemon: 3. DB::Field::~Field() @ 0x0000000007c848a0 2024.10.17 22:58:23.526626 [ 5233 ] {} BaseDaemon: 4. DB::IAggregateFunctionDataHelper, DB::(anonymous namespace)::GroupArraySorted, DB::Field>>::destroy(char*) const (.5a6a451027f732f9fd91c13f4a13200c) @ 0x000000000cb9e84c 2024.10.17 22:58:23.527322 [ 5233 ] {} BaseDaemon: 5. DB::SerializationAggregateFunction::deserializeBinaryBulk(DB::IColumn&, DB::ReadBuffer&, unsigned long, double) const @ 0x000000000f7d10d0 2024.10.17 22:58:23.528470 [ 5233 ] {} BaseDaemon: 6. DB::ISerialization::deserializeBinaryBulkWithMultipleStreams(COW::immutable_ptr&, unsigned long, DB::ISerialization::DeserializeBinaryBulkSettings&, std::shared_ptr&, std::unordered_map::immutable_ptr, std::hash, std::equal_to, std::allocator::immutable_ptr>>>*) const @ 0x000000000f7cba20 2024.10.17 22:58:23.529213 [ 5233 ] {} BaseDaemon: 7. DB::MergeTreeReaderCompact::readData(DB::NameAndTypePair const&, COW::immutable_ptr&, unsigned long, std::function const&) @ 0x000000001120bbfc 2024.10.17 22:58:23.529277 [ 5233 ] {} BaseDaemon: 8. DB::MergeTreeReaderCompactSingleBuffer::readRows(unsigned long, unsigned long, bool, unsigned long, std::vector::immutable_ptr, std::allocator::immutable_ptr>>&) @ 0x000000001120fab0 2024.10.17 22:58:23.529319 [ 5233 ] {} BaseDaemon: 9. DB::MergeTreeSequentialSource::generate() @ 0x000000001121bf50 2024.10.17 22:58:23.529346 [ 5233 ] {} BaseDaemon: 10. DB::ISource::tryGenerate() @ 0x00000000116f520c 2024.10.17 22:58:23.529653 [ 5233 ] {} BaseDaemon: 11. DB::ISource::work() @ 0x00000000116f4c74 2024.10.17 22:58:23.529679 [ 5233 ] {} BaseDaemon: 12. DB::ExecutionThreadContext::executeTask() @ 0x000000001170a150 2024.10.17 22:58:23.529733 [ 5233 ] {} BaseDaemon: 13. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic*) @ 0x00000000117009f0 2024.10.17 22:58:23.529763 [ 5233 ] {} BaseDaemon: 14. DB::PipelineExecutor::executeStep(std::atomic*) @ 0x0000000011700574 2024.10.17 22:58:23.530089 [ 5233 ] {} BaseDaemon: 15. DB::PullingPipelineExecutor::pull(DB::Chunk&) @ 0x000000001170e364 2024.10.17 22:58:23.530277 [ 5233 ] {} BaseDaemon: 16. DB::PullingPipelineExecutor::pull(DB::Block&) @ 0x000000001170e4fc 2024.10.17 22:58:23.530295 [ 5233 ] {} BaseDaemon: 17. DB::MergeTask::ExecuteAndFinalizeHorizontalPart::executeImpl() @ 0x0000000011074328 2024.10.17 22:58:23.530318 [ 5233 ] {} BaseDaemon: 18. DB::MergeTask::ExecuteAndFinalizeHorizontalPart::execute() @ 0x000000001107428c 2024.10.17 22:58:23.530339 [ 5233 ] {} BaseDaemon: 19. DB::MergeTask::execute() @ 0x0000000011077df0 2024.10.17 22:58:23.530362 [ 5233 ] {} BaseDaemon: 20. DB::SharedMergeMutateTaskBase::executeStep() @ 0x0000000011435a3c 2024.10.17 22:58:23.530384 [ 5233 ] {} BaseDaemon: 21. DB::MergeTreeBackgroundExecutor::threadFunction() @ 0x000000001108b234 2024.10.17 22:58:23.530410 [ 5233 ] {} BaseDaemon: 22. ThreadPoolImpl>::worker(std::__list_iterator, void*>) @ 0x000000000c52e264 2024.10.17 22:58:23.530448 [ 5233 ] {} BaseDaemon: 23. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl>::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>(void&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000c531dd0 2024.10.17 22:58:23.530476 [ 5233 ] {} BaseDaemon: 24. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c530a80 2024.10.17 22:58:23.530514 [ 5233 ] {} BaseDaemon: 25. ? @ 0x000000000007d5c8 2024.10.17 22:58:23.530534 [ 5233 ] {} BaseDaemon: 26. ? @ 0x00000000000e5edc 2024.10.17 22:58:23.530551 [ 5233 ] {} BaseDaemon: Integrity check of the executable skipped because the reference checksum could not be read. 2024.10.17 22:58:23.531083 [ 5233 ] {} BaseDaemon: Report this error to https://github.com/ClickHouse/ClickHouse/issues 2024.10.17 22:58:23.531294 [ 5233 ] {} BaseDaemon: Changed settings: max_insert_threads = 4, max_threads = 42, use_hedged_requests = false, distributed_foreground_insert = true, alter_sync = 0, enable_memory_bound_merging_of_aggregation_results = true, cluster_for_parallel_replicas = 'default', do_not_merge_across_partitions_select_final = false, log_queries = true, log_queries_probability = 1., max_http_get_redirects = 10, enable_deflate_qpl_codec = false, enable_zstd_qat_codec = false, query_profiler_real_time_period_ns = 0, query_profiler_cpu_time_period_ns = 0, max_bytes_before_external_group_by = 90194313216, max_bytes_before_external_sort = 90194313216, max_memory_usage = 180388626432, backup_restore_keeper_retry_max_backoff_ms = 60000, cancel_http_readonly_queries_on_client_close = true, max_table_size_to_drop = 1000000000000, max_partition_size_to_drop = 1000000000000, default_table_engine = 'ReplicatedMergeTree', mutations_sync = 0, optimize_trivial_insert_select = false, database_replicated_allow_only_replicated_engine = true, cloud_mode = true, cloud_mode_engine = 2, distributed_ddl_output_mode = 'none_only_active', distributed_ddl_entry_format_version = 6, async_insert_max_data_size = 10485760, async_insert_busy_timeout_max_ms = 1000, enable_filesystem_cache_on_write_operations = true, load_marks_asynchronously = true, allow_prefetched_read_pool_for_remote_filesystem = true, filesystem_prefetch_max_memory_usage = 18038862643, filesystem_prefetches_limit = 200, compatibility = '24.6', insert_keeper_max_retries = 20, allow_experimental_materialized_postgresql_table = false, date_time_input_format = 'best_effort' ```. [#70820](https://github.com/ClickHouse/ClickHouse/pull/70820) ([Michael Kolupaev](https://github.com/al13n321)). +* Disable enable_named_columns_in_function_tuple by default. [#70833](https://github.com/ClickHouse/ClickHouse/pull/70833) ([Raúl Marín](https://github.com/Algunenano)). +* Fix S3Queue table engine setting processing_threads_num not being effective in case it was deduced from the number of cpu cores on the server. [#70837](https://github.com/ClickHouse/ClickHouse/pull/70837) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Normalize named tuple arguments in aggregation states. This fixes [#69732](https://github.com/ClickHouse/ClickHouse/issues/69732) . [#70853](https://github.com/ClickHouse/ClickHouse/pull/70853) ([Amos Bird](https://github.com/amosbird)). +* Fix a logical error due to negative zeros in the two-level hash table. This closes [#70973](https://github.com/ClickHouse/ClickHouse/issues/70973). [#70979](https://github.com/ClickHouse/ClickHouse/pull/70979) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#71214](https://github.com/ClickHouse/ClickHouse/issues/71214): Fix logical error in `StorageS3Queue` "Cannot create a persistent node in /processed since it already exists". [#70984](https://github.com/ClickHouse/ClickHouse/pull/70984) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#71243](https://github.com/ClickHouse/ClickHouse/issues/71243): Fixed named sessions not being closed and hanging on forever under certain circumstances. [#70998](https://github.com/ClickHouse/ClickHouse/pull/70998) ([Márcio Martins](https://github.com/marcio-absmartly)). +* Backported in [#71157](https://github.com/ClickHouse/ClickHouse/issues/71157): Fix the bug that didn't consider _row_exists column in rebuild option of projection lightweight delete. [#71089](https://github.com/ClickHouse/ClickHouse/pull/71089) ([Shichao Jin](https://github.com/jsc0218)). +* Backported in [#71265](https://github.com/ClickHouse/ClickHouse/issues/71265): Fix wrong value in system.query_metric_log due to unexpected race condition. [#71124](https://github.com/ClickHouse/ClickHouse/pull/71124) ([Pablo Marcos](https://github.com/pamarcos)). +* Backported in [#71331](https://github.com/ClickHouse/ClickHouse/issues/71331): Fix async inserts with empty blocks via native protocol. [#71312](https://github.com/ClickHouse/ClickHouse/pull/71312) ([Anton Popov](https://github.com/CurtizJ)). + +#### Build/Testing/Packaging Improvement +* Docker in integration tests runner is updated to latest version. It was previously pinned u until patch release 24.0.3 was out. https://github.com/moby/moby/issues/45770#issuecomment-1618255130. - HDFS image was deprecated and not running with current docker version. Switched to newer version of a derivative image based on ubuntu. - HDFS tests were hardened to allow them to run with python-repeat. [#66867](https://github.com/ClickHouse/ClickHouse/pull/66867) ([Ilya Yatsishin](https://github.com/qoega)). +* Alpine docker images now use ubuntu 22.04 as glibc donor, results in upgrade of glibc version delivered with alpine images from 2.31 to 2.35. [#69033](https://github.com/ClickHouse/ClickHouse/pull/69033) ([filimonov](https://github.com/filimonov)). +* Makes dbms independent from clickhouse_functions. [#69914](https://github.com/ClickHouse/ClickHouse/pull/69914) ([Raúl Marín](https://github.com/Algunenano)). +* Fix FreeBSD compilation of the MariaDB connector. [#70007](https://github.com/ClickHouse/ClickHouse/pull/70007) ([Raúl Marín](https://github.com/Algunenano)). +* Building on Apple Mac OS X Darwin does not produce strange warnings anymore. [#70411](https://github.com/ClickHouse/ClickHouse/pull/70411) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix building with ARCH_NATIVE CMake flag. [#70585](https://github.com/ClickHouse/ClickHouse/pull/70585) ([Daniil Gentili](https://github.com/danog)). +* The universal installer will download Musl build on Alpine Linux. Some Docker containers are using Alpine Linux, but it was not possible to install ClickHouse there with `curl https://clickhouse.com/ | sh`. [#70767](https://github.com/ClickHouse/ClickHouse/pull/70767) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NO CL CATEGORY + +* Backported in [#71259](https://github.com/ClickHouse/ClickHouse/issues/71259):. [#71220](https://github.com/ClickHouse/ClickHouse/pull/71220) ([Raúl Marín](https://github.com/Algunenano)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "JSONCompactWithProgress query output format"'. [#69989](https://github.com/ClickHouse/ClickHouse/pull/69989) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Support CREATE OR REPLACE VIEW atomically"'. [#70535](https://github.com/ClickHouse/ClickHouse/pull/70535) ([Raúl Marín](https://github.com/Algunenano)). +* NO CL ENTRY: 'Revert "Revert "Support CREATE OR REPLACE VIEW atomically""'. [#70536](https://github.com/ClickHouse/ClickHouse/pull/70536) ([Raúl Marín](https://github.com/Algunenano)). +* NO CL ENTRY: 'Revert "Add projections size to system.projections"'. [#70858](https://github.com/ClickHouse/ClickHouse/pull/70858) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Allow writing argument of `has` or `hasAny` or `hasAll` as string values if array element type is `Enum`. [#56555](https://github.com/ClickHouse/ClickHouse/pull/56555) ([Duc Canh Le](https://github.com/canhld94)). +* Rename FileSegmentKind::Ephemeral and other changes. [#66600](https://github.com/ClickHouse/ClickHouse/pull/66600) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Closes [#67345](https://github.com/ClickHouse/ClickHouse/issues/67345). [#67346](https://github.com/ClickHouse/ClickHouse/pull/67346) ([KrJin](https://github.com/jincong8973)). +* Because it is too complicated to support. [#68410](https://github.com/ClickHouse/ClickHouse/pull/68410) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix 01600_parts_states_metrics_long flakiness. [#68521](https://github.com/ClickHouse/ClickHouse/pull/68521) ([Azat Khuzhin](https://github.com/azat)). +* Reduce client start time in debug/sanitizer mode. [#68980](https://github.com/ClickHouse/ClickHouse/pull/68980) ([Raúl Marín](https://github.com/Algunenano)). +* Closes [#69038](https://github.com/ClickHouse/ClickHouse/issues/69038). [#69040](https://github.com/ClickHouse/ClickHouse/pull/69040) ([Nikolay Degterinsky](https://github.com/evillique)). +* Better exception for unsupported full_text index with non-full parts. [#69067](https://github.com/ClickHouse/ClickHouse/pull/69067) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Catch additional zk connection erros while creating table and make sure to cleanup dirs if necessary for retries. [#69093](https://github.com/ClickHouse/ClickHouse/pull/69093) ([Sumit](https://github.com/sum12)). +* Update version_date.tsv and changelog after v24.7.5.37-stable. [#69185](https://github.com/ClickHouse/ClickHouse/pull/69185) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* DOCS: Replace live view with refreshable since the former is deprecated. [#69392](https://github.com/ClickHouse/ClickHouse/pull/69392) ([Damian Kula](https://github.com/heavelock)). +* Update ORC to the current HEAD. [#69473](https://github.com/ClickHouse/ClickHouse/pull/69473) ([Nikita Taranov](https://github.com/nickitat)). +* Make a test ready for flaky check. [#69586](https://github.com/ClickHouse/ClickHouse/pull/69586) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Support antlr parser to parse sql with some keywords as alias, make the behaviour same as the clickhouse-server - remove redundant `for` in the `keyword` field. [#69614](https://github.com/ClickHouse/ClickHouse/pull/69614) ([Z.H.](https://github.com/onlyacat)). +* Allow default implementations for null in function mapFromArrays for spark compatiability in apache gluten. Current change doesn't have any side effects on clickhouse in theory. [#69715](https://github.com/ClickHouse/ClickHouse/pull/69715) ([李扬](https://github.com/taiyang-li)). +* Fix exception message in AzureBlobStorage. [#69728](https://github.com/ClickHouse/ClickHouse/pull/69728) ([Pavel Kruglov](https://github.com/Avogar)). +* Add test parsing s3 URL with a bucket name including a dot. [#69743](https://github.com/ClickHouse/ClickHouse/pull/69743) ([Kaushik Iska](https://github.com/iskakaushik)). +* Make `clang-tidy` happy. [#69765](https://github.com/ClickHouse/ClickHouse/pull/69765) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Prepare to enable `clang-tidy` `readability-else-after-return`. [#69768](https://github.com/ClickHouse/ClickHouse/pull/69768) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* S3Queue: support having deprecated settings to not fail server startup. [#69769](https://github.com/ClickHouse/ClickHouse/pull/69769) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Use only adaptive heuristic to choose task sizes for remote reading. [#69778](https://github.com/ClickHouse/ClickHouse/pull/69778) ([Nikita Taranov](https://github.com/nickitat)). +* Remove unused buggy code. [#69780](https://github.com/ClickHouse/ClickHouse/pull/69780) ([Raúl Marín](https://github.com/Algunenano)). +* Fix bugfix check. [#69789](https://github.com/ClickHouse/ClickHouse/pull/69789) ([Antonio Andelic](https://github.com/antonio2368)). +* Followup for [#63279](https://github.com/ClickHouse/ClickHouse/issues/63279). [#69790](https://github.com/ClickHouse/ClickHouse/pull/69790) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Update version after release. [#69816](https://github.com/ClickHouse/ClickHouse/pull/69816) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update ext-dict-functions.md. [#69819](https://github.com/ClickHouse/ClickHouse/pull/69819) ([kurikuQwQ](https://github.com/kurikuQwQ)). +* Allow cyrillic characters in generated contributor names. [#69820](https://github.com/ClickHouse/ClickHouse/pull/69820) ([Raúl Marín](https://github.com/Algunenano)). +* CI: praktika integration 1. [#69822](https://github.com/ClickHouse/ClickHouse/pull/69822) ([Max Kainov](https://github.com/maxknv)). +* Fix `test_delayed_replica_failover`. [#69826](https://github.com/ClickHouse/ClickHouse/pull/69826) ([Antonio Andelic](https://github.com/antonio2368)). +* minor change, less conflicts. [#69830](https://github.com/ClickHouse/ClickHouse/pull/69830) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Improve error message DDLWorker.cpp. [#69835](https://github.com/ClickHouse/ClickHouse/pull/69835) ([Denny Crane](https://github.com/den-crane)). +* Fix typo in description: mutation_sync -> mutations_sync. [#69838](https://github.com/ClickHouse/ClickHouse/pull/69838) ([Alexander Gololobov](https://github.com/davenger)). +* Fix changelog. [#69841](https://github.com/ClickHouse/ClickHouse/pull/69841) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* This closes [#49940](https://github.com/ClickHouse/ClickHouse/issues/49940). [#69842](https://github.com/ClickHouse/ClickHouse/pull/69842) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* This closes [#51036](https://github.com/ClickHouse/ClickHouse/issues/51036). [#69844](https://github.com/ClickHouse/ClickHouse/pull/69844) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update README.md - Update meetups. [#69849](https://github.com/ClickHouse/ClickHouse/pull/69849) ([Tanya Bragin](https://github.com/tbragin)). +* Revert [#69790](https://github.com/ClickHouse/ClickHouse/issues/69790) and [#63279](https://github.com/ClickHouse/ClickHouse/issues/63279). [#69850](https://github.com/ClickHouse/ClickHouse/pull/69850) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* See [#63279](https://github.com/ClickHouse/ClickHouse/issues/63279). [#69851](https://github.com/ClickHouse/ClickHouse/pull/69851) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#50928](https://github.com/ClickHouse/ClickHouse/issues/50928). [#69852](https://github.com/ClickHouse/ClickHouse/pull/69852) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#55981](https://github.com/ClickHouse/ClickHouse/issues/55981). [#69853](https://github.com/ClickHouse/ClickHouse/pull/69853) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#56823](https://github.com/ClickHouse/ClickHouse/issues/56823). [#69854](https://github.com/ClickHouse/ClickHouse/pull/69854) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* This closes [#62350](https://github.com/ClickHouse/ClickHouse/issues/62350). [#69855](https://github.com/ClickHouse/ClickHouse/pull/69855) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Refactor functions and variables in statistics code. [#69860](https://github.com/ClickHouse/ClickHouse/pull/69860) ([Robert Schulze](https://github.com/rschu1ze)). +* Resubmit [#63279](https://github.com/ClickHouse/ClickHouse/issues/63279). [#69861](https://github.com/ClickHouse/ClickHouse/pull/69861) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Improve stateless test runner. [#69864](https://github.com/ClickHouse/ClickHouse/pull/69864) ([Alexey Katsman](https://github.com/alexkats)). +* Adjust fast test time limit a bit. [#69874](https://github.com/ClickHouse/ClickHouse/pull/69874) ([Raúl Marín](https://github.com/Algunenano)). +* Add initial 24.9 CHANGELOG. [#69876](https://github.com/ClickHouse/ClickHouse/pull/69876) ([Raúl Marín](https://github.com/Algunenano)). +* Fix test `01278_random_string_utf8`. [#69878](https://github.com/ClickHouse/ClickHouse/pull/69878) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix minor fuzzer issue with experimental statistics. [#69881](https://github.com/ClickHouse/ClickHouse/pull/69881) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix linking after settings refactoring. [#69882](https://github.com/ClickHouse/ClickHouse/pull/69882) ([Robert Schulze](https://github.com/rschu1ze)). +* Add Proj Obsolete Setting. [#69883](https://github.com/ClickHouse/ClickHouse/pull/69883) ([Shichao Jin](https://github.com/jsc0218)). +* Improve remote queries startup time. [#69884](https://github.com/ClickHouse/ClickHouse/pull/69884) ([Igor Nikonov](https://github.com/devcrafter)). +* Revert "Merge pull request [#69032](https://github.com/ClickHouse/ClickHouse/issues/69032) from alexon1234/include_real_time_execution_in_http_header". [#69885](https://github.com/ClickHouse/ClickHouse/pull/69885) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* A dedicated commits from https://github.com/ClickHouse/ClickHouse/pull/61473. [#69896](https://github.com/ClickHouse/ClickHouse/pull/69896) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Added aliases `time_bucket`(from TimescaleDB) and `date_bin`(from PostgreSQL) for `toStartOfInterval`. [#69900](https://github.com/ClickHouse/ClickHouse/pull/69900) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* RIPE is an acronym and thus should be capital. RIPE stands for **R**ACE **I**ntegrity **P**rimitives **E**valuation and RACE stands for **R**esearch and Development in **A**dvanced **C**ommunications **T**echnologies in **E**urope. [#69901](https://github.com/ClickHouse/ClickHouse/pull/69901) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Replace error codes with error names in stateless tests. [#69906](https://github.com/ClickHouse/ClickHouse/pull/69906) ([Dmitry Novik](https://github.com/novikd)). +* Move setting to 24.10. [#69913](https://github.com/ClickHouse/ClickHouse/pull/69913) ([Raúl Marín](https://github.com/Algunenano)). +* Minor: Reduce diff between public and private repo. [#69928](https://github.com/ClickHouse/ClickHouse/pull/69928) ([Robert Schulze](https://github.com/rschu1ze)). +* Followup for [#69861](https://github.com/ClickHouse/ClickHouse/issues/69861). [#69930](https://github.com/ClickHouse/ClickHouse/pull/69930) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Fix test_dictionaries_all_layouts_separate_sources. [#69962](https://github.com/ClickHouse/ClickHouse/pull/69962) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Fix test_keeper_mntr_data_size. [#69965](https://github.com/ClickHouse/ClickHouse/pull/69965) ([Antonio Andelic](https://github.com/antonio2368)). +* This closes [#49823](https://github.com/ClickHouse/ClickHouse/issues/49823). [#69981](https://github.com/ClickHouse/ClickHouse/pull/69981) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add changelog for 24.9. [#69982](https://github.com/ClickHouse/ClickHouse/pull/69982) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#45303](https://github.com/ClickHouse/ClickHouse/issues/45303). [#69987](https://github.com/ClickHouse/ClickHouse/pull/69987) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update CHANGELOG.md. [#69988](https://github.com/ClickHouse/ClickHouse/pull/69988) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update README.md. [#69991](https://github.com/ClickHouse/ClickHouse/pull/69991) ([Tyler Hannan](https://github.com/tylerhannan)). +* Disable `03215_parallel_replicas_crash_after_refactoring.sql` for Azure. [#69992](https://github.com/ClickHouse/ClickHouse/pull/69992) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Update CHANGELOG.md. [#69993](https://github.com/ClickHouse/ClickHouse/pull/69993) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update CHANGELOG.md. [#70004](https://github.com/ClickHouse/ClickHouse/pull/70004) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Revert "Add RIPEMD160 function". [#70005](https://github.com/ClickHouse/ClickHouse/pull/70005) ([Robert Schulze](https://github.com/rschu1ze)). +* Update CHANGELOG.md. [#70009](https://github.com/ClickHouse/ClickHouse/pull/70009) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update CHANGELOG.md. [#70010](https://github.com/ClickHouse/ClickHouse/pull/70010) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make the pylint stricter. [#70013](https://github.com/ClickHouse/ClickHouse/pull/70013) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Added a setting `restore_replace_external_dictionary_source_to_null` which enables replacing dictionary source with Null on restore for external dictionaries (useful for testing). [#70032](https://github.com/ClickHouse/ClickHouse/pull/70032) ([Alexander Tokmakov](https://github.com/tavplubix)). +* `isort` is a simple import sorter for the python to comply [pep-8](https://peps.python.org/pep-0008/#imports) requirements. It will allow to decrease conflicts during sync and beautify the code. The import block is divided into three sub-blocks: `standard library` -> `third-party libraries` -> `local imports` -> `.local imports`. Each sub-block is ordered alphabetically with sub-sub-blocks `import X` -> `from X import Y`. [#70038](https://github.com/ClickHouse/ClickHouse/pull/70038) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update version_date.tsv and changelog after v24.9.1.3278-stable. [#70049](https://github.com/ClickHouse/ClickHouse/pull/70049) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Despite the fact that we set the org-level workflow parameter `PYTHONUNBUFFERED`, it's not inherited in workflows. [#70050](https://github.com/ClickHouse/ClickHouse/pull/70050) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix ubsan issue in function sqid. [#70061](https://github.com/ClickHouse/ClickHouse/pull/70061) ([Robert Schulze](https://github.com/rschu1ze)). +* Delete a setting change. [#70071](https://github.com/ClickHouse/ClickHouse/pull/70071) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix `test_distributed_ddl`. [#70075](https://github.com/ClickHouse/ClickHouse/pull/70075) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove unused placeholder from exception message string. [#70086](https://github.com/ClickHouse/ClickHouse/pull/70086) ([Alsu Giliazova](https://github.com/alsugiliazova)). +* Better exception message when some of the permission is missing. [#70088](https://github.com/ClickHouse/ClickHouse/pull/70088) ([pufit](https://github.com/pufit)). +* Make vector similarity indexes work with adaptive granularity. [#70101](https://github.com/ClickHouse/ClickHouse/pull/70101) ([Robert Schulze](https://github.com/rschu1ze)). +* Add missing columns `total_rows`, `data_compressed_bytes`, and `data_uncompressed_bytes` to `system.projections`. Part of https://github.com/ClickHouse/ClickHouse/pull/68901. [#70106](https://github.com/ClickHouse/ClickHouse/pull/70106) ([Jordi Villar](https://github.com/jrdi)). +* Make `00938_fix_rwlock_segfault_long` non flaky. [#70109](https://github.com/ClickHouse/ClickHouse/pull/70109) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove TODO. [#70110](https://github.com/ClickHouse/ClickHouse/pull/70110) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change the default threshold to enable hyper threading. [#70111](https://github.com/ClickHouse/ClickHouse/pull/70111) ([Jiebin Sun](https://github.com/jiebinn)). +* Fixed [#69092](https://github.com/ClickHouse/ClickHouse/issues/69092): if `materialized_postgresql_tables_list=table1(id, code),table(id,name)` (`table1` has name that is a substring for `table`) `getTableAllowedColumns` method returns `[id, code]` for `table` before this fix. [#70114](https://github.com/ClickHouse/ClickHouse/pull/70114) ([Kruglov Kirill](https://github.com/1on)). +* Reduce log level. [#70117](https://github.com/ClickHouse/ClickHouse/pull/70117) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Rename `getNumberOfPhysicalCPUCores` and fix its decription. [#70130](https://github.com/ClickHouse/ClickHouse/pull/70130) ([Nikita Taranov](https://github.com/nickitat)). +* Adding 24.10. [#70132](https://github.com/ClickHouse/ClickHouse/pull/70132) ([Tyler Hannan](https://github.com/tylerhannan)). +* (Re?)-enable libcxx asserts for debug builds. [#70134](https://github.com/ClickHouse/ClickHouse/pull/70134) ([Robert Schulze](https://github.com/rschu1ze)). +* Refactor reading from object storage. [#70141](https://github.com/ClickHouse/ClickHouse/pull/70141) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Silence UBSAN for integer overflows in some datetime functions. [#70142](https://github.com/ClickHouse/ClickHouse/pull/70142) ([Michael Kolupaev](https://github.com/al13n321)). +* Improve pipdeptree generator for docker images. - Update requirements.txt for the integration tests runner container - Remove some small dependencies, improve `helpers/retry_decorator.py` - Upgrade docker-compose from EOL version 1 to version 2. [#70146](https://github.com/ClickHouse/ClickHouse/pull/70146) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix 'QueryPlan was not initialized' in 'loop' with empty MergeTree. [#70149](https://github.com/ClickHouse/ClickHouse/pull/70149) ([Michael Kolupaev](https://github.com/al13n321)). +* Remove QueryPlan DataStream. [#70158](https://github.com/ClickHouse/ClickHouse/pull/70158) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update test_storage_s3_queue/test.py. [#70159](https://github.com/ClickHouse/ClickHouse/pull/70159) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Small docs fix. [#70160](https://github.com/ClickHouse/ClickHouse/pull/70160) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Test: PR local plan, non-constant in source stream. [#70173](https://github.com/ClickHouse/ClickHouse/pull/70173) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix performance checks. [#70175](https://github.com/ClickHouse/ClickHouse/pull/70175) ([Antonio Andelic](https://github.com/antonio2368)). +* Simplify test 03246_range_literal_replacement_works. [#70176](https://github.com/ClickHouse/ClickHouse/pull/70176) ([Pablo Marcos](https://github.com/pamarcos)). +* Update 01079_parallel_alter_add_drop_column_zookeeper.sh. [#70196](https://github.com/ClickHouse/ClickHouse/pull/70196) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Require bugfix job for a set of labels. [#70197](https://github.com/ClickHouse/ClickHouse/pull/70197) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* CI: Praktika integration, fast test. [#70239](https://github.com/ClickHouse/ClickHouse/pull/70239) ([Max Kainov](https://github.com/maxknv)). +* Avoid `Cannot schedule a task` error when loading parts. [#70257](https://github.com/ClickHouse/ClickHouse/pull/70257) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Bump usearch to v2.15.2 and SimSIMD to v5.0.0. [#70270](https://github.com/ClickHouse/ClickHouse/pull/70270) ([Robert Schulze](https://github.com/rschu1ze)). +* Instead of balancing tests by `crc32(file_name)` we'll use `add tests to a group with a minimal number of tests`. [#70272](https://github.com/ClickHouse/ClickHouse/pull/70272) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Closes [#70263](https://github.com/ClickHouse/ClickHouse/issues/70263). [#70273](https://github.com/ClickHouse/ClickHouse/pull/70273) ([flynn](https://github.com/ucasfl)). +* Hide MergeTreeSettings implementation. [#70285](https://github.com/ClickHouse/ClickHouse/pull/70285) ([Raúl Marín](https://github.com/Algunenano)). +* CI: Remove await feature from release branches. [#70294](https://github.com/ClickHouse/ClickHouse/pull/70294) ([Max Kainov](https://github.com/maxknv)). +* Fix `test_keeper_four_word_command`. [#70298](https://github.com/ClickHouse/ClickHouse/pull/70298) ([Antonio Andelic](https://github.com/antonio2368)). +* Update version_date.tsv and changelog after v24.9.2.42-stable. [#70301](https://github.com/ClickHouse/ClickHouse/pull/70301) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Synchronize settings with private. [#70320](https://github.com/ClickHouse/ClickHouse/pull/70320) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add Ignore Option In DeduplicateMergeProjectionMode. [#70327](https://github.com/ClickHouse/ClickHouse/pull/70327) ([Shichao Jin](https://github.com/jsc0218)). +* CI: Enable Integration Tests for backport PRs. [#70329](https://github.com/ClickHouse/ClickHouse/pull/70329) ([Max Kainov](https://github.com/maxknv)). +* There is [a failed CI job](https://s3.amazonaws.com/clickhouse-test-reports/69778/2d81c38874958bd9d54a25524173bdb1ddf2b75c/stateless_tests__release_.html) which is triggered by [03237_create_or_replace_view_atomically_with_atomic_engine](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/03237_create_or_replace_view_atomically_with_atomic_engine.sh). [#70330](https://github.com/ClickHouse/ClickHouse/pull/70330) ([tuanpach](https://github.com/tuanpach)). +* Fix flaky test `03237_insert_sparse_columns_mem`. [#70333](https://github.com/ClickHouse/ClickHouse/pull/70333) ([Anton Popov](https://github.com/CurtizJ)). +* Rename enable_secure_identifiers -> enforce_strict_identifier_format. [#70335](https://github.com/ClickHouse/ClickHouse/pull/70335) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Attempt to fix flaky RabbitMQ tests. Maybe closes [#45160](https://github.com/ClickHouse/ClickHouse/issues/45160). [#70336](https://github.com/ClickHouse/ClickHouse/pull/70336) ([filimonov](https://github.com/filimonov)). +* Don't fail the stateless check script if we can't collect minio logs. [#70350](https://github.com/ClickHouse/ClickHouse/pull/70350) ([Raúl Marín](https://github.com/Algunenano)). +* Fix tiny mistake, responsible for some of kafka test flaps. Example [report](https://s3.amazonaws.com/clickhouse-test-reports/0/3198aafac59c368993e7b5f49d95674cc1b1be18/integration_tests__release__[2_4].html). [#70352](https://github.com/ClickHouse/ClickHouse/pull/70352) ([filimonov](https://github.com/filimonov)). +* Closes [#69634](https://github.com/ClickHouse/ClickHouse/issues/69634). [#70354](https://github.com/ClickHouse/ClickHouse/pull/70354) ([pufit](https://github.com/pufit)). +* Fix 02346_fulltext_index_bug52019. [#70357](https://github.com/ClickHouse/ClickHouse/pull/70357) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Use new JSON for collecting minio logs. [#70359](https://github.com/ClickHouse/ClickHouse/pull/70359) ([Antonio Andelic](https://github.com/antonio2368)). +* Update comments in VectorSimilarityCondition (WHERE is not supported). [#70360](https://github.com/ClickHouse/ClickHouse/pull/70360) ([Azat Khuzhin](https://github.com/azat)). +* Remove 02492_clickhouse_local_context_uaf test. [#70363](https://github.com/ClickHouse/ClickHouse/pull/70363) ([Azat Khuzhin](https://github.com/azat)). +* Fix `clang-19` build issues. [#70412](https://github.com/ClickHouse/ClickHouse/pull/70412) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Ignore "Invalid multibyte data detected" error during completion. [#70422](https://github.com/ClickHouse/ClickHouse/pull/70422) ([Azat Khuzhin](https://github.com/azat)). +* Make QueryPlan explain methods const. [#70444](https://github.com/ClickHouse/ClickHouse/pull/70444) ([Alexander Gololobov](https://github.com/davenger)). +* Fix 0.1 second delay for interactive queries (due to keystroke interceptor). [#70445](https://github.com/ClickHouse/ClickHouse/pull/70445) ([Azat Khuzhin](https://github.com/azat)). +* Increase lock timeout in attempt to fix 02125_many_mutations. [#70448](https://github.com/ClickHouse/ClickHouse/pull/70448) ([Azat Khuzhin](https://github.com/azat)). +* Fix order in 03249_dynamic_alter_consistency. [#70453](https://github.com/ClickHouse/ClickHouse/pull/70453) ([Alexander Gololobov](https://github.com/davenger)). +* Fix refreshable MV in system database breaking server startup. [#70460](https://github.com/ClickHouse/ClickHouse/pull/70460) ([Michael Kolupaev](https://github.com/al13n321)). +* Fix flaky test_refreshable_mv_in_replicated_db. [#70462](https://github.com/ClickHouse/ClickHouse/pull/70462) ([Michael Kolupaev](https://github.com/al13n321)). +* Update version_date.tsv and changelog after v24.8.5.115-lts. [#70463](https://github.com/ClickHouse/ClickHouse/pull/70463) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Decrease probability of "Server died" due to 00913_many_threads. [#70473](https://github.com/ClickHouse/ClickHouse/pull/70473) ([Azat Khuzhin](https://github.com/azat)). +* Fixes for killing leftovers in clikhouse-test. [#70474](https://github.com/ClickHouse/ClickHouse/pull/70474) ([Azat Khuzhin](https://github.com/azat)). +* Update version_date.tsv and changelog after v24.3.12.75-lts. [#70485](https://github.com/ClickHouse/ClickHouse/pull/70485) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Use logging instead of print. [#70505](https://github.com/ClickHouse/ClickHouse/pull/70505) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Remove slow poll() logs in keeper. [#70508](https://github.com/ClickHouse/ClickHouse/pull/70508) ([Raúl Marín](https://github.com/Algunenano)). +* Add timeouts for retry loops in test_storage_rabbitmq. It should prevent cascading failures of the whole test suite caused by deadloop in one of the test scenarios. Also added small sleeps in a 'tight' loops to make retries bit less agressive. [#70510](https://github.com/ClickHouse/ClickHouse/pull/70510) ([filimonov](https://github.com/filimonov)). +* CI: Fix for canceled Sync workflow. [#70521](https://github.com/ClickHouse/ClickHouse/pull/70521) ([Max Kainov](https://github.com/maxknv)). +* Debug build faild with clang-18 after https://github.com/ClickHouse/ClickHouse/pull/70412, don't know why it's ok in release build, simply changing `_` to `_1` is ok for both release and debug build. [#70532](https://github.com/ClickHouse/ClickHouse/pull/70532) ([Chang chen](https://github.com/baibaichen)). +* Refreshable materialized views are not experimental anymore. [#70550](https://github.com/ClickHouse/ClickHouse/pull/70550) ([Michael Kolupaev](https://github.com/al13n321)). +* Fix 24.9 setting compatibility `database_replicated_allow_explicit_uuid`. [#70565](https://github.com/ClickHouse/ClickHouse/pull/70565) ([Nikita Fomichev](https://github.com/fm4v)). +* Fix typos. [#70588](https://github.com/ClickHouse/ClickHouse/pull/70588) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Vector search: allow to specify HNSW parameter `ef_search` at query time. [#70616](https://github.com/ClickHouse/ClickHouse/pull/70616) ([Robert Schulze](https://github.com/rschu1ze)). +* Increase max_rows_to_read limit in some tests. [#70617](https://github.com/ClickHouse/ClickHouse/pull/70617) ([Raúl Marín](https://github.com/Algunenano)). +* Reduce sync efforts with private. [#70634](https://github.com/ClickHouse/ClickHouse/pull/70634) ([Raúl Marín](https://github.com/Algunenano)). +* Fix parsing of some formats into sparse columns. [#70635](https://github.com/ClickHouse/ClickHouse/pull/70635) ([Anton Popov](https://github.com/CurtizJ)). +* Fix typos. [#70637](https://github.com/ClickHouse/ClickHouse/pull/70637) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Try fix 00180_no_seek_avoiding_when_reading_from_cache. [#70640](https://github.com/ClickHouse/ClickHouse/pull/70640) ([Kseniia Sumarokova](https://github.com/kssenii)). +* When the `PR Check` status is set, it's a valid RunConfig job failure. [#70643](https://github.com/ClickHouse/ClickHouse/pull/70643) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix timeout in materialized pg tests. [#70646](https://github.com/ClickHouse/ClickHouse/pull/70646) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Introduced MergeTree setting which allow to change merge selecting algorithm. However we still have only one algorithm and it's mostly for future experiments. [#70647](https://github.com/ClickHouse/ClickHouse/pull/70647) ([alesapin](https://github.com/alesapin)). +* Docs: Follow-up for [#70585](https://github.com/ClickHouse/ClickHouse/issues/70585). [#70654](https://github.com/ClickHouse/ClickHouse/pull/70654) ([Robert Schulze](https://github.com/rschu1ze)). +* Remove strange file. [#70662](https://github.com/ClickHouse/ClickHouse/pull/70662) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Locally I had lots of errors like `'AllocList' does not refer to a value` around places which used `offsetof`. Changing it to `__builtin_offsetof ` helped and I didn't debug any further. [#70671](https://github.com/ClickHouse/ClickHouse/pull/70671) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Adding the report link to a test result and files' list. [#70677](https://github.com/ClickHouse/ClickHouse/pull/70677) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* materialized postgres: minor fixes. [#70710](https://github.com/ClickHouse/ClickHouse/pull/70710) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Probably fix flaky test_refreshable_mv_in_replicated_db. [#70714](https://github.com/ClickHouse/ClickHouse/pull/70714) ([Michael Kolupaev](https://github.com/al13n321)). +* Move more setting structs to pImpl. [#70739](https://github.com/ClickHouse/ClickHouse/pull/70739) ([Raúl Marín](https://github.com/Algunenano)). +* Reduce sync effort. [#70747](https://github.com/ClickHouse/ClickHouse/pull/70747) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#71198](https://github.com/ClickHouse/ClickHouse/issues/71198): Check number of arguments for function with Dynamic argument. [#70749](https://github.com/ClickHouse/ClickHouse/pull/70749) ([Nikita Taranov](https://github.com/nickitat)). +* Add s3queue settings check for cloud. [#70750](https://github.com/ClickHouse/ClickHouse/pull/70750) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix readiness/health check for OpenLDAP container. [#70755](https://github.com/ClickHouse/ClickHouse/pull/70755) ([Julian Maicher](https://github.com/jmaicher)). +* Allow update plan headers for all the steps. [#70761](https://github.com/ClickHouse/ClickHouse/pull/70761) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Autogenerate documentation for settings. [#70768](https://github.com/ClickHouse/ClickHouse/pull/70768) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Not a logical error. [#70770](https://github.com/ClickHouse/ClickHouse/pull/70770) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* CI: Aarch64 build with Asan. [#70778](https://github.com/ClickHouse/ClickHouse/pull/70778) ([Max Kainov](https://github.com/maxknv)). +* Minor fix. [#70783](https://github.com/ClickHouse/ClickHouse/pull/70783) ([Anton Popov](https://github.com/CurtizJ)). +* The docs for settings should be located in the source code. Now, the CI supports that. [#70784](https://github.com/ClickHouse/ClickHouse/pull/70784) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update style-test image. [#70785](https://github.com/ClickHouse/ClickHouse/pull/70785) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Avoid double finalization of `WriteBuffer` in library bridge. [#70799](https://github.com/ClickHouse/ClickHouse/pull/70799) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Make Array Field serialization consistent. [#70803](https://github.com/ClickHouse/ClickHouse/pull/70803) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* A follow-up for [#70785](https://github.com/ClickHouse/ClickHouse/issues/70785), [jwt](https://pypi.org/project/jwt/#history) looks very outdated, and we have issue with conflicting paths. [#70815](https://github.com/ClickHouse/ClickHouse/pull/70815) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Remove inneficient code. [#70816](https://github.com/ClickHouse/ClickHouse/pull/70816) ([Raúl Marín](https://github.com/Algunenano)). +* Allow large object files if OMIT_HEAVY_DEBUG_SYMBOLS = 0. [#70818](https://github.com/ClickHouse/ClickHouse/pull/70818) ([Michael Kolupaev](https://github.com/al13n321)). +* Add test with distributed queries for 15768. [#70834](https://github.com/ClickHouse/ClickHouse/pull/70834) ([Nikita Taranov](https://github.com/nickitat)). +* More setting structs to pImpl and reuse code. [#70840](https://github.com/ClickHouse/ClickHouse/pull/70840) ([Raúl Marín](https://github.com/Algunenano)). +* Update default HNSW parameter settings. [#70873](https://github.com/ClickHouse/ClickHouse/pull/70873) ([Robert Schulze](https://github.com/rschu1ze)). +* Limiting logging some lines about configs. [#70879](https://github.com/ClickHouse/ClickHouse/pull/70879) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Fix `limit by`, `limit with ties` for distributed and parallel replicas. [#70880](https://github.com/ClickHouse/ClickHouse/pull/70880) ([Nikita Taranov](https://github.com/nickitat)). +* Fix darwin build. [#70894](https://github.com/ClickHouse/ClickHouse/pull/70894) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add dots for consistency. [#70909](https://github.com/ClickHouse/ClickHouse/pull/70909) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Logical error fix for substrings, found by fuzzer. [#70914](https://github.com/ClickHouse/ClickHouse/pull/70914) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* More setting structs to pImpl. [#70942](https://github.com/ClickHouse/ClickHouse/pull/70942) ([Raúl Marín](https://github.com/Algunenano)). +* Add logging for mock HTTP servers used in minio integration tests. [#70943](https://github.com/ClickHouse/ClickHouse/pull/70943) ([Vitaly Baranov](https://github.com/vitlibar)). +* Minor fixups of [#70011](https://github.com/ClickHouse/ClickHouse/issues/70011) and [#69918](https://github.com/ClickHouse/ClickHouse/issues/69918). [#70959](https://github.com/ClickHouse/ClickHouse/pull/70959) ([Robert Schulze](https://github.com/rschu1ze)). +* CI: Do not skip Build report and status fix. [#70965](https://github.com/ClickHouse/ClickHouse/pull/70965) ([Max Kainov](https://github.com/maxknv)). +* Fix Keeper entry serialization compatibility. [#70972](https://github.com/ClickHouse/ClickHouse/pull/70972) ([Antonio Andelic](https://github.com/antonio2368)). +* Update exception message. [#70975](https://github.com/ClickHouse/ClickHouse/pull/70975) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix `utils/c++expr` option `-b`. [#70978](https://github.com/ClickHouse/ClickHouse/pull/70978) ([Sergei Trifonov](https://github.com/serxa)). +* Fix `test_keeper_broken_logs`. [#70982](https://github.com/ClickHouse/ClickHouse/pull/70982) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix `01039_test_setting_parse`. [#70986](https://github.com/ClickHouse/ClickHouse/pull/70986) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Tests for languages support for Embedded Dictionaries. [#71004](https://github.com/ClickHouse/ClickHouse/pull/71004) ([Max Vostrikov](https://github.com/max-vostrikov)). +* Required for internal test runs with the same image build in public CI. [#71008](https://github.com/ClickHouse/ClickHouse/pull/71008) ([Ilya Yatsishin](https://github.com/qoega)). +* Move remaining settings objects to pImpl and start simplification. [#71019](https://github.com/ClickHouse/ClickHouse/pull/71019) ([Raúl Marín](https://github.com/Algunenano)). +* CI: Rearrange directories for praktika ci. [#71029](https://github.com/ClickHouse/ClickHouse/pull/71029) ([Max Kainov](https://github.com/maxknv)). +* Fix assert in RemoteSource::onAsyncJobReady(). [#71034](https://github.com/ClickHouse/ClickHouse/pull/71034) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix showing error message in ReadBufferFromS3 when retrying. Without this PR information about a retryable failure in `ReadBufferFromS3` could look like this:. [#71038](https://github.com/ClickHouse/ClickHouse/pull/71038) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix `test_truncate_database`. [#71057](https://github.com/ClickHouse/ClickHouse/pull/71057) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix clickhouse-test useless 5 second delay in case of multiple threads are used. [#71069](https://github.com/ClickHouse/ClickHouse/pull/71069) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#71142](https://github.com/ClickHouse/ClickHouse/issues/71142): Followup [#70520](https://github.com/ClickHouse/ClickHouse/issues/70520). [#71129](https://github.com/ClickHouse/ClickHouse/pull/71129) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Backported in [#71189](https://github.com/ClickHouse/ClickHouse/issues/71189): Update compatibility setting for `hnsw_candidate_list_size_for_search`. [#71133](https://github.com/ClickHouse/ClickHouse/pull/71133) ([Robert Schulze](https://github.com/rschu1ze)). +* Backported in [#71222](https://github.com/ClickHouse/ClickHouse/issues/71222): Fixes for interactive metrics. [#71173](https://github.com/ClickHouse/ClickHouse/pull/71173) ([Julia Kartseva](https://github.com/jkartseva)). +* Backported in [#71205](https://github.com/ClickHouse/ClickHouse/issues/71205): Maybe not GWPAsan by default. [#71174](https://github.com/ClickHouse/ClickHouse/pull/71174) ([Antonio Andelic](https://github.com/antonio2368)). +* Backported in [#71277](https://github.com/ClickHouse/ClickHouse/issues/71277): Fix LOGICAL_ERROR on wrong scalar subquery argument to table functions. [#71216](https://github.com/ClickHouse/ClickHouse/pull/71216) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#71253](https://github.com/ClickHouse/ClickHouse/issues/71253): Disable enable_named_columns_in_function_tuple for 24.10. [#71219](https://github.com/ClickHouse/ClickHouse/pull/71219) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#71303](https://github.com/ClickHouse/ClickHouse/issues/71303): Improve system.query_metric_log to remove flakiness. [#71295](https://github.com/ClickHouse/ClickHouse/pull/71295) ([Pablo Marcos](https://github.com/pamarcos)). +* Backported in [#71317](https://github.com/ClickHouse/ClickHouse/issues/71317): Fix debug log timestamp. [#71311](https://github.com/ClickHouse/ClickHouse/pull/71311) ([Pablo Marcos](https://github.com/pamarcos)). + +#### Not for changeling + +* Reverted. [#69812](https://github.com/ClickHouse/ClickHouse/pull/69812) ([tuanpach](https://github.com/tuanpach)). + diff --git a/docs/changelogs/v24.3.13.40-lts.md b/docs/changelogs/v24.3.13.40-lts.md new file mode 100644 index 00000000000..bce45e88710 --- /dev/null +++ b/docs/changelogs/v24.3.13.40-lts.md @@ -0,0 +1,31 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.3.13.40-lts (7acabd77389) FIXME as compared to v24.3.12.75-lts (7cb5dff8019) + +#### Bug Fix (user-visible misbehavior in an official stable release) +* Backported in [#63976](https://github.com/ClickHouse/ClickHouse/issues/63976): Fix intersect parts when restart after drop range. [#63202](https://github.com/ClickHouse/ClickHouse/pull/63202) ([Han Fei](https://github.com/hanfei1991)). +* Backported in [#71482](https://github.com/ClickHouse/ClickHouse/issues/71482): Fix `Content-Encoding` not sent in some compressed responses. [#64802](https://github.com/ClickHouse/ClickHouse/issues/64802). [#68975](https://github.com/ClickHouse/ClickHouse/pull/68975) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Backported in [#70451](https://github.com/ClickHouse/ClickHouse/issues/70451): Fix vrash during insertion into FixedString column in PostgreSQL engine. [#69584](https://github.com/ClickHouse/ClickHouse/pull/69584) ([Pavel Kruglov](https://github.com/Avogar)). +* Backported in [#70619](https://github.com/ClickHouse/ClickHouse/issues/70619): Fix server segfault on creating a materialized view with two selects and an `INTERSECT`, e.g. `CREATE MATERIALIZED VIEW v0 AS (SELECT 1) INTERSECT (SELECT 1);`. [#70264](https://github.com/ClickHouse/ClickHouse/pull/70264) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Backported in [#70877](https://github.com/ClickHouse/ClickHouse/issues/70877): Fix table creation with `CREATE ... AS table_function()` with database `Replicated` and unavailable table function source on secondary replica. [#70511](https://github.com/ClickHouse/ClickHouse/pull/70511) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#70571](https://github.com/ClickHouse/ClickHouse/issues/70571): Ignore all output on async insert with `wait_for_async_insert=1`. Closes [#62644](https://github.com/ClickHouse/ClickHouse/issues/62644). [#70530](https://github.com/ClickHouse/ClickHouse/pull/70530) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Backported in [#71146](https://github.com/ClickHouse/ClickHouse/issues/71146): Ignore frozen_metadata.txt while traversing shadow directory from system.remote_data_paths. [#70590](https://github.com/ClickHouse/ClickHouse/pull/70590) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Backported in [#70682](https://github.com/ClickHouse/ClickHouse/issues/70682): Fix creation of stateful window functions on misaligned memory. [#70631](https://github.com/ClickHouse/ClickHouse/pull/70631) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#71113](https://github.com/ClickHouse/ClickHouse/issues/71113): Fix a crash and a leak in AggregateFunctionGroupArraySorted. [#70820](https://github.com/ClickHouse/ClickHouse/pull/70820) ([Michael Kolupaev](https://github.com/al13n321)). +* Backported in [#70990](https://github.com/ClickHouse/ClickHouse/issues/70990): Fix a logical error due to negative zeros in the two-level hash table. This closes [#70973](https://github.com/ClickHouse/ClickHouse/issues/70973). [#70979](https://github.com/ClickHouse/ClickHouse/pull/70979) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#71246](https://github.com/ClickHouse/ClickHouse/issues/71246): Fixed named sessions not being closed and hanging on forever under certain circumstances. [#70998](https://github.com/ClickHouse/ClickHouse/pull/70998) ([Márcio Martins](https://github.com/marcio-absmartly)). +* Backported in [#71371](https://github.com/ClickHouse/ClickHouse/issues/71371): Add try/catch to data parts destructors to avoid terminate. [#71364](https://github.com/ClickHouse/ClickHouse/pull/71364) ([alesapin](https://github.com/alesapin)). +* Backported in [#71594](https://github.com/ClickHouse/ClickHouse/issues/71594): Prevent crash in SortCursor with 0 columns (old analyzer). [#71494](https://github.com/ClickHouse/ClickHouse/pull/71494) ([Raúl Marín](https://github.com/Algunenano)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Backported in [#71022](https://github.com/ClickHouse/ClickHouse/issues/71022): Fix dropping of file cache in CHECK query in case of enabled transactions. [#69256](https://github.com/ClickHouse/ClickHouse/pull/69256) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#70384](https://github.com/ClickHouse/ClickHouse/issues/70384): CI: Enable Integration Tests for backport PRs. [#70329](https://github.com/ClickHouse/ClickHouse/pull/70329) ([Max Kainov](https://github.com/maxknv)). +* Backported in [#70538](https://github.com/ClickHouse/ClickHouse/issues/70538): Remove slow poll() logs in keeper. [#70508](https://github.com/ClickHouse/ClickHouse/pull/70508) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#70971](https://github.com/ClickHouse/ClickHouse/issues/70971): Limiting logging some lines about configs. [#70879](https://github.com/ClickHouse/ClickHouse/pull/70879) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). + diff --git a/docs/changelogs/v24.8.6.70-lts.md b/docs/changelogs/v24.8.6.70-lts.md new file mode 100644 index 00000000000..81fa4db1458 --- /dev/null +++ b/docs/changelogs/v24.8.6.70-lts.md @@ -0,0 +1,50 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.8.6.70-lts (ddb8c219771) FIXME as compared to v24.8.5.115-lts (8c4cb00a384) + +#### Backward Incompatible Change +* Backported in [#71359](https://github.com/ClickHouse/ClickHouse/issues/71359): Fix possible error `No such file or directory` due to unescaped special symbols in files for JSON subcolumns. [#71182](https://github.com/ClickHouse/ClickHouse/pull/71182) ([Pavel Kruglov](https://github.com/Avogar)). + +#### Improvement +* Backported in [#70680](https://github.com/ClickHouse/ClickHouse/issues/70680): Don't do validation when synchronizing user_directories from keeper. [#70644](https://github.com/ClickHouse/ClickHouse/pull/70644) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#71395](https://github.com/ClickHouse/ClickHouse/issues/71395): Do not call the object storage API when listing directories, as this may be cost-inefficient. Instead, store the list of filenames in the memory. The trade-offs are increased initial load time and memory required to store filenames. [#70823](https://github.com/ClickHouse/ClickHouse/pull/70823) ([Julia Kartseva](https://github.com/jkartseva)). +* Backported in [#71287](https://github.com/ClickHouse/ClickHouse/issues/71287): Reduce the number of object storage HEAD API requests in the plain_rewritable disk. [#70915](https://github.com/ClickHouse/ClickHouse/pull/70915) ([Julia Kartseva](https://github.com/jkartseva)). + +#### Bug Fix (user-visible misbehavior in an official stable release) +* Backported in [#70934](https://github.com/ClickHouse/ClickHouse/issues/70934): Fix incorrect JOIN ON section optimization in case of `IS NULL` check under any other function (like `NOT`) that may lead to wrong results. Closes [#67915](https://github.com/ClickHouse/ClickHouse/issues/67915). [#68049](https://github.com/ClickHouse/ClickHouse/pull/68049) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Backported in [#70735](https://github.com/ClickHouse/ClickHouse/issues/70735): Fix unexpected exception when passing empty tuple in array. This fixes [#68618](https://github.com/ClickHouse/ClickHouse/issues/68618). [#68848](https://github.com/ClickHouse/ClickHouse/pull/68848) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#71138](https://github.com/ClickHouse/ClickHouse/issues/71138): Fix propogating structure argument in s3Cluster. Previously the `DEFAULT` expression of the column could be lost when sending the query to the replicas in s3Cluster. [#69147](https://github.com/ClickHouse/ClickHouse/pull/69147) ([Pavel Kruglov](https://github.com/Avogar)). +* Backported in [#70561](https://github.com/ClickHouse/ClickHouse/issues/70561): Fix `getSubcolumn` with `LowCardinality` columns by overriding `useDefaultImplementationForLowCardinalityColumns` to return `true`. [#69831](https://github.com/ClickHouse/ClickHouse/pull/69831) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)). +* Backported in [#70903](https://github.com/ClickHouse/ClickHouse/issues/70903): Avoid reusing columns among different named tuples when evaluating `tuple` functions. This fixes [#70022](https://github.com/ClickHouse/ClickHouse/issues/70022). [#70103](https://github.com/ClickHouse/ClickHouse/pull/70103) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#70623](https://github.com/ClickHouse/ClickHouse/issues/70623): Fix server segfault on creating a materialized view with two selects and an `INTERSECT`, e.g. `CREATE MATERIALIZED VIEW v0 AS (SELECT 1) INTERSECT (SELECT 1);`. [#70264](https://github.com/ClickHouse/ClickHouse/pull/70264) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Backported in [#70688](https://github.com/ClickHouse/ClickHouse/issues/70688): Fix possible use-after-free in `SYSTEM DROP FORMAT SCHEMA CACHE FOR Protobuf`. [#70358](https://github.com/ClickHouse/ClickHouse/pull/70358) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#70494](https://github.com/ClickHouse/ClickHouse/issues/70494): Fix crash during GROUP BY JSON sub-object subcolumn. [#70374](https://github.com/ClickHouse/ClickHouse/pull/70374) ([Pavel Kruglov](https://github.com/Avogar)). +* Backported in [#70482](https://github.com/ClickHouse/ClickHouse/issues/70482): Don't prefetch parts for vertical merges if part has no rows. [#70452](https://github.com/ClickHouse/ClickHouse/pull/70452) ([Antonio Andelic](https://github.com/antonio2368)). +* Backported in [#70556](https://github.com/ClickHouse/ClickHouse/issues/70556): Fix crash in WHERE with lambda functions. [#70464](https://github.com/ClickHouse/ClickHouse/pull/70464) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#70878](https://github.com/ClickHouse/ClickHouse/issues/70878): Fix table creation with `CREATE ... AS table_function()` with database `Replicated` and unavailable table function source on secondary replica. [#70511](https://github.com/ClickHouse/ClickHouse/pull/70511) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#70575](https://github.com/ClickHouse/ClickHouse/issues/70575): Ignore all output on async insert with `wait_for_async_insert=1`. Closes [#62644](https://github.com/ClickHouse/ClickHouse/issues/62644). [#70530](https://github.com/ClickHouse/ClickHouse/pull/70530) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Backported in [#71052](https://github.com/ClickHouse/ClickHouse/issues/71052): Ignore frozen_metadata.txt while traversing shadow directory from system.remote_data_paths. [#70590](https://github.com/ClickHouse/ClickHouse/pull/70590) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Backported in [#70651](https://github.com/ClickHouse/ClickHouse/issues/70651): Fix creation of stateful window functions on misaligned memory. [#70631](https://github.com/ClickHouse/ClickHouse/pull/70631) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#70757](https://github.com/ClickHouse/ClickHouse/issues/70757): Fixed rare crashes in `SELECT`-s and merges after adding a column of `Array` type with non-empty default expression. [#70695](https://github.com/ClickHouse/ClickHouse/pull/70695) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#70763](https://github.com/ClickHouse/ClickHouse/issues/70763): Fix infinite recursion when infering a proto schema with skip unsupported fields enabled. [#70697](https://github.com/ClickHouse/ClickHouse/pull/70697) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#71118](https://github.com/ClickHouse/ClickHouse/issues/71118): `GroupArraySortedData` uses a PODArray with non-POD elements, manually calling constructors and destructors for the elements as needed. But it wasn't careful enough: in two places it forgot to call destructor, in one place it left elements uninitialized if an exception is thrown when deserializing previous elements. Then `GroupArraySortedData`'s destructor called destructors on uninitialized elements and crashed: ``` 2024.10.17 22:58:23.523790 [ 5233 ] {} BaseDaemon: ########## Short fault info ############ 2024.10.17 22:58:23.523834 [ 5233 ] {} BaseDaemon: (version 24.6.1.4609 (official build), build id: 5423339A6571004018D55BBE05D464AFA35E6718, git hash: fa6cdfda8a94890eb19bc7f22f8b0b56292f7a26) (from thread 682) Received signal 11 2024.10.17 22:58:23.523862 [ 5233 ] {} BaseDaemon: Signal description: Segmentation fault 2024.10.17 22:58:23.523883 [ 5233 ] {} BaseDaemon: Address: 0x8f. Access: . Address not mapped to object. 2024.10.17 22:58:23.523908 [ 5233 ] {} BaseDaemon: Stack trace: 0x0000aaaac4b78308 0x0000ffffb7701850 0x0000aaaac0104855 0x0000aaaac01048a0 0x0000aaaac501e84c 0x0000aaaac7c510d0 0x0000aaaac7c4ba20 0x0000aaaac968bbfc 0x0000aaaac968fab0 0x0000aaaac969bf50 0x0000aaaac9b7520c 0x0000aaaac9b74c74 0x0000aaaac9b8a150 0x0000aaaac9b809f0 0x0000aaaac9b80574 0x0000aaaac9b8e364 0x0000aaaac9b8e4fc 0x0000aaaac94f4328 0x0000aaaac94f428c 0x0000aaaac94f7df0 0x0000aaaac98b5a3c 0x0000aaaac950b234 0x0000aaaac49ae264 0x0000aaaac49b1dd0 0x0000aaaac49b0a80 0x0000ffffb755d5c8 0x0000ffffb75c5edc 2024.10.17 22:58:23.523936 [ 5233 ] {} BaseDaemon: ######################################## 2024.10.17 22:58:23.523959 [ 5233 ] {} BaseDaemon: (version 24.6.1.4609 (official build), build id: 5423339A6571004018D55BBE05D464AFA35E6718, git hash: fa6cdfda8a94890eb19bc7f22f8b0b56292f7a26) (from thread 682) (query_id: 6c8a33a2-f45a-4a3b-bd71-ded6a1c9ccd3::202410_534066_534078_2) (query: ) Received signal Segmentation fault (11) 2024.10.17 22:58:23.523977 [ 5233 ] {} BaseDaemon: Address: 0x8f. Access: . Address not mapped to object. 2024.10.17 22:58:23.523993 [ 5233 ] {} BaseDaemon: Stack trace: 0x0000aaaac4b78308 0x0000ffffb7701850 0x0000aaaac0104855 0x0000aaaac01048a0 0x0000aaaac501e84c 0x0000aaaac7c510d0 0x0000aaaac7c4ba20 0x0000aaaac968bbfc 0x0000aaaac968fab0 0x0000aaaac969bf50 0x0000aaaac9b7520c 0x0000aaaac9b74c74 0x0000aaaac9b8a150 0x0000aaaac9b809f0 0x0000aaaac9b80574 0x0000aaaac9b8e364 0x0000aaaac9b8e4fc 0x0000aaaac94f4328 0x0000aaaac94f428c 0x0000aaaac94f7df0 0x0000aaaac98b5a3c 0x0000aaaac950b234 0x0000aaaac49ae264 0x0000aaaac49b1dd0 0x0000aaaac49b0a80 0x0000ffffb755d5c8 0x0000ffffb75c5edc 2024.10.17 22:58:23.524817 [ 5233 ] {} BaseDaemon: 0. signalHandler(int, siginfo_t*, void*) @ 0x000000000c6f8308 2024.10.17 22:58:23.524917 [ 5233 ] {} BaseDaemon: 1. ? @ 0x0000ffffb7701850 2024.10.17 22:58:23.524962 [ 5233 ] {} BaseDaemon: 2. DB::Field::~Field() @ 0x0000000007c84855 2024.10.17 22:58:23.525012 [ 5233 ] {} BaseDaemon: 3. DB::Field::~Field() @ 0x0000000007c848a0 2024.10.17 22:58:23.526626 [ 5233 ] {} BaseDaemon: 4. DB::IAggregateFunctionDataHelper, DB::(anonymous namespace)::GroupArraySorted, DB::Field>>::destroy(char*) const (.5a6a451027f732f9fd91c13f4a13200c) @ 0x000000000cb9e84c 2024.10.17 22:58:23.527322 [ 5233 ] {} BaseDaemon: 5. DB::SerializationAggregateFunction::deserializeBinaryBulk(DB::IColumn&, DB::ReadBuffer&, unsigned long, double) const @ 0x000000000f7d10d0 2024.10.17 22:58:23.528470 [ 5233 ] {} BaseDaemon: 6. DB::ISerialization::deserializeBinaryBulkWithMultipleStreams(COW::immutable_ptr&, unsigned long, DB::ISerialization::DeserializeBinaryBulkSettings&, std::shared_ptr&, std::unordered_map::immutable_ptr, std::hash, std::equal_to, std::allocator::immutable_ptr>>>*) const @ 0x000000000f7cba20 2024.10.17 22:58:23.529213 [ 5233 ] {} BaseDaemon: 7. DB::MergeTreeReaderCompact::readData(DB::NameAndTypePair const&, COW::immutable_ptr&, unsigned long, std::function const&) @ 0x000000001120bbfc 2024.10.17 22:58:23.529277 [ 5233 ] {} BaseDaemon: 8. DB::MergeTreeReaderCompactSingleBuffer::readRows(unsigned long, unsigned long, bool, unsigned long, std::vector::immutable_ptr, std::allocator::immutable_ptr>>&) @ 0x000000001120fab0 2024.10.17 22:58:23.529319 [ 5233 ] {} BaseDaemon: 9. DB::MergeTreeSequentialSource::generate() @ 0x000000001121bf50 2024.10.17 22:58:23.529346 [ 5233 ] {} BaseDaemon: 10. DB::ISource::tryGenerate() @ 0x00000000116f520c 2024.10.17 22:58:23.529653 [ 5233 ] {} BaseDaemon: 11. DB::ISource::work() @ 0x00000000116f4c74 2024.10.17 22:58:23.529679 [ 5233 ] {} BaseDaemon: 12. DB::ExecutionThreadContext::executeTask() @ 0x000000001170a150 2024.10.17 22:58:23.529733 [ 5233 ] {} BaseDaemon: 13. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic*) @ 0x00000000117009f0 2024.10.17 22:58:23.529763 [ 5233 ] {} BaseDaemon: 14. DB::PipelineExecutor::executeStep(std::atomic*) @ 0x0000000011700574 2024.10.17 22:58:23.530089 [ 5233 ] {} BaseDaemon: 15. DB::PullingPipelineExecutor::pull(DB::Chunk&) @ 0x000000001170e364 2024.10.17 22:58:23.530277 [ 5233 ] {} BaseDaemon: 16. DB::PullingPipelineExecutor::pull(DB::Block&) @ 0x000000001170e4fc 2024.10.17 22:58:23.530295 [ 5233 ] {} BaseDaemon: 17. DB::MergeTask::ExecuteAndFinalizeHorizontalPart::executeImpl() @ 0x0000000011074328 2024.10.17 22:58:23.530318 [ 5233 ] {} BaseDaemon: 18. DB::MergeTask::ExecuteAndFinalizeHorizontalPart::execute() @ 0x000000001107428c 2024.10.17 22:58:23.530339 [ 5233 ] {} BaseDaemon: 19. DB::MergeTask::execute() @ 0x0000000011077df0 2024.10.17 22:58:23.530362 [ 5233 ] {} BaseDaemon: 20. DB::SharedMergeMutateTaskBase::executeStep() @ 0x0000000011435a3c 2024.10.17 22:58:23.530384 [ 5233 ] {} BaseDaemon: 21. DB::MergeTreeBackgroundExecutor::threadFunction() @ 0x000000001108b234 2024.10.17 22:58:23.530410 [ 5233 ] {} BaseDaemon: 22. ThreadPoolImpl>::worker(std::__list_iterator, void*>) @ 0x000000000c52e264 2024.10.17 22:58:23.530448 [ 5233 ] {} BaseDaemon: 23. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl>::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>(void&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000c531dd0 2024.10.17 22:58:23.530476 [ 5233 ] {} BaseDaemon: 24. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c530a80 2024.10.17 22:58:23.530514 [ 5233 ] {} BaseDaemon: 25. ? @ 0x000000000007d5c8 2024.10.17 22:58:23.530534 [ 5233 ] {} BaseDaemon: 26. ? @ 0x00000000000e5edc 2024.10.17 22:58:23.530551 [ 5233 ] {} BaseDaemon: Integrity check of the executable skipped because the reference checksum could not be read. 2024.10.17 22:58:23.531083 [ 5233 ] {} BaseDaemon: Report this error to https://github.com/ClickHouse/ClickHouse/issues 2024.10.17 22:58:23.531294 [ 5233 ] {} BaseDaemon: Changed settings: max_insert_threads = 4, max_threads = 42, use_hedged_requests = false, distributed_foreground_insert = true, alter_sync = 0, enable_memory_bound_merging_of_aggregation_results = true, cluster_for_parallel_replicas = 'default', do_not_merge_across_partitions_select_final = false, log_queries = true, log_queries_probability = 1., max_http_get_redirects = 10, enable_deflate_qpl_codec = false, enable_zstd_qat_codec = false, query_profiler_real_time_period_ns = 0, query_profiler_cpu_time_period_ns = 0, max_bytes_before_external_group_by = 90194313216, max_bytes_before_external_sort = 90194313216, max_memory_usage = 180388626432, backup_restore_keeper_retry_max_backoff_ms = 60000, cancel_http_readonly_queries_on_client_close = true, max_table_size_to_drop = 1000000000000, max_partition_size_to_drop = 1000000000000, default_table_engine = 'ReplicatedMergeTree', mutations_sync = 0, optimize_trivial_insert_select = false, database_replicated_allow_only_replicated_engine = true, cloud_mode = true, cloud_mode_engine = 2, distributed_ddl_output_mode = 'none_only_active', distributed_ddl_entry_format_version = 6, async_insert_max_data_size = 10485760, async_insert_busy_timeout_max_ms = 1000, enable_filesystem_cache_on_write_operations = true, load_marks_asynchronously = true, allow_prefetched_read_pool_for_remote_filesystem = true, filesystem_prefetch_max_memory_usage = 18038862643, filesystem_prefetches_limit = 200, compatibility = '24.6', insert_keeper_max_retries = 20, allow_experimental_materialized_postgresql_table = false, date_time_input_format = 'best_effort' ```. [#70820](https://github.com/ClickHouse/ClickHouse/pull/70820) ([Michael Kolupaev](https://github.com/al13n321)). +* Backported in [#70896](https://github.com/ClickHouse/ClickHouse/issues/70896): Disable enable_named_columns_in_function_tuple by default. [#70833](https://github.com/ClickHouse/ClickHouse/pull/70833) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#70994](https://github.com/ClickHouse/ClickHouse/issues/70994): Fix a logical error due to negative zeros in the two-level hash table. This closes [#70973](https://github.com/ClickHouse/ClickHouse/issues/70973). [#70979](https://github.com/ClickHouse/ClickHouse/pull/70979) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#71210](https://github.com/ClickHouse/ClickHouse/issues/71210): Fix logical error in `StorageS3Queue` "Cannot create a persistent node in /processed since it already exists". [#70984](https://github.com/ClickHouse/ClickHouse/pull/70984) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#71248](https://github.com/ClickHouse/ClickHouse/issues/71248): Fixed named sessions not being closed and hanging on forever under certain circumstances. [#70998](https://github.com/ClickHouse/ClickHouse/pull/70998) ([Márcio Martins](https://github.com/marcio-absmartly)). +* Backported in [#71375](https://github.com/ClickHouse/ClickHouse/issues/71375): Add try/catch to data parts destructors to avoid terminate. [#71364](https://github.com/ClickHouse/ClickHouse/pull/71364) ([alesapin](https://github.com/alesapin)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Backported in [#71026](https://github.com/ClickHouse/ClickHouse/issues/71026): Fix dropping of file cache in CHECK query in case of enabled transactions. [#69256](https://github.com/ClickHouse/ClickHouse/pull/69256) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#70388](https://github.com/ClickHouse/ClickHouse/issues/70388): CI: Enable Integration Tests for backport PRs. [#70329](https://github.com/ClickHouse/ClickHouse/pull/70329) ([Max Kainov](https://github.com/maxknv)). +* Backported in [#70701](https://github.com/ClickHouse/ClickHouse/issues/70701): Fix order in 03249_dynamic_alter_consistency. [#70453](https://github.com/ClickHouse/ClickHouse/pull/70453) ([Alexander Gololobov](https://github.com/davenger)). +* Backported in [#70542](https://github.com/ClickHouse/ClickHouse/issues/70542): Remove slow poll() logs in keeper. [#70508](https://github.com/ClickHouse/ClickHouse/pull/70508) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#70804](https://github.com/ClickHouse/ClickHouse/issues/70804): When the `PR Check` status is set, it's a valid RunConfig job failure. [#70643](https://github.com/ClickHouse/ClickHouse/pull/70643) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#71229](https://github.com/ClickHouse/ClickHouse/issues/71229): Maybe not GWPAsan by default. [#71174](https://github.com/ClickHouse/ClickHouse/pull/71174) ([Antonio Andelic](https://github.com/antonio2368)). + diff --git a/docs/en/engines/table-engines/integrations/embedded-rocksdb.md b/docs/en/engines/table-engines/integrations/embedded-rocksdb.md index 1958250ed73..41c4e8fc4a9 100644 --- a/docs/en/engines/table-engines/integrations/embedded-rocksdb.md +++ b/docs/en/engines/table-engines/integrations/embedded-rocksdb.md @@ -4,9 +4,13 @@ sidebar_position: 50 sidebar_label: EmbeddedRocksDB --- +import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge'; + # EmbeddedRocksDB Engine -This engine allows integrating ClickHouse with [rocksdb](http://rocksdb.org/). + + +This engine allows integrating ClickHouse with [RocksDB](http://rocksdb.org/). ## Creating a Table {#creating-a-table} diff --git a/docs/en/engines/table-engines/mergetree-family/annindexes.md b/docs/en/engines/table-engines/mergetree-family/annindexes.md index dc12a60e8ef..fcdc16637e6 100644 --- a/docs/en/engines/table-engines/mergetree-family/annindexes.md +++ b/docs/en/engines/table-engines/mergetree-family/annindexes.md @@ -54,7 +54,7 @@ Parameters: - `distance_function`: either `L2Distance` (the [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance) - the length of a line between two points in Euclidean space), or `cosineDistance` (the [cosine distance](https://en.wikipedia.org/wiki/Cosine_similarity#Cosine_distance)- the angle between two non-zero vectors). -- `quantization`: either `f64`, `f32`, `f16`, `bf16`, or `i8` for storing the vector with reduced precision (optional, default: `bf16`) +- `quantization`: either `f64`, `f32`, `f16`, `bf16`, or `i8` for storing vectors with reduced precision (optional, default: `bf16`) - `hnsw_max_connections_per_layer`: the number of neighbors per HNSW graph node, also known as `M` in the [HNSW paper](https://doi.org/10.1109/TPAMI.2018.2889473) (optional, default: 32) - `hnsw_candidate_list_size_for_construction`: the size of the dynamic candidate list when constructing the HNSW graph, also known as @@ -92,8 +92,8 @@ Vector similarity indexes currently support two distance functions: - `cosineDistance`, also called cosine similarity, is the cosine of the angle between two (non-zero) vectors ([Wikipedia](https://en.wikipedia.org/wiki/Cosine_similarity)). -Vector similarity indexes allows storing the vectors in reduced precision formats. Supported scalar kinds are `f64`, `f32`, `f16` or `i8`. -If no scalar kind was specified during index creation, `f16` is used as default. +Vector similarity indexes allows storing the vectors in reduced precision formats. Supported scalar kinds are `f64`, `f32`, `f16`, `bf16`, +and `i8`. If no scalar kind was specified during index creation, `bf16` is used as default. For normalized data, `L2Distance` is usually a better choice, otherwise `cosineDistance` is recommended to compensate for scale. If no distance function was specified during index creation, `L2Distance` is used as default. diff --git a/docs/en/getting-started/example-datasets/tpch.md b/docs/en/getting-started/example-datasets/tpch.md index 5fa0d779ecd..3ea4bffec38 100644 --- a/docs/en/getting-started/example-datasets/tpch.md +++ b/docs/en/getting-started/example-datasets/tpch.md @@ -33,6 +33,21 @@ Then, generate the data. Parameter `-s` specifies the scale factor. For example, ./dbgen -s 100 ``` +Detailed table sizes with scale factor 100: + +| Table | size (in rows) | size (compressed in ClickHouse) | +|----------|----------------|---------------------------------| +| nation | 25 | 2 kB | +| region | 5 | 1 kB | +| part | 20.000.000 | 895 MB | +| supplier | 1.000.000 | 75 MB | +| partsupp | 80.000.000 | 4.37 GB | +| customer | 15.000.000 | 1.19 GB | +| orders | 150.000.000 | 6.15 GB | +| lineitem | 600.00.00 | 26.69 GB | + +(Compressed sizes in ClickHouse are taken from `system.tables.total_bytes` and based on below table definitions.) + Now create tables in ClickHouse. We stick as closely as possible to the rules of the TPC-H specification: @@ -151,10 +166,37 @@ clickhouse-client --format_csv_delimiter '|' --query "INSERT INTO orders FORMAT clickhouse-client --format_csv_delimiter '|' --query "INSERT INTO lineitem FORMAT CSV" < lineitem.tbl ``` -The queries are generated by `./qgen -s `. Example queries for `s = 100`: +:::note +Instead of using tpch-kit and generating the tables by yourself, you can alternatively import the data from a public S3 bucket. Make sure +to create empty tables first using above `CREATE` statements. + +```sql +-- Scaling factor 1 +INSERT INTO nation SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/nation.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +INSERT INTO region SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/region.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +INSERT INTO part SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/part.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +INSERT INTO supplier SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/supplier.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +INSERT INTO partsupp SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/partsupp.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +INSERT INTO customer SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/customer.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +INSERT INTO orders SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/orders.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +INSERT INTO lineitem SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/lineitem.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; + +-- Scaling factor 100 +INSERT INTO nation SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/nation.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +INSERT INTO region SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/region.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +INSERT INTO part SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/part.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +INSERT INTO supplier SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/supplier.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +INSERT INTO partsupp SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/partsupp.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +INSERT INTO customer SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/customer.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +INSERT INTO orders SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/orders.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +INSERT INTO lineitem SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/lineitem.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1; +```` +::: ## Queries +The queries are generated by `./qgen -s `. Example queries for `s = 100`: + **Correctness** The result of the queries agrees with the official results unless mentioned otherwise. To verify, generate a TPC-H database with scale diff --git a/docs/en/getting-started/index.md b/docs/en/getting-started/index.md index b520220984c..7898ca01129 100644 --- a/docs/en/getting-started/index.md +++ b/docs/en/getting-started/index.md @@ -23,6 +23,7 @@ functions in ClickHouse. The sample datasets include: - The [NYPD Complaint Data](../getting-started/example-datasets/nypd_complaint_data.md) demonstrates how to use data inference to simplify creating tables - The ["What's on the Menu?" dataset](../getting-started/example-datasets/menus.md) has an example of denormalizing data - The [Laion dataset](../getting-started/example-datasets/laion.md) has an example of [Approximate nearest neighbor search indexes](../engines/table-engines/mergetree-family/annindexes.md) usage +- The [TPC-H](../getting-started/example-datasets/tpch.md), [TPC-DS](../getting-started/example-datasets/tpcds.md), and [Star Schema (SSB)](../getting-started/example-datasets/star-schema.md) industry benchmarks for analytics databases - [Getting Data Into ClickHouse - Part 1](https://clickhouse.com/blog/getting-data-into-clickhouse-part-1) provides examples of defining a schema and loading a small Hacker News dataset - [Getting Data Into ClickHouse - Part 3 - Using S3](https://clickhouse.com/blog/getting-data-into-clickhouse-part-3-s3) has examples of loading data from s3 - [Generating random data in ClickHouse](https://clickhouse.com/blog/generating-random-test-distribution-data-for-clickhouse) shows how to generate random data if none of the above fit your needs. diff --git a/docs/en/interfaces/prometheus.md b/docs/en/interfaces/prometheus.md index 8e7023cc51f..11f503b54d7 100644 --- a/docs/en/interfaces/prometheus.md +++ b/docs/en/interfaces/prometheus.md @@ -9,7 +9,7 @@ sidebar_label: Prometheus protocols ## Exposing metrics {#expose} :::note -ClickHouse Cloud does not currently support connecting to Prometheus. To be notified when this feature is supported, please contact support@clickhouse.com. +If you are using ClickHouse Cloud, you can expose metrics to Prometheus using the [Prometheus Integration](/en/integrations/prometheus). ::: ClickHouse can expose its own metrics for scraping from Prometheus: diff --git a/docs/en/operations/_troubleshooting.md b/docs/en/operations/_troubleshooting.md index 77389782675..f0ee1ca1d29 100644 --- a/docs/en/operations/_troubleshooting.md +++ b/docs/en/operations/_troubleshooting.md @@ -65,6 +65,34 @@ sudo rm -f /etc/yum.repos.d/clickhouse.repo After that follow the [install guide](../getting-started/install.md#from-rpm-packages) +### You Can't Run Docker Container + +You are running a simple `docker run clickhouse/clickhouse-server` and it crashes with a stack trace similar to following: + +``` +$ docker run -it clickhouse/clickhouse-server +........ +2024.11.06 21:04:48.912036 [ 1 ] {} SentryWriter: Sending crash reports is disabled +Poco::Exception. Code: 1000, e.code() = 0, System exception: cannot start thread, Stack trace (when copying this message, always include the lines below): + +0. Poco::ThreadImpl::startImpl(Poco::SharedPtr>) @ 0x00000000157c7b34 +1. Poco::Thread::start(Poco::Runnable&) @ 0x00000000157c8a0e +2. BaseDaemon::initializeTerminationAndSignalProcessing() @ 0x000000000d267a14 +3. BaseDaemon::initialize(Poco::Util::Application&) @ 0x000000000d2652cb +4. DB::Server::initialize(Poco::Util::Application&) @ 0x000000000d128b38 +5. Poco::Util::Application::run() @ 0x000000001581cfda +6. DB::Server::run() @ 0x000000000d1288f0 +7. Poco::Util::ServerApplication::run(int, char**) @ 0x0000000015825e27 +8. mainEntryClickHouseServer(int, char**) @ 0x000000000d125b38 +9. main @ 0x0000000007ea4eee +10. ? @ 0x00007f67ff946d90 +11. ? @ 0x00007f67ff946e40 +12. _start @ 0x00000000062e802e + (version 24.10.1.2812 (official build)) +``` + +The reason is an old docker daemon with version lower than `20.10.10`. A way to fix it either upgrading it, or running `docker run [--privileged | --security-opt seccomp=unconfined]`. The latter has security implications. + ## Connecting to the Server {#troubleshooting-accepts-no-connections} Possible issues: diff --git a/docs/en/operations/query-cache.md b/docs/en/operations/query-cache.md index 955cec0234e..f0941aa28aa 100644 --- a/docs/en/operations/query-cache.md +++ b/docs/en/operations/query-cache.md @@ -25,9 +25,10 @@ Query caches can generally be viewed as transactionally consistent or inconsiste slowly enough that the database only needs to compute the report once (represented by the first `SELECT` query). Further queries can be served directly from the query cache. In this example, a reasonable validity period could be 30 min. -Transactionally inconsistent caching is traditionally provided by client tools or proxy packages interacting with the database. As a result, -the same caching logic and configuration is often duplicated. With ClickHouse's query cache, the caching logic moves to the server side. -This reduces maintenance effort and avoids redundancy. +Transactionally inconsistent caching is traditionally provided by client tools or proxy packages (e.g. +[chproxy](https://www.chproxy.org/configuration/caching/)) interacting with the database. As a result, the same caching logic and +configuration is often duplicated. With ClickHouse's query cache, the caching logic moves to the server side. This reduces maintenance +effort and avoids redundancy. ## Configuration Settings and Usage @@ -138,7 +139,10 @@ is only cached if the query runs longer than 5 seconds. It is also possible to s cached - for that use setting [query_cache_min_query_runs](settings/settings.md#query-cache-min-query-runs). Entries in the query cache become stale after a certain time period (time-to-live). By default, this period is 60 seconds but a different -value can be specified at session, profile or query level using setting [query_cache_ttl](settings/settings.md#query-cache-ttl). +value can be specified at session, profile or query level using setting [query_cache_ttl](settings/settings.md#query-cache-ttl). The query +cache evicts entries "lazily", i.e. when an entry becomes stale, it is not immediately removed from the cache. Instead, when a new entry +is to be inserted into the query cache, the database checks whether the cache has enough free space for the new entry. If this is not the +case, the database tries to remove all stale entries. If the cache still has not enough free space, the new entry is not inserted. Entries in the query cache are compressed by default. This reduces the overall memory consumption at the cost of slower writes into / reads from the query cache. To disable compression, use setting [query_cache_compress_entries](settings/settings.md#query-cache-compress-entries). @@ -188,14 +192,9 @@ Also, results of queries with non-deterministic functions are not cached by defa To force caching of results of queries with non-deterministic functions regardless, use setting [query_cache_nondeterministic_function_handling](settings/settings.md#query-cache-nondeterministic-function-handling). -Results of queries that involve system tables, e.g. `system.processes` or `information_schema.tables`, are not cached by default. To force -caching of results of queries with system tables regardless, use setting -[query_cache_system_table_handling](settings/settings.md#query-cache-system-table-handling). - -:::note -Prior to ClickHouse v23.11, setting 'query_cache_store_results_of_queries_with_nondeterministic_functions = 0 / 1' controlled whether -results of queries with non-deterministic results were cached. In newer ClickHouse versions, this setting is obsolete and has no effect. -::: +Results of queries that involve system tables (e.g. [system.processes](system-tables/processes.md)` or +[information_schema.tables](system-tables/information_schema.md)) are not cached by default. To force caching of results of queries with +system tables regardless, use setting [query_cache_system_table_handling](settings/settings.md#query-cache-system-table-handling). Finally, entries in the query cache are not shared between users due to security reasons. For example, user A must not be able to bypass a row policy on a table by running the same query as another user B for whom no such policy exists. However, if necessary, cache entries can diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 02fa5a8ca58..c5f92ccdf68 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -131,16 +131,6 @@ Type: UInt64 Default: 8 -## background_pool_size - -Sets the number of threads performing background merges and mutations for tables with MergeTree engines. You can only increase the number of threads at runtime. To lower the number of threads you have to restart the server. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance. - -Before changing it, please also take a look at related MergeTree settings, such as `number_of_free_entries_in_pool_to_lower_max_size_of_merge` and `number_of_free_entries_in_pool_to_execute_mutation`. - -Type: UInt64 - -Default: 16 - ## background_schedule_pool_size The maximum number of threads that will be used for constantly executing some lightweight periodic operations for replicated tables, Kafka streaming, and DNS cache updates. diff --git a/docs/en/operations/system-tables/grants.md b/docs/en/operations/system-tables/grants.md index 262a53a87a5..debc3146008 100644 --- a/docs/en/operations/system-tables/grants.md +++ b/docs/en/operations/system-tables/grants.md @@ -19,7 +19,7 @@ Columns: - `column` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — Name of a column to which access is granted. - `is_partial_revoke` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Logical value. It shows whether some privileges have been revoked. Possible values: -- `0` — The row describes a partial revoke. -- `1` — The row describes a grant. +- `0` — The row describes a grant. +- `1` — The row describes a partial revoke. - `grant_option` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Permission is granted `WITH GRANT OPTION`, see [GRANT](../../sql-reference/statements/grant.md#granting-privilege-syntax). diff --git a/docs/en/sql-reference/aggregate-functions/reference/anylast.md b/docs/en/sql-reference/aggregate-functions/reference/anylast.md index 202d2e9fb10..4fe21531c76 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/anylast.md +++ b/docs/en/sql-reference/aggregate-functions/reference/anylast.md @@ -17,7 +17,7 @@ anyLast(column) [RESPECT NULLS] - `column`: The column name. :::note -Supports the `RESPECT NULLS` modifier after the function name. Using this modifier will ensure the function selects the first value passed, regardless of whether it is `NULL` or not. +Supports the `RESPECT NULLS` modifier after the function name. Using this modifier will ensure the function selects the last value passed, regardless of whether it is `NULL` or not. ::: **Returned value** @@ -40,4 +40,4 @@ SELECT anyLast(city) FROM any_last_nulls; ┌─anyLast(city)─┐ │ Valencia │ └───────────────┘ -``` \ No newline at end of file +``` diff --git a/docs/en/sql-reference/aggregate-functions/reference/index.md b/docs/en/sql-reference/aggregate-functions/reference/index.md index 2dce0afe2e1..ee8f0d5882e 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/index.md +++ b/docs/en/sql-reference/aggregate-functions/reference/index.md @@ -7,119 +7,4 @@ toc_hidden: true # List of Aggregate Functions -Standard aggregate functions: - -- [count](../reference/count.md) -- [min](../reference/min.md) -- [max](../reference/max.md) -- [sum](../reference/sum.md) -- [avg](../reference/avg.md) -- [any](../reference/any.md) -- [stddevPop](../reference/stddevpop.md) -- [stddevPopStable](../reference/stddevpopstable.md) -- [stddevSamp](../reference/stddevsamp.md) -- [stddevSampStable](../reference/stddevsampstable.md) -- [varPop](../reference/varpop.md) -- [varSamp](../reference/varsamp.md) -- [corr](../reference/corr.md) -- [corr](../reference/corrstable.md) -- [corrMatrix](../reference/corrmatrix.md) -- [covarPop](../reference/covarpop.md) -- [covarStable](../reference/covarpopstable.md) -- [covarPopMatrix](../reference/covarpopmatrix.md) -- [covarSamp](../reference/covarsamp.md) -- [covarSampStable](../reference/covarsampstable.md) -- [covarSampMatrix](../reference/covarsampmatrix.md) -- [entropy](../reference/entropy.md) -- [exponentialMovingAverage](../reference/exponentialmovingaverage.md) -- [intervalLengthSum](../reference/intervalLengthSum.md) -- [kolmogorovSmirnovTest](../reference/kolmogorovsmirnovtest.md) -- [mannwhitneyutest](../reference/mannwhitneyutest.md) -- [median](../reference/median.md) -- [rankCorr](../reference/rankCorr.md) -- [sumKahan](../reference/sumkahan.md) -- [studentTTest](../reference/studentttest.md) -- [welchTTest](../reference/welchttest.md) - -ClickHouse-specific aggregate functions: - -- [aggThrow](../reference/aggthrow.md) -- [analysisOfVariance](../reference/analysis_of_variance.md) -- [any](../reference/any.md) -- [anyHeavy](../reference/anyheavy.md) -- [anyLast](../reference/anylast.md) -- [boundingRatio](../reference/boundrat.md) -- [first_value](../reference/first_value.md) -- [last_value](../reference/last_value.md) -- [argMin](../reference/argmin.md) -- [argMax](../reference/argmax.md) -- [avgWeighted](../reference/avgweighted.md) -- [topK](../reference/topk.md) -- [topKWeighted](../reference/topkweighted.md) -- [deltaSum](../reference/deltasum.md) -- [deltaSumTimestamp](../reference/deltasumtimestamp.md) -- [flameGraph](../reference/flame_graph.md) -- [groupArray](../reference/grouparray.md) -- [groupArrayLast](../reference/grouparraylast.md) -- [groupUniqArray](../reference/groupuniqarray.md) -- [groupArrayInsertAt](../reference/grouparrayinsertat.md) -- [groupArrayMovingAvg](../reference/grouparraymovingavg.md) -- [groupArrayMovingSum](../reference/grouparraymovingsum.md) -- [groupArraySample](../reference/grouparraysample.md) -- [groupArraySorted](../reference/grouparraysorted.md) -- [groupArrayIntersect](../reference/grouparrayintersect.md) -- [groupBitAnd](../reference/groupbitand.md) -- [groupBitOr](../reference/groupbitor.md) -- [groupBitXor](../reference/groupbitxor.md) -- [groupBitmap](../reference/groupbitmap.md) -- [groupBitmapAnd](../reference/groupbitmapand.md) -- [groupBitmapOr](../reference/groupbitmapor.md) -- [groupBitmapXor](../reference/groupbitmapxor.md) -- [sumWithOverflow](../reference/sumwithoverflow.md) -- [sumMap](../reference/summap.md) -- [sumMapWithOverflow](../reference/summapwithoverflow.md) -- [sumMapFiltered](../parametric-functions.md/#summapfiltered) -- [sumMapFilteredWithOverflow](../parametric-functions.md/#summapfilteredwithoverflow) -- [minMap](../reference/minmap.md) -- [maxMap](../reference/maxmap.md) -- [skewSamp](../reference/skewsamp.md) -- [skewPop](../reference/skewpop.md) -- [kurtSamp](../reference/kurtsamp.md) -- [kurtPop](../reference/kurtpop.md) -- [uniq](../reference/uniq.md) -- [uniqExact](../reference/uniqexact.md) -- [uniqCombined](../reference/uniqcombined.md) -- [uniqCombined64](../reference/uniqcombined64.md) -- [uniqHLL12](../reference/uniqhll12.md) -- [uniqTheta](../reference/uniqthetasketch.md) -- [quantile](../reference/quantile.md) -- [quantiles](../reference/quantiles.md) -- [quantileExact](../reference/quantileexact.md) -- [quantileExactLow](../reference/quantileexact.md#quantileexactlow) -- [quantileExactHigh](../reference/quantileexact.md#quantileexacthigh) -- [quantileExactWeighted](../reference/quantileexactweighted.md) -- [quantileTiming](../reference/quantiletiming.md) -- [quantileTimingWeighted](../reference/quantiletimingweighted.md) -- [quantileDeterministic](../reference/quantiledeterministic.md) -- [quantileTDigest](../reference/quantiletdigest.md) -- [quantileTDigestWeighted](../reference/quantiletdigestweighted.md) -- [quantileBFloat16](../reference/quantilebfloat16.md#quantilebfloat16) -- [quantileBFloat16Weighted](../reference/quantilebfloat16.md#quantilebfloat16weighted) -- [quantileDD](../reference/quantileddsketch.md#quantileddsketch) -- [simpleLinearRegression](../reference/simplelinearregression.md) -- [singleValueOrNull](../reference/singlevalueornull.md) -- [stochasticLinearRegression](../reference/stochasticlinearregression.md) -- [stochasticLogisticRegression](../reference/stochasticlogisticregression.md) -- [categoricalInformationValue](../reference/categoricalinformationvalue.md) -- [contingency](../reference/contingency.md) -- [cramersV](../reference/cramersv.md) -- [cramersVBiasCorrected](../reference/cramersvbiascorrected.md) -- [theilsU](../reference/theilsu.md) -- [maxIntersections](../reference/maxintersections.md) -- [maxIntersectionsPosition](../reference/maxintersectionsposition.md) -- [meanZTest](../reference/meanztest.md) -- [quantileGK](../reference/quantileGK.md) -- [quantileInterpolatedWeighted](../reference/quantileinterpolatedweighted.md) -- [sparkBar](../reference/sparkbar.md) -- [sumCount](../reference/sumcount.md) -- [largestTriangleThreeBuckets](../reference/largestTriangleThreeBuckets.md) +ClickHouse supports all standard SQL aggregate functions ([sum](../reference/sum.md), [avg](../reference/avg.md), [min](../reference/min.md), [max](../reference/max.md), [count](../reference/count.md)), as well as a wide range of other aggregate functions. diff --git a/docs/en/sql-reference/data-types/aggregatefunction.md b/docs/en/sql-reference/data-types/aggregatefunction.md index 37f0d0e50ae..4cad27db68b 100644 --- a/docs/en/sql-reference/data-types/aggregatefunction.md +++ b/docs/en/sql-reference/data-types/aggregatefunction.md @@ -6,7 +6,9 @@ sidebar_label: AggregateFunction # AggregateFunction -Aggregate functions can have an implementation-defined intermediate state that can be serialized to an `AggregateFunction(...)` data type and stored in a table, usually, by means of [a materialized view](../../sql-reference/statements/create/view.md). The common way to produce an aggregate function state is by calling the aggregate function with the `-State` suffix. To get the final result of aggregation in the future, you must use the same aggregate function with the `-Merge`suffix. +Aggregate functions have an implementation-defined intermediate state that can be serialized to an `AggregateFunction(...)` data type and stored in a table, usually, by means of [a materialized view](../../sql-reference/statements/create/view.md). +The common way to produce an aggregate function state is by calling the aggregate function with the `-State` suffix. +To get the final result of aggregation in the future, you must use the same aggregate function with the `-Merge`suffix. `AggregateFunction(name, types_of_arguments...)` — parametric data type. diff --git a/docs/en/sql-reference/data-types/dynamic.md b/docs/en/sql-reference/data-types/dynamic.md index 5fbf47f7ef2..aa7455c8f68 100644 --- a/docs/en/sql-reference/data-types/dynamic.md +++ b/docs/en/sql-reference/data-types/dynamic.md @@ -512,6 +512,8 @@ The result of operator `<` for values `d1` with underlying type `T1` and `d2` wi - If `T1 = T2 = T`, the result will be `d1.T < d2.T` (underlying values will be compared). - If `T1 != T2`, the result will be `T1 < T2` (type names will be compared). +By default `Dynamic` type is not allowed in `GROUP BY`/`ORDER BY` keys, if you want to use it consider its special comparison rule and enable `allow_suspicious_types_in_group_by`/`allow_suspicious_types_in_order_by` settings. + Examples: ```sql CREATE TABLE test (d Dynamic) ENGINE=Memory; @@ -535,7 +537,7 @@ SELECT d, dynamicType(d) FROM test; ``` ```sql -SELECT d, dynamicType(d) FROM test ORDER BY d; +SELECT d, dynamicType(d) FROM test ORDER BY d SETTINGS allow_suspicious_types_in_order_by=1; ``` ```sql @@ -557,7 +559,7 @@ Example: ```sql CREATE TABLE test (d Dynamic) ENGINE=Memory; INSERT INTO test VALUES (1::UInt32), (1::Int64), (100::UInt32), (100::Int64); -SELECT d, dynamicType(d) FROM test ORDER by d; +SELECT d, dynamicType(d) FROM test ORDER BY d SETTINGS allow_suspicious_types_in_order_by=1; ``` ```text @@ -570,7 +572,7 @@ SELECT d, dynamicType(d) FROM test ORDER by d; ``` ```sql -SELECT d, dynamicType(d) FROM test GROUP by d; +SELECT d, dynamicType(d) FROM test GROUP by d SETTINGS allow_suspicious_types_in_group_by=1; ``` ```text @@ -582,7 +584,7 @@ SELECT d, dynamicType(d) FROM test GROUP by d; └─────┴────────────────┘ ``` -**Note**: the described comparison rule is not applied during execution of comparison functions like `<`/`>`/`=` and others because of [special work](#using-dynamic-type-in-functions) of functions with `Dynamic` type +**Note:** the described comparison rule is not applied during execution of comparison functions like `<`/`>`/`=` and others because of [special work](#using-dynamic-type-in-functions) of functions with `Dynamic` type ## Reaching the limit in number of different data types stored inside Dynamic diff --git a/docs/en/sql-reference/data-types/index.md b/docs/en/sql-reference/data-types/index.md index 2b89dd145e6..134678f71bb 100644 --- a/docs/en/sql-reference/data-types/index.md +++ b/docs/en/sql-reference/data-types/index.md @@ -6,29 +6,8 @@ sidebar_position: 1 # Data Types in ClickHouse -ClickHouse can store various kinds of data in table cells. This section describes the supported data types and special considerations for using and/or implementing them if any. +This section describes the data types supported by ClickHouse, for example [integers](int-uint.md), [floats](float.md) and [strings](string.md). -:::note -You can check whether a data type name is case-sensitive in the [system.data_type_families](../../operations/system-tables/data_type_families.md#system_tables-data_type_families) table. -::: - -ClickHouse data types include: - -- **Integer types**: [signed and unsigned integers](./int-uint.md) (`UInt8`, `UInt16`, `UInt32`, `UInt64`, `UInt128`, `UInt256`, `Int8`, `Int16`, `Int32`, `Int64`, `Int128`, `Int256`) -- **Floating-point numbers**: [floats](./float.md)(`Float32` and `Float64`) and [`Decimal` values](./decimal.md) -- **Boolean**: ClickHouse has a [`Boolean` type](./boolean.md) -- **Strings**: [`String`](./string.md) and [`FixedString`](./fixedstring.md) -- **Dates**: use [`Date`](./date.md) and [`Date32`](./date32.md) for days, and [`DateTime`](./datetime.md) and [`DateTime64`](./datetime64.md) for instances in time -- **Object**: the [`Object`](./json.md) stores a JSON document in a single column (deprecated) -- **JSON**: the [`JSON` object](./newjson.md) stores a JSON document in a single column -- **UUID**: a performant option for storing [`UUID` values](./uuid.md) -- **Low cardinality types**: use an [`Enum`](./enum.md) when you have a handful of unique values, or use [`LowCardinality`](./lowcardinality.md) when you have up to 10,000 unique values of a column -- **Arrays**: any column can be defined as an [`Array` of values](./array.md) -- **Maps**: use [`Map`](./map.md) for storing key/value pairs -- **Aggregation function types**: use [`SimpleAggregateFunction`](./simpleaggregatefunction.md) and [`AggregateFunction`](./aggregatefunction.md) for storing the intermediate status of aggregate function results -- **Nested data structures**: A [`Nested` data structure](./nested-data-structures/index.md) is like a table inside a cell -- **Tuples**: A [`Tuple` of elements](./tuple.md), each having an individual type. -- **Nullable**: [`Nullable`](./nullable.md) allows you to store a value as `NULL` when a value is "missing" (instead of the column settings its default value for the data type) -- **IP addresses**: use [`IPv4`](./ipv4.md) and [`IPv6`](./ipv6.md) to efficiently store IP addresses -- **Geo types**: for [geographical data](./geo.md), including `Point`, `Ring`, `Polygon` and `MultiPolygon` -- **Special data types**: including [`Expression`](./special-data-types/expression.md), [`Set`](./special-data-types/set.md), [`Nothing`](./special-data-types/nothing.md) and [`Interval`](./special-data-types/interval.md) +System table [system.data_type_families](../../operations/system-tables/data_type_families.md#system_tables-data_type_families) provides an +overview of all available data types. +It also shows whether a data type is an alias to another data type and its name is case-sensitive (e.g. `bool` vs. `BOOL`). diff --git a/docs/en/sql-reference/data-types/json.md b/docs/en/sql-reference/data-types/json.md index e48b308a620..ce69f15f0fa 100644 --- a/docs/en/sql-reference/data-types/json.md +++ b/docs/en/sql-reference/data-types/json.md @@ -7,7 +7,7 @@ keywords: [object, data type] # Object Data Type (deprecated) -**This feature is not production-ready and is now deprecated.** If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-formats/json/overview) instead. A new implementation to support JSON object is in progress and can be tracked [here](https://github.com/ClickHouse/ClickHouse/issues/54864). +**This feature is not production-ready and deprecated.** If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-formats/json/overview) instead. A new implementation to support JSON object is in progress and can be tracked [here](https://github.com/ClickHouse/ClickHouse/issues/54864).
diff --git a/docs/en/sql-reference/data-types/newjson.md b/docs/en/sql-reference/data-types/newjson.md index 7e6d4dd934f..4a21900545d 100644 --- a/docs/en/sql-reference/data-types/newjson.md +++ b/docs/en/sql-reference/data-types/newjson.md @@ -58,10 +58,10 @@ SELECT json FROM test; └───────────────────────────────────┘ ``` -Using CAST from 'String': +Using CAST from `String`: ```sql -SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::JSON as json; +SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::JSON AS json; ``` ```text @@ -70,7 +70,47 @@ SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::JSON as json └────────────────────────────────────────────────┘ ``` -CAST from `JSON`, named `Tuple`, `Map` and `Object('json')` to `JSON` type will be supported later. +Using CAST from `Tuple`: + +```sql +SELECT (tuple(42 AS b) AS a, [1, 2, 3] AS c, 'Hello, World!' AS d)::JSON AS json; +``` + +```text +┌─json───────────────────────────────────────────┐ +│ {"a":{"b":42},"c":[1,2,3],"d":"Hello, World!"} │ +└────────────────────────────────────────────────┘ +``` + +Using CAST from `Map`: + +```sql +SELECT map('a', map('b', 42), 'c', [1,2,3], 'd', 'Hello, World!')::JSON AS json; +``` + +```text +┌─json───────────────────────────────────────────┐ +│ {"a":{"b":42},"c":[1,2,3],"d":"Hello, World!"} │ +└────────────────────────────────────────────────┘ +``` + +Using CAST from deprecated `Object('json')`: + +```sql + SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::Object('json')::JSON AS json; + ``` + +```text +┌─json───────────────────────────────────────────┐ +│ {"a":{"b":42},"c":[1,2,3],"d":"Hello, World!"} │ +└────────────────────────────────────────────────┘ +``` + +:::note +CAST from `Tuple`/`Map`/`Object('json')` to `JSON` is implemented via serializing the column into `String` column containing JSON objects and deserializing it back to `JSON` type column. +::: + +CAST between `JSON` types with different arguments will be supported later. ## Reading JSON paths as subcolumns @@ -630,6 +670,28 @@ SELECT arrayJoin(distinctJSONPathsAndTypes(json)) FROM s3('s3://clickhouse-publi └─arrayJoin(distinctJSONPathsAndTypes(json))──────────────────┘ ``` +## ALTER MODIFY COLUMN to JSON type + +It's possible to alter an existing table and change the type of the column to the new `JSON` type. Right now only alter from `String` type is supported. + +**Example** + +```sql +CREATE TABLE test (json String) ENGINE=MergeTree ORDeR BY tuple(); +INSERT INTO test VALUES ('{"a" : 42}'), ('{"a" : 43, "b" : "Hello"}'), ('{"a" : 44, "b" : [1, 2, 3]}')), ('{"c" : "2020-01-01"}'); +ALTER TABLE test MODIFY COLUMN json JSON; +SELECT json, json.a, json.b, json.c FROM test; +``` + +```text +┌─json─────────────────────────┬─json.a─┬─json.b──┬─json.c─────┐ +│ {"a":"42"} │ 42 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ +│ {"a":"43","b":"Hello"} │ 43 │ Hello │ ᴺᵁᴸᴸ │ +│ {"a":"44","b":["1","2","3"]} │ 44 │ [1,2,3] │ ᴺᵁᴸᴸ │ +│ {"c":"2020-01-01"} │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 2020-01-01 │ +└──────────────────────────────┴────────┴─────────┴────────────┘ +``` + ## Tips for better usage of the JSON type Before creating `JSON` column and loading data into it, consider the following tips: diff --git a/docs/en/sql-reference/data-types/simpleaggregatefunction.md b/docs/en/sql-reference/data-types/simpleaggregatefunction.md index 4fb74ac30e4..8edd8b5b8ff 100644 --- a/docs/en/sql-reference/data-types/simpleaggregatefunction.md +++ b/docs/en/sql-reference/data-types/simpleaggregatefunction.md @@ -5,7 +5,9 @@ sidebar_label: SimpleAggregateFunction --- # SimpleAggregateFunction -`SimpleAggregateFunction(name, types_of_arguments...)` data type stores current value of the aggregate function, and does not store its full state as [`AggregateFunction`](../../sql-reference/data-types/aggregatefunction.md) does. This optimization can be applied to functions for which the following property holds: the result of applying a function `f` to a row set `S1 UNION ALL S2` can be obtained by applying `f` to parts of the row set separately, and then again applying `f` to the results: `f(S1 UNION ALL S2) = f(f(S1) UNION ALL f(S2))`. This property guarantees that partial aggregation results are enough to compute the combined one, so we do not have to store and process any extra data. +`SimpleAggregateFunction(name, types_of_arguments...)` data type stores current value (intermediate state) of the aggregate function, but not its full state as [`AggregateFunction`](../../sql-reference/data-types/aggregatefunction.md) does. +This optimization can be applied to functions for which the following property holds: the result of applying a function `f` to a row set `S1 UNION ALL S2` can be obtained by applying `f` to parts of the row set separately, and then again applying `f` to the results: `f(S1 UNION ALL S2) = f(f(S1) UNION ALL f(S2))`. +This property guarantees that partial aggregation results are enough to compute the combined one, so we do not have to store and process any extra data. The common way to produce an aggregate function value is by calling the aggregate function with the [-SimpleState](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-simplestate) suffix. diff --git a/docs/en/sql-reference/data-types/variant.md b/docs/en/sql-reference/data-types/variant.md index 3c2b6e0a362..7cb0f4ad4ea 100644 --- a/docs/en/sql-reference/data-types/variant.md +++ b/docs/en/sql-reference/data-types/variant.md @@ -441,6 +441,8 @@ SELECT v, variantType(v) FROM test ORDER by v; └─────┴────────────────┘ ``` +**Note** by default `Variant` type is not allowed in `GROUP BY`/`ORDER BY` keys, if you want to use it consider its special comparison rule and enable `allow_suspicious_types_in_group_by`/`allow_suspicious_types_in_order_by` settings. + ## JSONExtract functions with Variant All `JSONExtract*` functions support `Variant` type: diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 2357b5b2fdd..34dc6e996ee 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -4773,7 +4773,7 @@ Result: ## toUTCTimestamp -Convert DateTime/DateTime64 type value from other time zone to UTC timezone timestamp +Convert DateTime/DateTime64 type value from other time zone to UTC timezone timestamp. This function is mainly included for compatibility with Apache Spark and similar frameworks. **Syntax** @@ -4799,14 +4799,14 @@ SELECT toUTCTimestamp(toDateTime('2023-03-16'), 'Asia/Shanghai'); Result: ``` text -┌─toUTCTimestamp(toDateTime('2023-03-16'),'Asia/Shanghai')┐ +┌─toUTCTimestamp(toDateTime('2023-03-16'), 'Asia/Shanghai')┐ │ 2023-03-15 16:00:00 │ └─────────────────────────────────────────────────────────┘ ``` ## fromUTCTimestamp -Convert DateTime/DateTime64 type value from UTC timezone to other time zone timestamp +Convert DateTime/DateTime64 type value from UTC timezone to other time zone timestamp. This function is mainly included for compatibility with Apache Spark and similar frameworks. **Syntax** @@ -4832,7 +4832,7 @@ SELECT fromUTCTimestamp(toDateTime64('2023-03-16 10:00:00', 3), 'Asia/Shanghai') Result: ``` text -┌─fromUTCTimestamp(toDateTime64('2023-03-16 10:00:00',3),'Asia/Shanghai')─┐ +┌─fromUTCTimestamp(toDateTime64('2023-03-16 10:00:00',3), 'Asia/Shanghai')─┐ │ 2023-03-16 18:00:00.000 │ └─────────────────────────────────────────────────────────────────────────┘ ``` diff --git a/docs/en/sql-reference/functions/geo/index.md b/docs/en/sql-reference/functions/geo/index.md index d46e60281e2..51b6868611a 100644 --- a/docs/en/sql-reference/functions/geo/index.md +++ b/docs/en/sql-reference/functions/geo/index.md @@ -5,70 +5,4 @@ sidebar_position: 62 title: "Geo Functions" --- - -## Geographical Coordinates Functions - -- [greatCircleDistance](./coordinates.md#greatcircledistance) -- [geoDistance](./coordinates.md#geodistance) -- [greatCircleAngle](./coordinates.md#greatcircleangle) -- [pointInEllipses](./coordinates.md#pointinellipses) -- [pointInPolygon](./coordinates.md#pointinpolygon) - -## Geohash Functions -- [geohashEncode](./geohash.md#geohashencode) -- [geohashDecode](./geohash.md#geohashdecode) -- [geohashesInBox](./geohash.md#geohashesinbox) - -## H3 Indexes Functions - -- [h3IsValid](./h3.md#h3isvalid) -- [h3GetResolution](./h3.md#h3getresolution) -- [h3EdgeAngle](./h3.md#h3edgeangle) -- [h3EdgeLengthM](./h3.md#h3edgelengthm) -- [h3EdgeLengthKm](./h3.md#h3edgelengthkm) -- [geoToH3](./h3.md#geotoh3) -- [h3ToGeo](./h3.md#h3togeo) -- [h3ToGeoBoundary](./h3.md#h3togeoboundary) -- [h3kRing](./h3.md#h3kring) -- [h3GetBaseCell](./h3.md#h3getbasecell) -- [h3HexAreaM2](./h3.md#h3hexaream2) -- [h3HexAreaKm2](./h3.md#h3hexareakm2) -- [h3IndexesAreNeighbors](./h3.md#h3indexesareneighbors) -- [h3ToChildren](./h3.md#h3tochildren) -- [h3ToParent](./h3.md#h3toparent) -- [h3ToString](./h3.md#h3tostring) -- [stringToH3](./h3.md#stringtoh3) -- [h3GetResolution](./h3.md#h3getresolution) -- [h3IsResClassIII](./h3.md#h3isresclassiii) -- [h3IsPentagon](./h3.md#h3ispentagon) -- [h3GetFaces](./h3.md#h3getfaces) -- [h3CellAreaM2](./h3.md#h3cellaream2) -- [h3CellAreaRads2](./h3.md#h3cellarearads2) -- [h3ToCenterChild](./h3.md#h3tocenterchild) -- [h3ExactEdgeLengthM](./h3.md#h3exactedgelengthm) -- [h3ExactEdgeLengthKm](./h3.md#h3exactedgelengthkm) -- [h3ExactEdgeLengthRads](./h3.md#h3exactedgelengthrads) -- [h3NumHexagons](./h3.md#h3numhexagons) -- [h3Line](./h3.md#h3line) -- [h3Distance](./h3.md#h3distance) -- [h3HexRing](./h3.md#h3hexring) -- [h3GetUnidirectionalEdge](./h3.md#h3getunidirectionaledge) -- [h3UnidirectionalEdgeIsValid](./h3.md#h3unidirectionaledgeisvalid) -- [h3GetOriginIndexFromUnidirectionalEdge](./h3.md#h3getoriginindexfromunidirectionaledge) -- [h3GetDestinationIndexFromUnidirectionalEdge](./h3.md#h3getdestinationindexfromunidirectionaledge) -- [h3GetIndexesFromUnidirectionalEdge](./h3.md#h3getindexesfromunidirectionaledge) -- [h3GetUnidirectionalEdgesFromHexagon](./h3.md#h3getunidirectionaledgesfromhexagon) -- [h3GetUnidirectionalEdgeBoundary](./h3.md#h3getunidirectionaledgeboundary) - -## S2 Index Functions - -- [geoToS2](./s2.md#geotos2) -- [s2ToGeo](./s2.md#s2togeo) -- [s2GetNeighbors](./s2.md#s2getneighbors) -- [s2CellsIntersect](./s2.md#s2cellsintersect) -- [s2CapContains](./s2.md#s2capcontains) -- [s2CapUnion](./s2.md#s2capunion) -- [s2RectAdd](./s2.md#s2rectadd) -- [s2RectContains](./s2.md#s2rectcontains) -- [s2RectUnion](./s2.md#s2rectunion) -- [s2RectIntersection](./s2.md#s2rectintersection) +Functions for working with geometric objects, for example [to calculate distances between points on a sphere](./coordinates.md), [compute geohashes](./geohash.md), and work with [h3 indexes](./h3.md). diff --git a/docs/en/sql-reference/functions/index.md b/docs/en/sql-reference/functions/index.md index c0256ba4735..04a87c369ab 100644 --- a/docs/en/sql-reference/functions/index.md +++ b/docs/en/sql-reference/functions/index.md @@ -24,7 +24,7 @@ All expressions in a query that have the same AST (the same record or same resul ## Types of Results -All functions return a single return as the result (not several values, and not zero values). The type of result is usually defined only by the types of arguments, not by the values. Exceptions are the tupleElement function (the a.N operator), and the toFixedString function. +All functions return a single value as the result (not several values, and not zero values). The type of result is usually defined only by the types of arguments, not by the values. Exceptions are the tupleElement function (the a.N operator), and the toFixedString function. ## Constants diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index 29df041ccc6..fb16dacb7c8 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -279,7 +279,7 @@ For columns with a new or updated `MATERIALIZED` value expression, all existing For columns with a new or updated `DEFAULT` value expression, the behavior depends on the ClickHouse version: - In ClickHouse < v24.2, all existing rows are rewritten. -- ClickHouse >= v24.2 distinguishes if a row value in a column with `DEFAULT` value expression was explicitly specified when it was inserted, or not, i.e. calculated from the `DEFAULT` value expression. If the value was explicitly specified, ClickHouse keeps it as is. If the value was was calculated, ClickHouse changes it to the new or updated `MATERIALIZED` value expression. +- ClickHouse >= v24.2 distinguishes if a row value in a column with `DEFAULT` value expression was explicitly specified when it was inserted, or not, i.e. calculated from the `DEFAULT` value expression. If the value was explicitly specified, ClickHouse keeps it as is. If the value was calculated, ClickHouse changes it to the new or updated `MATERIALIZED` value expression. Syntax: diff --git a/docs/en/sql-reference/statements/create/index.md b/docs/en/sql-reference/statements/create/index.md index fa39526a53e..5854d7cf9d2 100644 --- a/docs/en/sql-reference/statements/create/index.md +++ b/docs/en/sql-reference/statements/create/index.md @@ -6,16 +6,4 @@ sidebar_label: CREATE # CREATE Queries -Create queries make a new entity of one of the following kinds: - -- [DATABASE](/docs/en/sql-reference/statements/create/database.md) -- [TABLE](/docs/en/sql-reference/statements/create/table.md) -- [VIEW](/docs/en/sql-reference/statements/create/view.md) -- [DICTIONARY](/docs/en/sql-reference/statements/create/dictionary.md) -- [FUNCTION](/docs/en/sql-reference/statements/create/function.md) -- [USER](/docs/en/sql-reference/statements/create/user.md) -- [ROLE](/docs/en/sql-reference/statements/create/role.md) -- [ROW POLICY](/docs/en/sql-reference/statements/create/row-policy.md) -- [QUOTA](/docs/en/sql-reference/statements/create/quota.md) -- [SETTINGS PROFILE](/docs/en/sql-reference/statements/create/settings-profile.md) -- [NAMED COLLECTION](/docs/en/sql-reference/statements/create/named-collection.md) +CREATE queries create (for example) new [databases](/docs/en/sql-reference/statements/create/database.md), [tables](/docs/en/sql-reference/statements/create/table.md) and [views](/docs/en/sql-reference/statements/create/view.md). diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index decb28d44d5..6decaf19d5b 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -117,6 +117,7 @@ GRANT SELECT ON db*.* TO john -- correct GRANT SELECT ON *.my_table TO john -- wrong GRANT SELECT ON foo*bar TO john -- wrong GRANT SELECT ON *suffix TO john -- wrong +GRANT SELECT(foo) ON db.table* TO john -- wrong ``` ## Privileges diff --git a/docs/en/sql-reference/statements/index.md b/docs/en/sql-reference/statements/index.md index 5aa61cf8d21..f288b30b27b 100644 --- a/docs/en/sql-reference/statements/index.md +++ b/docs/en/sql-reference/statements/index.md @@ -6,27 +6,4 @@ sidebar_label: List of statements # ClickHouse SQL Statements -Statements represent various kinds of action you can perform using SQL queries. Each kind of statement has it’s own syntax and usage details that are described separately: - -- [SELECT](/docs/en/sql-reference/statements/select/index.md) -- [INSERT INTO](/docs/en/sql-reference/statements/insert-into.md) -- [CREATE](/docs/en/sql-reference/statements/create/index.md) -- [ALTER](/docs/en/sql-reference/statements/alter/index.md) -- [SYSTEM](/docs/en/sql-reference/statements/system.md) -- [SHOW](/docs/en/sql-reference/statements/show.md) -- [GRANT](/docs/en/sql-reference/statements/grant.md) -- [REVOKE](/docs/en/sql-reference/statements/revoke.md) -- [ATTACH](/docs/en/sql-reference/statements/attach.md) -- [CHECK TABLE](/docs/en/sql-reference/statements/check-table.md) -- [DESCRIBE TABLE](/docs/en/sql-reference/statements/describe-table.md) -- [DETACH](/docs/en/sql-reference/statements/detach.md) -- [DROP](/docs/en/sql-reference/statements/drop.md) -- [EXISTS](/docs/en/sql-reference/statements/exists.md) -- [KILL](/docs/en/sql-reference/statements/kill.md) -- [OPTIMIZE](/docs/en/sql-reference/statements/optimize.md) -- [RENAME](/docs/en/sql-reference/statements/rename.md) -- [SET](/docs/en/sql-reference/statements/set.md) -- [SET ROLE](/docs/en/sql-reference/statements/set-role.md) -- [TRUNCATE](/docs/en/sql-reference/statements/truncate.md) -- [USE](/docs/en/sql-reference/statements/use.md) -- [EXPLAIN](/docs/en/sql-reference/statements/explain.md) +Users interact with ClickHouse using SQL statements. ClickHouse supports common SQL statements like [SELECT](select/index.md) and [CREATE](create/index.md), but it also provides specialized statements like [KILL](kill.md) and [OPTIMIZE](optimize.md). diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index 512a58d7cd9..25d2e7123fd 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -291,7 +291,7 @@ All missed values of `expr` column will be filled sequentially and other columns To fill multiple columns, add `WITH FILL` modifier with optional parameters after each field name in `ORDER BY` section. ``` sql -ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr] +ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr] [STALENESS const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr] [STALENESS numeric_expr] [INTERPOLATE [(col [AS expr], ... colN [AS exprN])]] ``` @@ -300,6 +300,7 @@ When `FROM const_expr` not defined sequence of filling use minimal `expr` field When `TO const_expr` not defined sequence of filling use maximum `expr` field value from `ORDER BY`. When `STEP const_numeric_expr` defined then `const_numeric_expr` interprets `as is` for numeric types, as `days` for Date type, as `seconds` for DateTime type. It also supports [INTERVAL](https://clickhouse.com/docs/en/sql-reference/data-types/special-data-types/interval/) data type representing time and date intervals. When `STEP const_numeric_expr` omitted then sequence of filling use `1.0` for numeric type, `1 day` for Date type and `1 second` for DateTime type. +When `STALENESS const_numeric_expr` is defined, the query will generate rows until the difference from the previous row in the original data exceeds `const_numeric_expr`. `INTERPOLATE` can be applied to columns not participating in `ORDER BY WITH FILL`. Such columns are filled based on previous fields values by applying `expr`. If `expr` is not present will repeat previous value. Omitted list will result in including all allowed columns. Example of a query without `WITH FILL`: @@ -497,6 +498,64 @@ Result: └────────────┴────────────┴──────────┘ ``` +Example of a query without `STALENESS`: + +``` sql +SELECT number as key, 5 * number value, 'original' AS source +FROM numbers(16) WHERE key % 5 == 0 +ORDER BY key WITH FILL; +``` + +Result: + +``` text + ┌─key─┬─value─┬─source───┐ + 1. │ 0 │ 0 │ original │ + 2. │ 1 │ 0 │ │ + 3. │ 2 │ 0 │ │ + 4. │ 3 │ 0 │ │ + 5. │ 4 │ 0 │ │ + 6. │ 5 │ 25 │ original │ + 7. │ 6 │ 0 │ │ + 8. │ 7 │ 0 │ │ + 9. │ 8 │ 0 │ │ +10. │ 9 │ 0 │ │ +11. │ 10 │ 50 │ original │ +12. │ 11 │ 0 │ │ +13. │ 12 │ 0 │ │ +14. │ 13 │ 0 │ │ +15. │ 14 │ 0 │ │ +16. │ 15 │ 75 │ original │ + └─────┴───────┴──────────┘ +``` + +Same query after applying `STALENESS 3`: + +``` sql +SELECT number as key, 5 * number value, 'original' AS source +FROM numbers(16) WHERE key % 5 == 0 +ORDER BY key WITH FILL STALENESS 3; +``` + +Result: + +``` text + ┌─key─┬─value─┬─source───┐ + 1. │ 0 │ 0 │ original │ + 2. │ 1 │ 0 │ │ + 3. │ 2 │ 0 │ │ + 4. │ 5 │ 25 │ original │ + 5. │ 6 │ 0 │ │ + 6. │ 7 │ 0 │ │ + 7. │ 10 │ 50 │ original │ + 8. │ 11 │ 0 │ │ + 9. │ 12 │ 0 │ │ +10. │ 15 │ 75 │ original │ +11. │ 16 │ 0 │ │ +12. │ 17 │ 0 │ │ + └─────┴───────┴──────────┘ +``` + Example of a query without `INTERPOLATE`: ``` sql diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md index f8a660fbec9..083ddc8c39c 100644 --- a/docs/ru/getting-started/install.md +++ b/docs/ru/getting-started/install.md @@ -95,7 +95,7 @@ sudo yum install -y clickhouse-server clickhouse-client sudo systemctl enable clickhouse-server sudo systemctl start clickhouse-server sudo systemctl status clickhouse-server -clickhouse-client # илм "clickhouse-client --password" если установлен пароль +clickhouse-client # или "clickhouse-client --password" если установлен пароль ``` Для использования наиболее свежих версий нужно заменить `stable` на `testing` (рекомендуется для тестовых окружений). Также иногда доступен `prestable`. diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index d7190444f0b..05e1e61be7b 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -431,7 +431,7 @@ catch (const Exception & e) bool need_print_stack_trace = config().getBool("stacktrace", false) && e.code() != ErrorCodes::NETWORK_ERROR; std::cerr << getExceptionMessage(e, need_print_stack_trace, true) << std::endl << std::endl; /// If exception code isn't zero, we should return non-zero return code anyway. - return e.code() ? e.code() : -1; + return static_cast(e.code()) ? e.code() : -1; } catch (...) { @@ -1390,7 +1390,8 @@ int mainEntryClickHouseClient(int argc, char ** argv) catch (const DB::Exception & e) { std::cerr << DB::getExceptionMessage(e, false) << std::endl; - return 1; + auto code = DB::getCurrentExceptionCode(); + return static_cast(code) ? code : 1; } catch (const boost::program_options::error & e) { @@ -1399,7 +1400,8 @@ int mainEntryClickHouseClient(int argc, char ** argv) } catch (...) { - std::cerr << DB::getCurrentExceptionMessage(true) << std::endl; - return 1; + std::cerr << DB::getCurrentExceptionMessage(true) << '\n'; + auto code = DB::getCurrentExceptionCode(); + return static_cast(code) ? code : 1; } } diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index 819f16cfd64..bf67db8ff2e 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,8 @@ #include #include #include +#include +#include #include @@ -29,6 +32,13 @@ namespace DB } } +namespace CurrentMetrics +{ + extern const Metric LocalThread; + extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; +} + namespace { @@ -77,11 +87,12 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) ("decompress,d", "decompress") ("offset-in-compressed-file", po::value()->default_value(0ULL), "offset to the compressed block (i.e. physical file offset)") ("offset-in-decompressed-block", po::value()->default_value(0ULL), "offset to the decompressed block (i.e. virtual offset)") - ("block-size,b", po::value()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size") + ("block-size,b", po::value()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size") ("hc", "use LZ4HC instead of LZ4") ("zstd", "use ZSTD instead of LZ4") ("codec", po::value>()->multitoken(), "use codecs combination instead of LZ4") ("level", po::value(), "compression level for codecs specified via flags") + ("threads", po::value()->default_value(1), "number of threads for parallel compression") ("none", "use no compression instead of LZ4") ("stat", "print block statistics of compressed data") ("stacktrace", "print stacktrace of exception") @@ -109,7 +120,8 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) bool stat_mode = options.count("stat"); bool use_none = options.count("none"); print_stacktrace = options.count("stacktrace"); - unsigned block_size = options["block-size"].as(); + size_t block_size = options["block-size"].as(); + size_t num_threads = options["threads"].as(); std::vector codecs; if (options.count("codec")) codecs = options["codec"].as>(); @@ -117,6 +129,12 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) if ((use_lz4hc || use_zstd || use_none) && !codecs.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong options, codec flags like --zstd and --codec options are mutually exclusive"); + if (num_threads < 1) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid value of `threads` parameter"); + + if (num_threads > 1 && decompress) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Parallel mode is only implemented for compression (not for decompression)"); + if (!codecs.empty() && options.count("level")) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong options, --level is not compatible with --codec list"); @@ -145,7 +163,6 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) else codec = CompressionCodecFactory::instance().get(method_family, level); - std::unique_ptr rb; std::unique_ptr wb; @@ -186,9 +203,20 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) else { /// Compression - CompressedWriteBuffer to(*wb, codec, block_size); - copyData(*rb, to); - to.finalize(); + + if (num_threads == 1) + { + CompressedWriteBuffer to(*wb, codec, block_size); + copyData(*rb, to); + to.finalize(); + } + else + { + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_threads); + ParallelCompressedWriteBuffer to(*wb, codec, block_size, num_threads, pool); + copyData(*rb, to); + to.finalize(); + } } } catch (...) diff --git a/programs/disks/DisksApp.cpp b/programs/disks/DisksApp.cpp index 610d8eaa638..d6541e99288 100644 --- a/programs/disks/DisksApp.cpp +++ b/programs/disks/DisksApp.cpp @@ -546,16 +546,18 @@ int mainEntryClickHouseDisks(int argc, char ** argv) catch (const DB::Exception & e) { std::cerr << DB::getExceptionMessage(e, false) << std::endl; - return 0; + auto code = DB::getCurrentExceptionCode(); + return static_cast(code) ? code : 1; } catch (const boost::program_options::error & e) { std::cerr << "Bad arguments: " << e.what() << std::endl; - return 0; + return DB::ErrorCodes::BAD_ARGUMENTS; } catch (...) { std::cerr << DB::getCurrentExceptionMessage(true) << std::endl; - return 0; + auto code = DB::getCurrentExceptionCode(); + return static_cast(code) ? code : 1; } } diff --git a/programs/keeper-client/KeeperClient.cpp b/programs/keeper-client/KeeperClient.cpp index 2a426fad7ac..4bdddaec59c 100644 --- a/programs/keeper-client/KeeperClient.cpp +++ b/programs/keeper-client/KeeperClient.cpp @@ -448,7 +448,8 @@ int mainEntryClickHouseKeeperClient(int argc, char ** argv) catch (const DB::Exception & e) { std::cerr << DB::getExceptionMessage(e, false) << std::endl; - return 1; + auto code = DB::getCurrentExceptionCode(); + return static_cast(code) ? code : 1; } catch (const boost::program_options::error & e) { @@ -458,6 +459,7 @@ int mainEntryClickHouseKeeperClient(int argc, char ** argv) catch (...) { std::cerr << DB::getCurrentExceptionMessage(true) << std::endl; - return 1; + auto code = DB::getCurrentExceptionCode(); + return static_cast(code) ? code : 1; } } diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 74af9950e13..936ce15f4c9 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -81,7 +81,7 @@ int mainEntryClickHouseKeeper(int argc, char ** argv) { std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; auto code = DB::getCurrentExceptionCode(); - return code ? code : 1; + return static_cast(code) ? code : 1; } } @@ -672,7 +672,7 @@ catch (...) /// Poco does not provide stacktrace. tryLogCurrentException("Application"); auto code = getCurrentExceptionCode(); - return code ? code : -1; + return static_cast(code) ? code : -1; } diff --git a/programs/library-bridge/LibraryBridge.cpp b/programs/library-bridge/LibraryBridge.cpp index 261484ac744..62dbd12aaf0 100644 --- a/programs/library-bridge/LibraryBridge.cpp +++ b/programs/library-bridge/LibraryBridge.cpp @@ -13,7 +13,7 @@ int mainEntryClickHouseLibraryBridge(int argc, char ** argv) { std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; auto code = DB::getCurrentExceptionCode(); - return code ? code : 1; + return static_cast(code) ? code : 1; } } diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 1dcef5eb25e..e6f8ecef097 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -22,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -50,7 +49,6 @@ #include #include #include -#include #include #include #include @@ -71,9 +69,11 @@ namespace CurrentMetrics namespace DB { + namespace Setting { extern const SettingsBool allow_introspection_functions; + extern const SettingsBool implicit_select; extern const SettingsLocalFSReadMethod storage_file_read_method; } @@ -126,6 +126,7 @@ void applySettingsOverridesForLocal(ContextMutablePtr context) settings[Setting::allow_introspection_functions] = true; settings[Setting::storage_file_read_method] = LocalFSReadMethod::mmap; + settings[Setting::implicit_select] = true; context->setSettings(settings); } @@ -257,12 +258,12 @@ static DatabasePtr createMemoryDatabaseIfNotExists(ContextPtr context, const Str return system_database; } -static DatabasePtr createClickHouseLocalDatabaseOverlay(const String & name_, ContextPtr context_) +static DatabasePtr createClickHouseLocalDatabaseOverlay(const String & name_, ContextPtr context) { - auto databaseCombiner = std::make_shared(name_, context_); - databaseCombiner->registerNextDatabase(std::make_shared(name_, "", context_)); - databaseCombiner->registerNextDatabase(std::make_shared(name_, context_)); - return databaseCombiner; + auto overlay = std::make_shared(name_, context); + overlay->registerNextDatabase(std::make_shared(name_, fs::weakly_canonical(context->getPath()), UUIDHelpers::generateV4(), context)); + overlay->registerNextDatabase(std::make_shared(name_, "", context)); + return overlay; } /// If path is specified and not empty, will try to setup server environment and load existing metadata @@ -615,12 +616,14 @@ catch (const DB::Exception & e) { bool need_print_stack_trace = getClientConfiguration().getBool("stacktrace", false); std::cerr << getExceptionMessage(e, need_print_stack_trace, true) << std::endl; - return e.code() ? e.code() : -1; + auto code = DB::getCurrentExceptionCode(); + return static_cast(code) ? code : 1; } catch (...) { - std::cerr << getCurrentExceptionMessage(false) << std::endl; - return getCurrentExceptionCode(); + std::cerr << DB::getCurrentExceptionMessage(true) << '\n'; + auto code = DB::getCurrentExceptionCode(); + return static_cast(code) ? code : 1; } void LocalServer::updateLoggerLevel(const String & logs_level) @@ -809,7 +812,12 @@ void LocalServer::processConfig() DatabaseCatalog::instance().initializeAndLoadTemporaryDatabase(); std::string default_database = server_settings[ServerSetting::default_database]; - DatabaseCatalog::instance().attachDatabase(default_database, createClickHouseLocalDatabaseOverlay(default_database, global_context)); + { + DatabasePtr database = createClickHouseLocalDatabaseOverlay(default_database, global_context); + if (UUID uuid = database->getUUID(); uuid != UUIDHelpers::Nil) + DatabaseCatalog::instance().addUUIDMapping(uuid); + DatabaseCatalog::instance().attachDatabase(default_database, database); + } global_context->setCurrentDatabase(default_database); if (getClientConfiguration().has("path")) @@ -1029,7 +1037,7 @@ int mainEntryClickHouseLocal(int argc, char ** argv) { std::cerr << DB::getExceptionMessage(e, false) << std::endl; auto code = DB::getCurrentExceptionCode(); - return code ? code : 1; + return static_cast(code) ? code : 1; } catch (const boost::program_options::error & e) { @@ -1040,6 +1048,6 @@ int mainEntryClickHouseLocal(int argc, char ** argv) { std::cerr << DB::getCurrentExceptionMessage(true) << '\n'; auto code = DB::getCurrentExceptionCode(); - return code ? code : 1; + return static_cast(code) ? code : 1; } } diff --git a/programs/main.cpp b/programs/main.cpp index 02ea1471108..d15c20867d1 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -1,27 +1,22 @@ -#include -#include +#include +#include +#include +#include -#include -#include -#include -#include -#include -#include /// pair - -#include +#if defined(SANITIZE_COVERAGE) +# include +#endif #include "config.h" #include "config_tools.h" -#include -#include -#include -#include -#include - -#include -#include - +#include +#include +#include +#include +#include +#include /// pair +#include /// Universal executable for various clickhouse applications int mainEntryClickHouseServer(int argc, char ** argv); @@ -238,9 +233,12 @@ int main(int argc_, char ** argv_) /// clickhouse # spawn local /// clickhouse local # spawn local /// clickhouse "select ..." # spawn local + /// clickhouse /tmp/repro --enable-analyzer /// - if (main_func == printHelp && !argv.empty() && (argv.size() == 1 || argv[1][0] == '-' - || std::string_view(argv[1]).contains(' '))) + std::error_code ec; + if (main_func == printHelp && !argv.empty() + && (argv.size() == 1 || argv[1][0] == '-' || std::string_view(argv[1]).contains(' ') + || std::filesystem::is_regular_file(std::filesystem::path{argv[1]}, ec))) { main_func = mainEntryClickHouseLocal; } diff --git a/programs/obfuscator/Obfuscator.cpp b/programs/obfuscator/Obfuscator.cpp index 324a4573b24..6bd3865b591 100644 --- a/programs/obfuscator/Obfuscator.cpp +++ b/programs/obfuscator/Obfuscator.cpp @@ -1480,5 +1480,5 @@ catch (...) { std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; auto code = DB::getCurrentExceptionCode(); - return code ? code : 1; + return static_cast(code) ? code : 1; } diff --git a/programs/odbc-bridge/ODBCBridge.cpp b/programs/odbc-bridge/ODBCBridge.cpp index 096d1b2dcca..e5ae3272d40 100644 --- a/programs/odbc-bridge/ODBCBridge.cpp +++ b/programs/odbc-bridge/ODBCBridge.cpp @@ -13,7 +13,7 @@ int mainEntryClickHouseODBCBridge(int argc, char ** argv) { std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; auto code = DB::getCurrentExceptionCode(); - return code ? code : 1; + return static_cast(code) ? code : 1; } } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 1f481381b2b..68f262079ff 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -343,7 +343,7 @@ int mainEntryClickHouseServer(int argc, char ** argv) { std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; auto code = DB::getCurrentExceptionCode(); - return code ? code : 1; + return static_cast(code) ? code : 1; } } @@ -1353,9 +1353,11 @@ try } FailPointInjection::enableFromGlobalConfig(config()); +#endif memory_worker.start(); +#if defined(OS_LINUX) int default_oom_score = 0; #if !defined(NDEBUG) @@ -2535,7 +2537,7 @@ catch (...) /// Poco does not provide stacktrace. tryLogCurrentException("Application"); auto code = getCurrentExceptionCode(); - return code ? code : -1; + return static_cast(code) ? code : -1; } std::unique_ptr Server::buildProtocolStackFromConfig( diff --git a/programs/su/su.cpp b/programs/su/su.cpp index 33d929898f4..40242d0687f 100644 --- a/programs/su/su.cpp +++ b/programs/su/su.cpp @@ -59,7 +59,13 @@ void setUserAndGroup(std::string arg_uid, std::string arg_gid) throw ErrnoException(ErrorCodes::SYSTEM_ERROR, "Cannot do 'getgrnam_r' to obtain gid from group name ({})", arg_gid); if (!result) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Group {} is not found in the system", arg_gid); + { + if (0 != getgrgid_r(gid, &entry, buf.get(), buf_size, &result)) + throw ErrnoException(ErrorCodes::SYSTEM_ERROR, "Cannot do 'getgrnam_r' to obtain gid from group name ({})", arg_gid); + + if (!result) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Group {} is not found in the system", arg_gid); + } gid = entry.gr_gid; } @@ -84,7 +90,13 @@ void setUserAndGroup(std::string arg_uid, std::string arg_gid) throw ErrnoException(ErrorCodes::SYSTEM_ERROR, "Cannot do 'getpwnam_r' to obtain uid from user name ({})", arg_uid); if (!result) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "User {} is not found in the system", arg_uid); + { + if (0 != getpwuid_r(uid, &entry, buf.get(), buf_size, &result)) + throw ErrnoException(ErrorCodes::SYSTEM_ERROR, "Cannot do 'getpwuid_r' to obtain uid from user name ({})", uid); + + if (!result) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "User {} is not found in the system", arg_uid); + } uid = entry.pw_uid; } diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index e8ee363be1a..9b3b8d2a977 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -608,7 +608,7 @@ AuthResult AccessControl::authenticate(const Credentials & credentials, const Po } catch (...) { - tryLogCurrentException(getLogger(), "from: " + address.toString() + ", user: " + credentials.getUserName() + ": Authentication failed"); + tryLogCurrentException(getLogger(), "from: " + address.toString() + ", user: " + credentials.getUserName() + ": Authentication failed", LogsLevel::information); WriteBufferFromOwnString message; message << credentials.getUserName() << ": Authentication failed: password is incorrect, or there is no user with such name."; @@ -622,8 +622,9 @@ AuthResult AccessControl::authenticate(const Credentials & credentials, const Po << "and deleting this file will reset the password.\n" << "See also /etc/clickhouse-server/users.xml on the server where ClickHouse is installed.\n\n"; - /// We use the same message for all authentication failures because we don't want to give away any unnecessary information for security reasons, - /// only the log will show the exact reason. + /// We use the same message for all authentication failures because we don't want to give away any unnecessary information for security reasons. + /// Only the log ((*), above) will show the exact reason. Note that (*) logs at information level instead of the default error level as + /// authentication failures are not an unusual event. throw Exception(PreformattedMessage{message.str(), "{}: Authentication failed: password is incorrect, or there is no user with such name", std::vector{credentials.getUserName()}}, diff --git a/src/Access/Credentials.h b/src/Access/Credentials.h index f220b8d2c48..b21b7e6921f 100644 --- a/src/Access/Credentials.h +++ b/src/Access/Credentials.h @@ -15,6 +15,9 @@ public: explicit Credentials() = default; explicit Credentials(const String & user_name_); + Credentials(const Credentials &) = default; + Credentials(Credentials &&) = default; + virtual ~Credentials() = default; const String & getUserName() const; diff --git a/src/AggregateFunctions/AggregateFunctionQuantileExactWeighted.cpp b/src/AggregateFunctions/AggregateFunctionQuantileExactWeighted.cpp index 58b3b75b056..116b04bf4ba 100644 --- a/src/AggregateFunctions/AggregateFunctionQuantileExactWeighted.cpp +++ b/src/AggregateFunctions/AggregateFunctionQuantileExactWeighted.cpp @@ -387,7 +387,7 @@ template using FuncQuantileExactWeighted = AggregateFunctionQuantile< Value, QuantileExactWeighted, - NameQuantileExactWeighted, + std::conditional_t, true, std::conditional_t, false, @@ -396,7 +396,7 @@ template using FuncQuantilesExactWeighted = AggregateFunctionQuantile< Value, QuantileExactWeighted, - NameQuantilesExactWeighted, + std::conditional_t, true, std::conditional_t, true, diff --git a/src/AggregateFunctions/fuzzers/CMakeLists.txt b/src/AggregateFunctions/fuzzers/CMakeLists.txt index 6a7be0d4377..f01bcb0b631 100644 --- a/src/AggregateFunctions/fuzzers/CMakeLists.txt +++ b/src/AggregateFunctions/fuzzers/CMakeLists.txt @@ -1,2 +1,2 @@ clickhouse_add_executable(aggregate_function_state_deserialization_fuzzer aggregate_function_state_deserialization_fuzzer.cpp ${SRCS}) -target_link_libraries(aggregate_function_state_deserialization_fuzzer PRIVATE clickhouse_aggregate_functions) +target_link_libraries(aggregate_function_state_deserialization_fuzzer PRIVATE clickhouse_aggregate_functions dbms) diff --git a/src/Analyzer/FunctionNode.cpp b/src/Analyzer/FunctionNode.cpp index 1ad7e7996c7..27b46a112e2 100644 --- a/src/Analyzer/FunctionNode.cpp +++ b/src/Analyzer/FunctionNode.cpp @@ -88,6 +88,7 @@ void FunctionNode::resolveAsFunction(FunctionBasePtr function_value) function_name = function_value->getName(); function = std::move(function_value); kind = FunctionKind::ORDINARY; + nulls_action = NullsAction::EMPTY; } void FunctionNode::resolveAsAggregateFunction(AggregateFunctionPtr aggregate_function_value) @@ -95,6 +96,12 @@ void FunctionNode::resolveAsAggregateFunction(AggregateFunctionPtr aggregate_fun function_name = aggregate_function_value->getName(); function = std::move(aggregate_function_value); kind = FunctionKind::AGGREGATE; + /** When the function is resolved, we do not need the nulls action anymore. + * The only thing that the nulls action does is map from one function to another. + * Thus, the nulls action is encoded in the function name and does not make sense anymore. + * Keeping the nulls action may lead to incorrect comparison of functions, e.g., count() and count() IGNORE NULLS are the same function. + */ + nulls_action = NullsAction::EMPTY; } void FunctionNode::resolveAsWindowFunction(AggregateFunctionPtr window_function_value) diff --git a/src/Analyzer/JoinNode.cpp b/src/Analyzer/JoinNode.cpp index bf99c014826..722c1e19b7e 100644 --- a/src/Analyzer/JoinNode.cpp +++ b/src/Analyzer/JoinNode.cpp @@ -48,9 +48,15 @@ ASTPtr JoinNode::toASTTableJoin() const auto join_expression_ast = children[join_expression_child_index]->toAST(); if (is_using_join_expression) - join_ast->using_expression_list = std::move(join_expression_ast); + { + join_ast->using_expression_list = join_expression_ast; + join_ast->children.push_back(join_ast->using_expression_list); + } else - join_ast->on_expression = std::move(join_expression_ast); + { + join_ast->on_expression = join_expression_ast; + join_ast->children.push_back(join_ast->on_expression); + } } return join_ast; diff --git a/src/Analyzer/Passes/FuseFunctionsPass.cpp b/src/Analyzer/Passes/FuseFunctionsPass.cpp index 17a765a068b..cd2577c3d76 100644 --- a/src/Analyzer/Passes/FuseFunctionsPass.cpp +++ b/src/Analyzer/Passes/FuseFunctionsPass.cpp @@ -85,10 +85,9 @@ QueryTreeNodePtr createResolvedFunction(const ContextPtr & context, const String } FunctionNodePtr createResolvedAggregateFunction( - const String & name, const QueryTreeNodePtr & argument, const Array & parameters = {}, NullsAction action = NullsAction::EMPTY) + const String & name, const QueryTreeNodePtr & argument, const Array & parameters = {}) { auto function_node = std::make_shared(name); - function_node->setNullsAction(action); if (!parameters.empty()) { @@ -100,7 +99,7 @@ FunctionNodePtr createResolvedAggregateFunction( function_node->getArguments().getNodes() = { argument }; AggregateFunctionProperties properties; - auto aggregate_function = AggregateFunctionFactory::instance().get(name, action, {argument->getResultType()}, parameters, properties); + auto aggregate_function = AggregateFunctionFactory::instance().get(name, NullsAction::EMPTY, {argument->getResultType()}, parameters, properties); function_node->resolveAsAggregateFunction(std::move(aggregate_function)); return function_node; diff --git a/src/Analyzer/QueryNode.h b/src/Analyzer/QueryNode.h index aef0c8805bb..2333fc56218 100644 --- a/src/Analyzer/QueryNode.h +++ b/src/Analyzer/QueryNode.h @@ -602,9 +602,21 @@ public: return projection_columns; } + /// Returns true if query node is resolved, false otherwise + bool isResolved() const + { + return !projection_columns.empty(); + } + /// Resolve query node projection columns void resolveProjectionColumns(NamesAndTypes projection_columns_value); + /// Clear query node projection columns + void clearProjectionColumns() + { + projection_columns.clear(); + } + /// Remove unused projection columns void removeUnusedProjectionColumns(const std::unordered_set & used_projection_columns); diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 39c59d27e2c..d3c88d39213 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -498,6 +498,8 @@ QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_express sort_node->getFillTo() = buildExpression(order_by_element.getFillTo(), context); if (order_by_element.getFillStep()) sort_node->getFillStep() = buildExpression(order_by_element.getFillStep(), context); + if (order_by_element.getFillStaleness()) + sort_node->getFillStaleness() = buildExpression(order_by_element.getFillStaleness(), context); list_node->getNodes().push_back(std::move(sort_node)); } diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index 4443f83596f..0005aa6c8bc 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -3,7 +3,6 @@ #include #include -#include "Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.h" #include #include @@ -16,39 +15,39 @@ #include #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include -#include -#include -#include - +#include +#include +#include +#include namespace DB { diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index cb3087af707..03ebd893c47 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -51,7 +52,6 @@ #include #include #include -#include #include #include #include @@ -103,6 +103,8 @@ namespace Setting extern const SettingsBool single_join_prefer_left_table; extern const SettingsBool transform_null_in; extern const SettingsUInt64 use_structure_from_insertion_table_in_table_functions; + extern const SettingsBool allow_suspicious_types_in_group_by; + extern const SettingsBool allow_suspicious_types_in_order_by; extern const SettingsBool use_concurrency_control; } @@ -437,8 +439,13 @@ ProjectionName QueryAnalyzer::calculateWindowProjectionName(const QueryTreeNodeP return buffer.str(); } -ProjectionName QueryAnalyzer::calculateSortColumnProjectionName(const QueryTreeNodePtr & sort_column_node, const ProjectionName & sort_expression_projection_name, - const ProjectionName & fill_from_expression_projection_name, const ProjectionName & fill_to_expression_projection_name, const ProjectionName & fill_step_expression_projection_name) +ProjectionName QueryAnalyzer::calculateSortColumnProjectionName( + const QueryTreeNodePtr & sort_column_node, + const ProjectionName & sort_expression_projection_name, + const ProjectionName & fill_from_expression_projection_name, + const ProjectionName & fill_to_expression_projection_name, + const ProjectionName & fill_step_expression_projection_name, + const ProjectionName & fill_staleness_expression_projection_name) { auto & sort_node_typed = sort_column_node->as(); @@ -468,6 +475,9 @@ ProjectionName QueryAnalyzer::calculateSortColumnProjectionName(const QueryTreeN if (sort_node_typed.hasFillStep()) sort_column_projection_name_buffer << " STEP " << fill_step_expression_projection_name; + + if (sort_node_typed.hasFillStaleness()) + sort_column_projection_name_buffer << " STALENESS " << fill_staleness_expression_projection_name; } return sort_column_projection_name_buffer.str(); @@ -667,6 +677,8 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden "tuple"}); } } + + logProcessorProfile(context, io.pipeline.getProcessors()); } scalars_cache.emplace(node_with_hash, scalar_block); @@ -2958,27 +2970,29 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi /// Replace storage with values storage of insertion block if (StoragePtr storage = scope.context->getViewSource()) { - QueryTreeNodePtr table_expression; - /// Process possibly nested sub-selects - for (auto * query_node = in_second_argument->as(); query_node; query_node = table_expression->as()) - table_expression = extractLeftTableExpression(query_node->getJoinTree()); + QueryTreeNodePtr table_expression = in_second_argument; - if (table_expression) + /// Process possibly nested sub-selects + while (table_expression) { - if (auto * query_table_node = table_expression->as()) - { - if (query_table_node->getStorageID().getFullNameNotQuoted() == storage->getStorageID().getFullNameNotQuoted()) - { - auto replacement_table_expression = std::make_shared(storage, scope.context); - if (std::optional table_expression_modifiers = query_table_node->getTableExpressionModifiers()) - replacement_table_expression->setTableExpressionModifiers(*table_expression_modifiers); - in_second_argument = in_second_argument->cloneAndReplace(table_expression, std::move(replacement_table_expression)); - } - } + if (auto * query_node = table_expression->as()) + table_expression = extractLeftTableExpression(query_node->getJoinTree()); + else if (auto * union_node = table_expression->as()) + table_expression = union_node->getQueries().getNodes().at(0); + else + break; + } + + TableNode * table_expression_table_node = table_expression ? table_expression->as() : nullptr; + + if (table_expression_table_node && + table_expression_table_node->getStorageID().getFullNameNotQuoted() == storage->getStorageID().getFullNameNotQuoted()) + { + auto replacement_table_expression_table_node = table_expression_table_node->clone(); + replacement_table_expression_table_node->as().updateStorage(storage, scope.context); + in_second_argument = in_second_argument->cloneAndReplace(table_expression, std::move(replacement_table_expression_table_node)); } } - - resolveExpressionNode(in_second_argument, scope, false /*allow_lambda_expression*/, true /*allow_table_expression*/); } /// Edge case when the first argument of IN is scalar subquery. @@ -3011,9 +3025,10 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi argument_column.name = arguments_projection_names[function_argument_index]; /** If function argument is lambda, save lambda argument index and initialize argument type as DataTypeFunction - * where function argument types are initialized with empty array of lambda arguments size. + * where function argument types are initialized with empty arrays of lambda arguments size. */ - if (const auto * lambda_node = function_argument->as()) + const auto * lambda_node = function_argument->as(); + if (lambda_node) { size_t lambda_arguments_size = lambda_node->getArguments().getNodes().size(); argument_column.type = std::make_shared(DataTypes(lambda_arguments_size, nullptr), nullptr); @@ -3485,15 +3500,11 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi else function_base = function->build(argument_columns); - /// Do not constant fold get scalar functions - // bool disable_constant_folding = function_name == "__getScalar" || function_name == "shardNum" || - // function_name == "shardCount" || function_name == "hostName" || function_name == "tcpPort"; - /** If function is suitable for constant folding try to convert it to constant. * Example: SELECT plus(1, 1); * Result: SELECT 2; */ - if (function_base->isSuitableForConstantFolding()) // && !disable_constant_folding) + if (function_base->isSuitableForConstantFolding()) { auto result_type = function_base->getResultType(); auto executable_function = function_base->prepare(argument_columns); @@ -3502,7 +3513,9 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi if (all_arguments_constants) { - size_t num_rows = function_arguments.empty() ? 0 : argument_columns.front().column->size(); + size_t num_rows = 0; + if (!argument_columns.empty()) + num_rows = argument_columns.front().column->size(); column = executable_function->execute(argument_columns, result_type, num_rows, true); } else @@ -3998,6 +4011,7 @@ ProjectionNames QueryAnalyzer::resolveSortNodeList(QueryTreeNodePtr & sort_node_ ProjectionNames fill_from_expression_projection_names; ProjectionNames fill_to_expression_projection_names; ProjectionNames fill_step_expression_projection_names; + ProjectionNames fill_staleness_expression_projection_names; auto & sort_node_list_typed = sort_node_list->as(); for (auto & node : sort_node_list_typed.getNodes()) @@ -4019,6 +4033,8 @@ ProjectionNames QueryAnalyzer::resolveSortNodeList(QueryTreeNodePtr & sort_node_ sort_node.getExpression() = sort_column_list_node->getNodes().front(); } + validateSortingKeyType(sort_node.getExpression()->getResultType(), scope); + size_t sort_expression_projection_names_size = sort_expression_projection_names.size(); if (sort_expression_projection_names_size != 1) throw Exception(ErrorCodes::LOGICAL_ERROR, @@ -4088,11 +4104,38 @@ ProjectionNames QueryAnalyzer::resolveSortNodeList(QueryTreeNodePtr & sort_node_ fill_step_expression_projection_names_size); } + if (sort_node.hasFillStaleness()) + { + fill_staleness_expression_projection_names = resolveExpressionNode(sort_node.getFillStaleness(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + + const auto * constant_node = sort_node.getFillStaleness()->as(); + if (!constant_node) + throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, + "Sort FILL STALENESS expression must be constant with numeric or interval type. Actual {}. In scope {}", + sort_node.getFillStaleness()->formatASTForErrorMessage(), + scope.scope_node->formatASTForErrorMessage()); + + bool is_number = isColumnedAsNumber(constant_node->getResultType()); + bool is_interval = WhichDataType(constant_node->getResultType()).isInterval(); + if (!is_number && !is_interval) + throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, + "Sort FILL STALENESS expression must be constant with numeric or interval type. Actual {}. In scope {}", + sort_node.getFillStaleness()->formatASTForErrorMessage(), + scope.scope_node->formatASTForErrorMessage()); + + size_t fill_staleness_expression_projection_names_size = fill_staleness_expression_projection_names.size(); + if (fill_staleness_expression_projection_names_size != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Sort FILL STALENESS expression expected 1 projection name. Actual {}", + fill_staleness_expression_projection_names_size); + } + auto sort_column_projection_name = calculateSortColumnProjectionName(node, sort_expression_projection_names[0], fill_from_expression_projection_names.empty() ? "" : fill_from_expression_projection_names.front(), fill_to_expression_projection_names.empty() ? "" : fill_to_expression_projection_names.front(), - fill_step_expression_projection_names.empty() ? "" : fill_step_expression_projection_names.front()); + fill_step_expression_projection_names.empty() ? "" : fill_step_expression_projection_names.front(), + fill_staleness_expression_projection_names.empty() ? "" : fill_staleness_expression_projection_names.front()); result_projection_names.push_back(std::move(sort_column_projection_name)); @@ -4100,11 +4143,32 @@ ProjectionNames QueryAnalyzer::resolveSortNodeList(QueryTreeNodePtr & sort_node_ fill_from_expression_projection_names.clear(); fill_to_expression_projection_names.clear(); fill_step_expression_projection_names.clear(); + fill_staleness_expression_projection_names.clear(); } return result_projection_names; } +void QueryAnalyzer::validateSortingKeyType(const DataTypePtr & sorting_key_type, const IdentifierResolveScope & scope) const +{ + if (scope.context->getSettingsRef()[Setting::allow_suspicious_types_in_order_by]) + return; + + auto check = [](const IDataType & type) + { + if (isDynamic(type) || isVariant(type)) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Data types Variant/Dynamic are not allowed in ORDER BY keys, because it can lead to unexpected results. " + "Consider using a subcolumn with a specific data type instead (for example 'column.Int64' or 'json.some.path.:Int64' if " + "its a JSON path subcolumn) or casting this column to a specific data type. " + "Set setting allow_suspicious_types_in_order_by = 1 in order to allow it"); + }; + + check(*sorting_key_type); + sorting_key_type->forEachChild(check); +} + namespace { @@ -4144,11 +4208,12 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR expandTuplesInList(group_by_list); } - if (scope.group_by_use_nulls) + for (const auto & grouping_set : query_node_typed.getGroupBy().getNodes()) { - for (const auto & grouping_set : query_node_typed.getGroupBy().getNodes()) + for (const auto & group_by_elem : grouping_set->as()->getNodes()) { - for (const auto & group_by_elem : grouping_set->as()->getNodes()) + validateGroupByKeyType(group_by_elem->getResultType(), scope); + if (scope.group_by_use_nulls) scope.nullable_group_by_keys.insert(group_by_elem); } } @@ -4164,14 +4229,37 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR auto & group_by_list = query_node_typed.getGroupBy().getNodes(); expandTuplesInList(group_by_list); - if (scope.group_by_use_nulls) + for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) { - for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) + validateGroupByKeyType(group_by_elem->getResultType(), scope); + if (scope.group_by_use_nulls) scope.nullable_group_by_keys.insert(group_by_elem); } } } +/** Validate data types of GROUP BY key. + */ +void QueryAnalyzer::validateGroupByKeyType(const DataTypePtr & group_by_key_type, const IdentifierResolveScope & scope) const +{ + if (scope.context->getSettingsRef()[Setting::allow_suspicious_types_in_group_by]) + return; + + auto check = [](const IDataType & type) + { + if (isDynamic(type) || isVariant(type)) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Data types Variant/Dynamic are not allowed in GROUP BY keys, because it can lead to unexpected results. " + "Consider using a subcolumn with a specific data type instead (for example 'column.Int64' or 'json.some.path.:Int64' if " + "its a JSON path subcolumn) or casting this column to a specific data type. " + "Set setting allow_suspicious_types_in_group_by = 1 in order to allow it"); + }; + + check(*group_by_key_type); + group_by_key_type->forEachChild(check); +} + /** Resolve interpolate columns nodes list. */ void QueryAnalyzer::resolveInterpolateColumnsNodeList(QueryTreeNodePtr & interpolate_node_list, IdentifierResolveScope & scope) @@ -5310,6 +5398,16 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier auto & query_node_typed = query_node->as(); + /** It is unsafe to call resolveQuery on already resolved query node, because during identifier resolution process + * we replace identifiers with expressions without aliases, also at the end of resolveQuery all aliases from all nodes will be removed. + * For subsequent resolveQuery executions it is possible to have wrong projection header, because for nodes + * with aliases projection name is alias. + * + * If for client it is necessary to resolve query node after clone, client must clear projection columns from query node before resolve. + */ + if (query_node_typed.isResolved()) + return; + if (query_node_typed.isCTE()) ctes_in_resolve_process.insert(query_node_typed.getCTEName()); @@ -5448,16 +5546,13 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier */ scope.use_identifier_lookup_to_result_cache = false; - if (query_node_typed.getJoinTree()) - { - TableExpressionsAliasVisitor table_expressions_visitor(scope); - table_expressions_visitor.visit(query_node_typed.getJoinTree()); + TableExpressionsAliasVisitor table_expressions_visitor(scope); + table_expressions_visitor.visit(query_node_typed.getJoinTree()); - initializeQueryJoinTreeNode(query_node_typed.getJoinTree(), scope); - scope.aliases.alias_name_to_table_expression_node.clear(); + initializeQueryJoinTreeNode(query_node_typed.getJoinTree(), scope); + scope.aliases.alias_name_to_table_expression_node.clear(); - resolveQueryJoinTreeNode(query_node_typed.getJoinTree(), scope, visitor); - } + resolveQueryJoinTreeNode(query_node_typed.getJoinTree(), scope, visitor); if (!scope.group_by_use_nulls) scope.use_identifier_lookup_to_result_cache = true; @@ -5675,6 +5770,9 @@ void QueryAnalyzer::resolveUnion(const QueryTreeNodePtr & union_node, Identifier { auto & union_node_typed = union_node->as(); + if (union_node_typed.isResolved()) + return; + if (union_node_typed.isCTE()) ctes_in_resolve_process.insert(union_node_typed.getCTEName()); diff --git a/src/Analyzer/Resolve/QueryAnalyzer.h b/src/Analyzer/Resolve/QueryAnalyzer.h index 0d4309843e6..ae6cf05bcdc 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.h +++ b/src/Analyzer/Resolve/QueryAnalyzer.h @@ -140,7 +140,8 @@ private: const ProjectionName & sort_expression_projection_name, const ProjectionName & fill_from_expression_projection_name, const ProjectionName & fill_to_expression_projection_name, - const ProjectionName & fill_step_expression_projection_name); + const ProjectionName & fill_step_expression_projection_name, + const ProjectionName & fill_staleness_expression_projection_name); QueryTreeNodePtr tryGetLambdaFromSQLUserDefinedFunctions(const std::string & function_name, ContextPtr context); @@ -219,8 +220,12 @@ private: ProjectionNames resolveSortNodeList(QueryTreeNodePtr & sort_node_list, IdentifierResolveScope & scope); + void validateSortingKeyType(const DataTypePtr & sorting_key_type, const IdentifierResolveScope & scope) const; + void resolveGroupByNode(QueryNode & query_node_typed, IdentifierResolveScope & scope); + void validateGroupByKeyType(const DataTypePtr & group_by_key_type, const IdentifierResolveScope & scope) const; + void resolveInterpolateColumnsNodeList(QueryTreeNodePtr & interpolate_node_list, IdentifierResolveScope & scope); void resolveWindowNodeList(QueryTreeNodePtr & window_node_list, IdentifierResolveScope & scope); diff --git a/src/Analyzer/SortNode.cpp b/src/Analyzer/SortNode.cpp index e891046626a..42c010e4784 100644 --- a/src/Analyzer/SortNode.cpp +++ b/src/Analyzer/SortNode.cpp @@ -69,6 +69,12 @@ void SortNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, si buffer << '\n' << std::string(indent + 2, ' ') << "FILL STEP\n"; getFillStep()->dumpTreeImpl(buffer, format_state, indent + 4); } + + if (hasFillStaleness()) + { + buffer << '\n' << std::string(indent + 2, ' ') << "FILL STALENESS\n"; + getFillStaleness()->dumpTreeImpl(buffer, format_state, indent + 4); + } } bool SortNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const @@ -132,6 +138,8 @@ ASTPtr SortNode::toASTImpl(const ConvertToASTOptions & options) const result->setFillTo(getFillTo()->toAST(options)); if (hasFillStep()) result->setFillStep(getFillStep()->toAST(options)); + if (hasFillStaleness()) + result->setFillStaleness(getFillStaleness()->toAST(options)); return result; } diff --git a/src/Analyzer/SortNode.h b/src/Analyzer/SortNode.h index 0ebdde61912..6f0010abdaa 100644 --- a/src/Analyzer/SortNode.h +++ b/src/Analyzer/SortNode.h @@ -105,6 +105,24 @@ public: return children[fill_step_child_index]; } + /// Returns true if sort node has fill staleness, false otherwise + bool hasFillStaleness() const + { + return children[fill_staleness_child_index] != nullptr; + } + + /// Get fill staleness + const QueryTreeNodePtr & getFillStaleness() const + { + return children[fill_staleness_child_index]; + } + + /// Get fill staleness + QueryTreeNodePtr & getFillStaleness() + { + return children[fill_staleness_child_index]; + } + /// Get collator const std::shared_ptr & getCollator() const { @@ -144,7 +162,8 @@ private: static constexpr size_t fill_from_child_index = 1; static constexpr size_t fill_to_child_index = 2; static constexpr size_t fill_step_child_index = 3; - static constexpr size_t children_size = fill_step_child_index + 1; + static constexpr size_t fill_staleness_child_index = 4; + static constexpr size_t children_size = fill_staleness_child_index + 1; SortDirection sort_direction = SortDirection::ASCENDING; std::optional nulls_sort_direction; diff --git a/src/Analyzer/UnionNode.cpp b/src/Analyzer/UnionNode.cpp index 6f70f01e519..545a6b2195b 100644 --- a/src/Analyzer/UnionNode.cpp +++ b/src/Analyzer/UnionNode.cpp @@ -35,6 +35,7 @@ namespace ErrorCodes { extern const int TYPE_MISMATCH; extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; } UnionNode::UnionNode(ContextMutablePtr context_, SelectUnionMode union_mode_) @@ -50,6 +51,26 @@ UnionNode::UnionNode(ContextMutablePtr context_, SelectUnionMode union_mode_) children[queries_child_index] = std::make_shared(); } +bool UnionNode::isResolved() const +{ + for (const auto & query_node : getQueries().getNodes()) + { + bool is_resolved = false; + + if (auto * query_node_typed = query_node->as()) + is_resolved = query_node_typed->isResolved(); + else if (auto * union_node_typed = query_node->as()) + is_resolved = union_node_typed->isResolved(); + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected query tree node type in UNION node"); + + if (!is_resolved) + return false; + } + + return true; +} + NamesAndTypes UnionNode::computeProjectionColumns() const { if (recursive_cte_table) diff --git a/src/Analyzer/UnionNode.h b/src/Analyzer/UnionNode.h index 40baad1ad57..85d6afb1e47 100644 --- a/src/Analyzer/UnionNode.h +++ b/src/Analyzer/UnionNode.h @@ -163,6 +163,9 @@ public: return children[queries_child_index]; } + /// Returns true if union node is resolved, false otherwise + bool isResolved() const; + /// Compute union node projection columns NamesAndTypes computeProjectionColumns() const; diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 0a824753dc0..ce47f503931 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1650,6 +1650,11 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des if (!parsed_insert_query) return; + /// If it's clickhouse-local, and the input data reading is already baked into the query pipeline, + /// don't read the data again here. This happens in some cases (e.g. input() table function) but not others (e.g. INFILE). + if (!connection->isSendDataNeeded()) + return; + bool have_data_in_stdin = !is_interactive && !stdin_is_a_tty && isStdinNotEmptyAndValid(std_in); if (need_render_progress) @@ -2674,7 +2679,10 @@ void ClientBase::runInteractive() #if USE_REPLXX replxx::Replxx::highlighter_callback_t highlight_callback{}; if (getClientConfiguration().getBool("highlight", true)) - highlight_callback = highlight; + highlight_callback = [this](const String & query, std::vector & colors) + { + highlight(query, colors, *client_context); + }; ReplxxLineReader lr( *suggest, diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index 156c0c87fb6..e8e009ec306 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include @@ -12,6 +14,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool implicit_select; +} + /// Should we celebrate a bit? bool isNewYearMode() { @@ -95,7 +102,7 @@ bool isChineseNewYearMode(const String & local_tz) } #if USE_REPLXX -void highlight(const String & query, std::vector & colors) +void highlight(const String & query, std::vector & colors, const Context & context) { using namespace replxx; @@ -135,13 +142,27 @@ void highlight(const String & query, std::vector & colors /// Currently we highlight only the first query in the multi-query mode. - ParserQuery parser(end); + ParserQuery parser(end, false, context.getSettingsRef()[Setting::implicit_select]); ASTPtr ast; bool parse_res = false; try { - parse_res = parser.parse(token_iterator, ast, expected); + while (!token_iterator->isEnd()) + { + parse_res = parser.parse(token_iterator, ast, expected); + if (!parse_res) + break; + + if (!token_iterator->isEnd() && token_iterator->type != TokenType::Semicolon) + { + parse_res = false; + break; + } + + while (token_iterator->type == TokenType::Semicolon) + ++token_iterator; + } } catch (...) { @@ -175,7 +196,7 @@ void highlight(const String & query, std::vector & colors /// Highlight the last error in red. If the parser failed or the lexer found an invalid token, /// or if it didn't parse all the data (except, the data for INSERT query, which is legitimately unparsed) - if ((!parse_res || last_token.isError() || (!token_iterator->isEnd() && token_iterator->type != TokenType::Semicolon)) + if ((!parse_res || last_token.isError()) && !(insert_data && expected.max_parsed_pos >= insert_data) && expected.max_parsed_pos >= prev) { diff --git a/src/Client/ClientBaseHelpers.h b/src/Client/ClientBaseHelpers.h index adc1c81b3c5..dcfac21c500 100644 --- a/src/Client/ClientBaseHelpers.h +++ b/src/Client/ClientBaseHelpers.h @@ -11,13 +11,15 @@ namespace DB { +class Context; + /// Should we celebrate a bit? bool isNewYearMode(); bool isChineseNewYearMode(const String & local_tz); #if USE_REPLXX -void highlight(const String & query, std::vector & colors); +void highlight(const String & query, std::vector & colors, const Context & context); #endif } diff --git a/src/Client/ClientBaseOptimizedParts.cpp b/src/Client/ClientBaseOptimizedParts.cpp index ac4d3417779..6eaa3708df6 100644 --- a/src/Client/ClientBaseOptimizedParts.cpp +++ b/src/Client/ClientBaseOptimizedParts.cpp @@ -1,5 +1,7 @@ #include +#include + namespace DB { @@ -108,6 +110,7 @@ void ClientApplicationBase::parseAndCheckOptions(OptionsDescription & options_de { /// Two special cases for better usability: /// - if the option contains a whitespace, it might be a query: clickhouse "SELECT 1" + /// - if the option is a filesystem file, then it's likely a queries file (clickhouse repro.sql) /// These are relevant for interactive usage - user-friendly, but questionable in general. /// In case of ambiguity or for scripts, prefer using proper options. @@ -115,8 +118,11 @@ void ClientApplicationBase::parseAndCheckOptions(OptionsDescription & options_de po::variable_value value(boost::any(op.value), false); const char * option; + std::error_code ec; if (token.contains(' ')) option = "query"; + else if (std::filesystem::is_regular_file(std::filesystem::path{token}, ec)) + option = "queries-file"; else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Positional option `{}` is not supported.", token); diff --git a/src/Client/IServerConnection.h b/src/Client/IServerConnection.h index 6ab4234bca2..fe69be8788a 100644 --- a/src/Client/IServerConnection.h +++ b/src/Client/IServerConnection.h @@ -109,6 +109,10 @@ public: /// Send block of data; if name is specified, server will write it to external (temporary) table of that name. virtual void sendData(const Block & block, const String & name, bool scalar) = 0; + /// Whether the client needs to read and send the data for the INSERT. + /// False if the server will read the data through other means (in particular if clickhouse-local added input reading step directly into the query pipeline). + virtual bool isSendDataNeeded() const { return true; } + /// Send all contents of external (temporary) tables. virtual void sendExternalTablesData(ExternalTablesData & data) = 0; diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index e4915a77c83..4ca209c29c7 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -328,6 +328,11 @@ void LocalConnection::sendData(const Block & block, const String &, bool) sendProfileEvents(); } +bool LocalConnection::isSendDataNeeded() const +{ + return !state || state->input_pipeline == nullptr; +} + void LocalConnection::sendCancel() { state->is_cancelled = true; diff --git a/src/Client/LocalConnection.h b/src/Client/LocalConnection.h index b424c5b5aa3..a70ed6ffa7e 100644 --- a/src/Client/LocalConnection.h +++ b/src/Client/LocalConnection.h @@ -120,6 +120,8 @@ public: void sendData(const Block & block, const String & name/* = "" */, bool scalar/* = false */) override; + bool isSendDataNeeded() const override; + void sendExternalTablesData(ExternalTablesData &) override; void sendMergeTreeReadTaskResponse(const ParallelReadResponse & response) override; diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index ec14b096055..a66f9041213 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -196,6 +196,13 @@ public: bool hasDynamicStructure() const override { return getData().hasDynamicStructure(); } void takeDynamicStructureFromSourceColumns(const Columns & source_columns) override; + bool dynamicStructureEquals(const IColumn & rhs) const override + { + if (const auto * rhs_concrete = typeid_cast(&rhs)) + return data->dynamicStructureEquals(*rhs_concrete->data); + return false; + } + private: WrappedPtr data; WrappedPtr offsets; diff --git a/src/Columns/ColumnDynamic.cpp b/src/Columns/ColumnDynamic.cpp index 41a9096bc0c..6eb22a8bdf7 100644 --- a/src/Columns/ColumnDynamic.cpp +++ b/src/Columns/ColumnDynamic.cpp @@ -1208,6 +1208,15 @@ void ColumnDynamic::prepareVariantsForSquashing(const Columns & source_columns) } } +bool ColumnDynamic::dynamicStructureEquals(const IColumn & rhs) const +{ + if (const auto * rhs_concrete = typeid_cast(&rhs)) + return max_dynamic_types == rhs_concrete->max_dynamic_types && global_max_dynamic_types == rhs_concrete->global_max_dynamic_types + && variant_info.variant_name == rhs_concrete->variant_info.variant_name + && variant_column->dynamicStructureEquals(*rhs_concrete->variant_column); + return false; +} + void ColumnDynamic::takeDynamicStructureFromSourceColumns(const Columns & source_columns) { if (!empty()) diff --git a/src/Columns/ColumnDynamic.h b/src/Columns/ColumnDynamic.h index 57a1545a832..fbab4d5da4c 100644 --- a/src/Columns/ColumnDynamic.h +++ b/src/Columns/ColumnDynamic.h @@ -376,6 +376,7 @@ public: bool addNewVariant(const DataTypePtr & new_variant) { return addNewVariant(new_variant, new_variant->getName()); } bool hasDynamicStructure() const override { return true; } + bool dynamicStructureEquals(const IColumn & rhs) const override; void takeDynamicStructureFromSourceColumns(const Columns & source_columns) override; const StatisticsPtr & getStatistics() const { return statistics; } diff --git a/src/Columns/ColumnFunction.cpp b/src/Columns/ColumnFunction.cpp index 18c343c6ca6..cc80d04444e 100644 --- a/src/Columns/ColumnFunction.cpp +++ b/src/Columns/ColumnFunction.cpp @@ -72,6 +72,26 @@ ColumnPtr ColumnFunction::cut(size_t start, size_t length) const return ColumnFunction::create(length, function, capture, is_short_circuit_argument, is_function_compiled); } +Field ColumnFunction::operator[](size_t n) const +{ + Field res; + get(n, res); + return res; +} + +void ColumnFunction::get(size_t n, Field & res) const +{ + const size_t tuple_size = captured_columns.size(); + + res = Tuple(); + Tuple & res_tuple = res.safeGet(); + res_tuple.reserve(tuple_size); + + for (size_t i = 0; i < tuple_size; ++i) + res_tuple.push_back((*captured_columns[i].column)[n]); +} + + #if !defined(DEBUG_OR_SANITIZER_BUILD) void ColumnFunction::insertFrom(const IColumn & src, size_t n) #else diff --git a/src/Columns/ColumnFunction.h b/src/Columns/ColumnFunction.h index b62c6bf70eb..8df9e23c0e8 100644 --- a/src/Columns/ColumnFunction.h +++ b/src/Columns/ColumnFunction.h @@ -60,15 +60,9 @@ public: void appendArguments(const ColumnsWithTypeAndName & columns); ColumnWithTypeAndName reduce() const; - Field operator[](size_t) const override - { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get value from {}", getName()); - } + Field operator[](size_t n) const override; - void get(size_t, Field &) const override - { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get value from {}", getName()); - } + void get(size_t n, Field & res) const override; StringRef getDataAt(size_t) const override { diff --git a/src/Columns/ColumnMap.cpp b/src/Columns/ColumnMap.cpp index 7ebbed930d8..a5511dfeeb4 100644 --- a/src/Columns/ColumnMap.cpp +++ b/src/Columns/ColumnMap.cpp @@ -345,6 +345,13 @@ bool ColumnMap::structureEquals(const IColumn & rhs) const return false; } +bool ColumnMap::dynamicStructureEquals(const IColumn & rhs) const +{ + if (const auto * rhs_map = typeid_cast(&rhs)) + return nested->dynamicStructureEquals(*rhs_map->nested); + return false; +} + ColumnPtr ColumnMap::compress() const { auto compressed = nested->compress(); diff --git a/src/Columns/ColumnMap.h b/src/Columns/ColumnMap.h index 575114f8d3a..8dfa5bb5845 100644 --- a/src/Columns/ColumnMap.h +++ b/src/Columns/ColumnMap.h @@ -123,6 +123,7 @@ public: ColumnPtr compress() const override; bool hasDynamicStructure() const override { return nested->hasDynamicStructure(); } + bool dynamicStructureEquals(const IColumn & rhs) const override; void takeDynamicStructureFromSourceColumns(const Columns & source_columns) override; }; diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index 18ba8ed36ee..f4121435be9 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -1415,6 +1415,31 @@ void ColumnObject::prepareForSquashing(const std::vector & source_col } } +bool ColumnObject::dynamicStructureEquals(const IColumn & rhs) const +{ + const auto * rhs_object = typeid_cast(&rhs); + if (!rhs_object || typed_paths.size() != rhs_object->typed_paths.size() + || global_max_dynamic_paths != rhs_object->global_max_dynamic_paths || max_dynamic_types != rhs_object->max_dynamic_types + || dynamic_paths.size() != rhs_object->dynamic_paths.size()) + return false; + + for (const auto & [path, column] : typed_paths) + { + auto it = rhs_object->typed_paths.find(path); + if (it == rhs_object->typed_paths.end() || !it->second->dynamicStructureEquals(*column)) + return false; + } + + for (const auto & [path, column] : dynamic_paths) + { + auto it = rhs_object->dynamic_paths.find(path); + if (it == rhs_object->dynamic_paths.end() || !it->second->dynamicStructureEquals(*column)) + return false; + } + + return true; +} + void ColumnObject::takeDynamicStructureFromSourceColumns(const DB::Columns & source_columns) { if (!empty()) diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h index 74ae7e136ce..7b8a381d571 100644 --- a/src/Columns/ColumnObject.h +++ b/src/Columns/ColumnObject.h @@ -177,6 +177,7 @@ public: bool isFinalized() const override; bool hasDynamicStructure() const override { return true; } + bool dynamicStructureEquals(const IColumn & rhs) const override; void takeDynamicStructureFromSourceColumns(const Columns & source_columns) override; const PathToColumnMap & getTypedPaths() const { return typed_paths; } @@ -227,6 +228,7 @@ public: void setDynamicPaths(const std::vector & paths); void setDynamicPaths(const std::vector> & paths); void setMaxDynamicPaths(size_t max_dynamic_paths_); + void setGlobalMaxDynamicPaths(size_t global_max_dynamic_paths_); void setStatistics(const StatisticsPtr & statistics_) { statistics = statistics_; } void serializePathAndValueIntoSharedData(ColumnString * shared_data_paths, ColumnString * shared_data_values, std::string_view path, const IColumn & column, size_t n); diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index c3f7d10f650..28e5f03cc3c 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -757,6 +757,26 @@ bool ColumnTuple::hasDynamicStructure() const return false; } +bool ColumnTuple::dynamicStructureEquals(const IColumn & rhs) const +{ + if (const auto * rhs_tuple = typeid_cast(&rhs)) + { + const size_t tuple_size = columns.size(); + if (tuple_size != rhs_tuple->columns.size()) + return false; + + for (size_t i = 0; i < tuple_size; ++i) + if (!columns[i]->dynamicStructureEquals(*rhs_tuple->columns[i])) + return false; + + return true; + } + else + { + return false; + } +} + void ColumnTuple::takeDynamicStructureFromSourceColumns(const Columns & source_columns) { std::vector nested_source_columns; diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index c73f90f13d9..d5eee911edc 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -141,6 +141,7 @@ public: ColumnPtr & getColumnPtr(size_t idx) { return columns[idx]; } bool hasDynamicStructure() const override; + bool dynamicStructureEquals(const IColumn & rhs) const override; void takeDynamicStructureFromSourceColumns(const Columns & source_columns) override; /// Empty tuple needs a public method to manage its size. diff --git a/src/Columns/ColumnVariant.cpp b/src/Columns/ColumnVariant.cpp index 564b60e1c1d..2fa59b8e33c 100644 --- a/src/Columns/ColumnVariant.cpp +++ b/src/Columns/ColumnVariant.cpp @@ -952,7 +952,7 @@ ColumnPtr ColumnVariant::permute(const Permutation & perm, size_t limit) const if (hasOnlyNulls()) { if (limit) - return cloneResized(limit); + return cloneResized(limit ? std::min(size(), limit) : size()); /// If no limit, we can just return current immutable column. return this->getPtr(); @@ -1409,6 +1409,23 @@ bool ColumnVariant::structureEquals(const IColumn & rhs) const return true; } +bool ColumnVariant::dynamicStructureEquals(const IColumn & rhs) const +{ + const auto * rhs_variant = typeid_cast(&rhs); + if (!rhs_variant) + return false; + + const size_t num_variants = variants.size(); + if (num_variants != rhs_variant->variants.size()) + return false; + + for (size_t i = 0; i < num_variants; ++i) + if (!variants[i]->dynamicStructureEquals(rhs_variant->getVariantByGlobalDiscriminator(globalDiscriminatorByLocal(i)))) + return false; + + return true; +} + ColumnPtr ColumnVariant::compress() const { ColumnPtr local_discriminators_compressed = local_discriminators->compress(); diff --git a/src/Columns/ColumnVariant.h b/src/Columns/ColumnVariant.h index f90a812703d..a68a961169c 100644 --- a/src/Columns/ColumnVariant.h +++ b/src/Columns/ColumnVariant.h @@ -336,6 +336,7 @@ public: void extend(const std::vector & old_to_new_global_discriminators, std::vector> && new_variants_and_discriminators); bool hasDynamicStructure() const override; + bool dynamicStructureEquals(const IColumn & rhs) const override; void takeDynamicStructureFromSourceColumns(const Columns & source_columns) override; private: diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 95becba3fdb..c77b089812e 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -635,6 +635,9 @@ public: /// Checks if column has dynamic subcolumns. virtual bool hasDynamicStructure() const { return false; } + + /// For columns with dynamic subcolumns checks if columns have equal dynamic structure. + [[nodiscard]] virtual bool dynamicStructureEquals(const IColumn & rhs) const { return structureEquals(rhs); } /// For columns with dynamic subcolumns this method takes dynamic structure from source columns /// and creates proper resulting dynamic structure in advance for merge of these source columns. virtual void takeDynamicStructureFromSourceColumns(const std::vector & /*source_columns*/) {} diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index f419a23b209..262f3ca3a7a 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -331,7 +331,7 @@ AsynchronousMetrics::~AsynchronousMetrics() AsynchronousMetricValues AsynchronousMetrics::getValues() const { - std::lock_guard lock(data_mutex); + SharedLockGuard lock(values_mutex); return values; } @@ -1807,7 +1807,10 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) first_run = false; // Finally, update the current metrics. - values = new_values; + { + std::lock_guard values_lock(values_mutex); + values.swap(new_values); + } } } diff --git a/src/Common/AsynchronousMetrics.h b/src/Common/AsynchronousMetrics.h index 215dc6e1337..e8d4b7980f9 100644 --- a/src/Common/AsynchronousMetrics.h +++ b/src/Common/AsynchronousMetrics.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -100,6 +101,7 @@ private: std::condition_variable wait_cond; bool quit TSA_GUARDED_BY(thread_mutex) = false; + /// Protects all raw data and serializes multiple updates. mutable std::mutex data_mutex; /// Some values are incremental and we have to calculate the difference. @@ -107,7 +109,15 @@ private: bool first_run TSA_GUARDED_BY(data_mutex) = true; TimePoint previous_update_time TSA_GUARDED_BY(data_mutex); - AsynchronousMetricValues values TSA_GUARDED_BY(data_mutex); + /// Protects saved values. + mutable SharedMutex values_mutex; + /// Values store the result of the last update prepared for reading. +#ifdef OS_LINUX + AsynchronousMetricValues values TSA_GUARDED_BY(values_mutex); +#else + /// When SharedMutex == std::shared_mutex it may not be annotated with the 'capability'. + AsynchronousMetricValues values; +#endif #if defined(OS_LINUX) || defined(OS_FREEBSD) MemoryStatisticsOS memory_stat TSA_GUARDED_BY(data_mutex); diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 076bec35e05..7cdabb05ce0 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -41,6 +41,10 @@ M(PostgreSQLConnection, "Number of client connections using PostgreSQL protocol") \ M(OpenFileForRead, "Number of files open for reading") \ M(OpenFileForWrite, "Number of files open for writing") \ + M(Compressing, "Number of compress operations using internal compression codecs") \ + M(Decompressing, "Number of decompress operations using internal compression codecs") \ + M(ParallelCompressedWriteBufferThreads, "Number of threads in all instances of ParallelCompressedWriteBuffer - these threads are doing parallel compression and writing") \ + M(ParallelCompressedWriteBufferWait, "Number of threads in all instances of ParallelCompressedWriteBuffer that are currently waiting for buffer to become available for writing") \ M(TotalTemporaryFiles, "Number of temporary files created") \ M(TemporaryFilesForSort, "Number of temporary files created for external sorting") \ M(TemporaryFilesForAggregation, "Number of temporary files created for external aggregation") \ @@ -100,6 +104,9 @@ M(IOThreads, "Number of threads in the IO thread pool.") \ M(IOThreadsActive, "Number of threads in the IO thread pool running a task.") \ M(IOThreadsScheduled, "Number of queued or active jobs in the IO thread pool.") \ + M(CompressionThread, "Number of threads in compression thread pools.") \ + M(CompressionThreadActive, "Number of threads in compression thread pools running a task.") \ + M(CompressionThreadScheduled, "Number of queued or active jobs in compression thread pools.") \ M(ThreadPoolRemoteFSReaderThreads, "Number of threads in the thread pool for remote_filesystem_read_method=threadpool.") \ M(ThreadPoolRemoteFSReaderThreadsActive, "Number of threads in the thread pool for remote_filesystem_read_method=threadpool running a task.") \ M(ThreadPoolRemoteFSReaderThreadsScheduled, "Number of queued or active jobs in the thread pool for remote_filesystem_read_method=threadpool.") \ diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 320fc06cb2f..644c9a19738 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -251,7 +251,7 @@ void Exception::setThreadFramePointers(ThreadFramePointersBase frame_pointers) thread_frame_pointers.frame_pointers = std::move(frame_pointers); } -static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string & start_of_message) +static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string & start_of_message, LogsLevel level) { if (!isLoggingEnabled()) return; @@ -262,14 +262,25 @@ static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string if (!start_of_message.empty()) message.text = fmt::format("{}: {}", start_of_message, message.text); - LOG_ERROR(logger, message); + switch (level) + { + case LogsLevel::none: break; + case LogsLevel::test: LOG_TEST(logger, message); break; + case LogsLevel::trace: LOG_TRACE(logger, message); break; + case LogsLevel::debug: LOG_DEBUG(logger, message); break; + case LogsLevel::information: LOG_INFO(logger, message); break; + case LogsLevel::warning: LOG_WARNING(logger, message); break; + case LogsLevel::error: LOG_ERROR(logger, message); break; + case LogsLevel::fatal: LOG_FATAL(logger, message); break; + } + } catch (...) // NOLINT(bugprone-empty-catch) { } } -void tryLogCurrentException(const char * log_name, const std::string & start_of_message) +void tryLogCurrentException(const char * log_name, const std::string & start_of_message, LogsLevel level) { if (!isLoggingEnabled()) return; @@ -283,10 +294,10 @@ void tryLogCurrentException(const char * log_name, const std::string & start_of_ /// getLogger can allocate memory too auto logger = getLogger(log_name); - tryLogCurrentExceptionImpl(logger.get(), start_of_message); + tryLogCurrentExceptionImpl(logger.get(), start_of_message, level); } -void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_message) +void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_message, LogsLevel level) { /// Under high memory pressure, new allocations throw a /// MEMORY_LIMIT_EXCEEDED exception. @@ -295,17 +306,17 @@ void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_ /// MemoryTracker until the exception will be logged. LockMemoryExceptionInThread lock_memory_tracker(VariableContext::Global); - tryLogCurrentExceptionImpl(logger, start_of_message); + tryLogCurrentExceptionImpl(logger, start_of_message, level); } -void tryLogCurrentException(LoggerPtr logger, const std::string & start_of_message) +void tryLogCurrentException(LoggerPtr logger, const std::string & start_of_message, LogsLevel level) { - tryLogCurrentException(logger.get(), start_of_message); + tryLogCurrentException(logger.get(), start_of_message, level); } -void tryLogCurrentException(const AtomicLogger & logger, const std::string & start_of_message) +void tryLogCurrentException(const AtomicLogger & logger, const std::string & start_of_message, LogsLevel level) { - tryLogCurrentException(logger.load(), start_of_message); + tryLogCurrentException(logger.load(), start_of_message, level); } static void getNoSpaceLeftInfoMessage(std::filesystem::path path, String & msg) diff --git a/src/Common/Exception.h b/src/Common/Exception.h index 8ec640ff642..edc1b95bca4 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -276,10 +277,10 @@ using Exceptions = std::vector; * Can be used in destructors in the catch-all block. */ /// TODO: Logger leak constexpr overload -void tryLogCurrentException(const char * log_name, const std::string & start_of_message = ""); -void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_message = ""); -void tryLogCurrentException(LoggerPtr logger, const std::string & start_of_message = ""); -void tryLogCurrentException(const AtomicLogger & logger, const std::string & start_of_message = ""); +void tryLogCurrentException(const char * log_name, const std::string & start_of_message = "", LogsLevel level = LogsLevel::error); +void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_message = "", LogsLevel level = LogsLevel::error); +void tryLogCurrentException(LoggerPtr logger, const std::string & start_of_message = "", LogsLevel level = LogsLevel::error); +void tryLogCurrentException(const AtomicLogger & logger, const std::string & start_of_message = "", LogsLevel level = LogsLevel::error); /** Prints current exception in canonical format. diff --git a/src/Common/FieldVisitorScale.cpp b/src/Common/FieldVisitorScale.cpp new file mode 100644 index 00000000000..a6c0f6d0c5b --- /dev/null +++ b/src/Common/FieldVisitorScale.cpp @@ -0,0 +1,30 @@ +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +FieldVisitorScale::FieldVisitorScale(Int32 rhs_) : rhs(rhs_) {} + +void FieldVisitorScale::operator() (Int64 & x) const { x *= rhs; } +void FieldVisitorScale::operator() (UInt64 & x) const { x *= rhs; } +void FieldVisitorScale::operator() (Float64 & x) const { x *= rhs; } +void FieldVisitorScale::operator() (Null &) const { /*Do not scale anything*/ } + +void FieldVisitorScale::operator() (String &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Strings"); } +void FieldVisitorScale::operator() (Array &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Arrays"); } +void FieldVisitorScale::operator() (Tuple &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Tuples"); } +void FieldVisitorScale::operator() (Map &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Maps"); } +void FieldVisitorScale::operator() (Object &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Objects"); } +void FieldVisitorScale::operator() (UUID &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale UUIDs"); } +void FieldVisitorScale::operator() (IPv4 &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale IPv4s"); } +void FieldVisitorScale::operator() (IPv6 &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale IPv6s"); } +void FieldVisitorScale::operator() (CustomType & x) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale custom type {}", x.getTypeName()); } +void FieldVisitorScale::operator() (AggregateFunctionStateData &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale AggregateFunctionStates"); } +void FieldVisitorScale::operator() (bool &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Bools"); } + +} diff --git a/src/Common/FieldVisitorScale.h b/src/Common/FieldVisitorScale.h new file mode 100644 index 00000000000..90d86cc53bd --- /dev/null +++ b/src/Common/FieldVisitorScale.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include + +namespace DB +{ + +/** Implements `*=` operation by number + */ +class FieldVisitorScale : public StaticVisitor +{ +private: + Int32 rhs; + +public: + explicit FieldVisitorScale(Int32 rhs_); + + void operator() (Int64 & x) const; + void operator() (UInt64 & x) const; + void operator() (Float64 & x) const; + void operator() (Null &) const; + [[noreturn]] void operator() (String &) const; + [[noreturn]] void operator() (Array &) const; + [[noreturn]] void operator() (Tuple &) const; + [[noreturn]] void operator() (Map &) const; + [[noreturn]] void operator() (Object &) const; + [[noreturn]] void operator() (UUID &) const; + [[noreturn]] void operator() (IPv4 &) const; + [[noreturn]] void operator() (IPv6 &) const; + [[noreturn]] void operator() (AggregateFunctionStateData &) const; + [[noreturn]] void operator() (CustomType &) const; + [[noreturn]] void operator() (bool &) const; + + template + void operator() (DecimalField & x) const { x = DecimalField(x.getValue() * T(rhs), x.getScale()); } + + template + requires is_big_int_v + void operator() (T & x) const { x *= rhs; } +}; + +} diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h index f4374a0f2ca..d379c3f6a87 100644 --- a/src/Common/HashTable/HashTable.h +++ b/src/Common/HashTable/HashTable.h @@ -658,16 +658,11 @@ protected: { if (!std::is_trivially_destructible_v) { - for (iterator it = begin(), it_end = end(); it != it_end; ++it) + for (iterator it = begin(), it_end = end(); it != it_end;) { - it.ptr->~Cell(); - /// In case of poison_in_dtor=1 it will be poisoned, - /// but it maybe used later, during iteration. - /// - /// NOTE, that technically this is UB [1], but OK for now. - /// - /// [1]: https://github.com/google/sanitizers/issues/854#issuecomment-329661378 - __msan_unpoison(it.ptr, sizeof(*it.ptr)); + auto ptr = it.ptr; + ++it; + ptr->~Cell(); } /// Everything had been destroyed in the loop above, reset the flag diff --git a/src/Common/LockGuard.h b/src/Common/LockGuard.h index 8a98c5f553a..03c8a3e7617 100644 --- a/src/Common/LockGuard.h +++ b/src/Common/LockGuard.h @@ -1,23 +1,47 @@ #pragma once -#include #include +#include +#include namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +}; + /** LockGuard provides RAII-style locking mechanism for a mutex. - ** It's intended to be used like std::unique_ptr but with TSA annotations + ** It's intended to be used like std::unique_lock but with TSA annotations */ template class TSA_SCOPED_LOCKABLE LockGuard { public: - explicit LockGuard(Mutex & mutex_) TSA_ACQUIRE(mutex_) : mutex(mutex_) { mutex.lock(); } - ~LockGuard() TSA_RELEASE() { mutex.unlock(); } + explicit LockGuard(Mutex & mutex_) TSA_ACQUIRE(mutex_) : mutex(mutex_) { lock(); } + ~LockGuard() TSA_RELEASE() { if (locked) unlock(); } + + void lock() TSA_ACQUIRE() + { + /// Don't allow recursive_mutex for now. + if (locked) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't lock twice the same mutex"); + mutex.lock(); + locked = true; + } + + void unlock() TSA_RELEASE() + { + if (!locked) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't unlock the mutex without locking it first"); + mutex.unlock(); + locked = false; + } private: Mutex & mutex; + bool locked = false; }; template typename TLockGuard, typename Mutex> diff --git a/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp b/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp index b8413bfadd7..8bb411f1437 100644 --- a/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp +++ b/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp @@ -568,7 +568,7 @@ std::vector NamedCollectionsMetadataStorage::listCollections() cons std::vector collections; collections.reserve(paths.size()); for (const auto & path : paths) - collections.push_back(std::filesystem::path(path).stem()); + collections.push_back(unescapeForFileName(std::filesystem::path(path).stem())); return collections; } diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 3a102238fbe..7b9f670d340 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -547,6 +547,7 @@ The server successfully detected this situation and will download merged part fr M(FilesystemCacheLoadMetadataMicroseconds, "Time spent loading filesystem cache metadata", ValueType::Microseconds) \ M(FilesystemCacheEvictedBytes, "Number of bytes evicted from filesystem cache", ValueType::Bytes) \ M(FilesystemCacheEvictedFileSegments, "Number of file segments evicted from filesystem cache", ValueType::Number) \ + M(FilesystemCacheBackgroundDownloadQueuePush, "Number of file segments sent for background download in filesystem cache", ValueType::Number) \ M(FilesystemCacheEvictionSkippedFileSegments, "Number of file segments skipped for eviction because of being in unreleasable state", ValueType::Number) \ M(FilesystemCacheEvictionSkippedEvictingFileSegments, "Number of file segments skipped for eviction because of being in evicting state", ValueType::Number) \ M(FilesystemCacheEvictionTries, "Number of filesystem cache eviction attempts", ValueType::Number) \ @@ -745,6 +746,12 @@ The server successfully detected this situation and will download merged part fr M(ReadTaskRequestsSentElapsedMicroseconds, "Time spent in callbacks requested from the remote server back to the initiator server to choose the read task (for s3Cluster table function and similar). Measured on the remote server side.", ValueType::Microseconds) \ M(MergeTreeReadTaskRequestsSentElapsedMicroseconds, "Time spent in callbacks requested from the remote server back to the initiator server to choose the read task (for MergeTree tables). Measured on the remote server side.", ValueType::Microseconds) \ M(MergeTreeAllRangesAnnouncementsSentElapsedMicroseconds, "Time spent in sending the announcement from the remote server to the initiator server about the set of data parts (for MergeTree tables). Measured on the remote server side.", ValueType::Microseconds) \ + M(MergerMutatorsGetPartsForMergeElapsedMicroseconds, "Time spent to take data parts snapshot to build ranges from them.", ValueType::Microseconds) \ + M(MergerMutatorPrepareRangesForMergeElapsedMicroseconds, "Time spent to prepare parts ranges which can be merged according to merge predicate.", ValueType::Microseconds) \ + M(MergerMutatorSelectPartsForMergeElapsedMicroseconds, "Time spent to select parts from ranges which can be merged.", ValueType::Microseconds) \ + M(MergerMutatorRangesForMergeCount, "Amount of candidate ranges for merge", ValueType::Number) \ + M(MergerMutatorPartsInRangesForMergeCount, "Amount of candidate parts for merge", ValueType::Number) \ + M(MergerMutatorSelectRangePartsCount, "Amount of parts in selected range for merge", ValueType::Number) \ \ M(ConnectionPoolIsFullMicroseconds, "Total time spent waiting for a slot in connection pool.", ValueType::Microseconds) \ M(AsyncLoaderWaitMicroseconds, "Total time a query was waiting for async loader jobs.", ValueType::Microseconds) \ diff --git a/src/Common/Scheduler/Workload/WorkloadEntityStorageBase.cpp b/src/Common/Scheduler/Workload/WorkloadEntityStorageBase.cpp index 968dfd90796..c758111a53e 100644 --- a/src/Common/Scheduler/Workload/WorkloadEntityStorageBase.cpp +++ b/src/Common/Scheduler/Workload/WorkloadEntityStorageBase.cpp @@ -48,9 +48,9 @@ ASTPtr normalizeCreateWorkloadEntityQuery(const IAST & create_query) /// Returns a type of a workload entity `ptr` WorkloadEntityType getEntityType(const ASTPtr & ptr) { - if (auto * res = typeid_cast(ptr.get())) + if (auto * res = typeid_cast(ptr.get()); res) return WorkloadEntityType::Workload; - if (auto * res = typeid_cast(ptr.get())) + if (auto * res = typeid_cast(ptr.get()); res) return WorkloadEntityType::Resource; chassert(false); return WorkloadEntityType::MAX; @@ -106,7 +106,7 @@ void forEachReference( for (const String & resource : resources) func(resource, res->getWorkloadName(), ReferenceType::ForResource); } - if (auto * res = typeid_cast(source_entity.get())) + if (auto * res = typeid_cast(source_entity.get()); res) { // RESOURCE has no references to be validated, we allow mentioned disks to be created later } diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h index 7e497245acc..b52e4a60571 100644 --- a/src/Common/ThreadPool.h +++ b/src/Common/ThreadPool.h @@ -122,7 +122,7 @@ public: void scheduleOrThrowOnError(Job job, Priority priority = {}); /// Similar to scheduleOrThrowOnError(...). Wait for specified amount of time and schedule a job or return false. - bool trySchedule(Job job, Priority priority = {}, uint64_t wait_microseconds = 0) noexcept; + [[nodiscard]] bool trySchedule(Job job, Priority priority = {}, uint64_t wait_microseconds = 0) noexcept; /// Similar to scheduleOrThrowOnError(...). Wait for specified amount of time and schedule a job or throw an exception. void scheduleOrThrow(Job job, Priority priority = {}, uint64_t wait_microseconds = 0, bool propagate_opentelemetry_tracing_context = true); @@ -142,7 +142,7 @@ public: /// Returns true if the pool already terminated /// (and any further scheduling will produce CANNOT_SCHEDULE_TASK exception) - bool finished() const; + [[nodiscard]] bool finished() const; void setMaxThreads(size_t value); void setMaxFreeThreads(size_t value); diff --git a/src/Compression/CompressedWriteBuffer.cpp b/src/Compression/CompressedWriteBuffer.cpp index c3acfcb7da6..b6dab2a190e 100644 --- a/src/Compression/CompressedWriteBuffer.cpp +++ b/src/Compression/CompressedWriteBuffer.cpp @@ -2,7 +2,6 @@ #include #include -#include #include #include diff --git a/src/Compression/ICompressionCodec.cpp b/src/Compression/ICompressionCodec.cpp index 418667a3a8f..a31d0485982 100644 --- a/src/Compression/ICompressionCodec.cpp +++ b/src/Compression/ICompressionCodec.cpp @@ -5,11 +5,18 @@ #include #include #include +#include #include #include #include +namespace CurrentMetrics +{ + extern const Metric Compressing; + extern const Metric Decompressing; +} + namespace DB { @@ -80,6 +87,8 @@ UInt32 ICompressionCodec::compress(const char * source, UInt32 source_size, char { assert(source != nullptr && dest != nullptr); + CurrentMetrics::Increment metric_increment(CurrentMetrics::Compressing); + dest[0] = getMethodByte(); UInt8 header_size = getHeaderSize(); /// Write data from header_size @@ -93,6 +102,8 @@ UInt32 ICompressionCodec::decompress(const char * source, UInt32 source_size, ch { assert(source != nullptr && dest != nullptr); + CurrentMetrics::Increment metric_increment(CurrentMetrics::Decompressing); + UInt8 header_size = getHeaderSize(); if (source_size < header_size) throw Exception(decompression_error_code, diff --git a/src/Compression/ParallelCompressedWriteBuffer.cpp b/src/Compression/ParallelCompressedWriteBuffer.cpp new file mode 100644 index 00000000000..3831d07e91a --- /dev/null +++ b/src/Compression/ParallelCompressedWriteBuffer.cpp @@ -0,0 +1,166 @@ +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include + + +namespace CurrentMetrics +{ + extern const Metric ParallelCompressedWriteBufferThreads; + extern const Metric ParallelCompressedWriteBufferWait; +} + +namespace DB +{ + +ParallelCompressedWriteBuffer::ParallelCompressedWriteBuffer( + WriteBuffer & out_, + CompressionCodecPtr codec_, + size_t buf_size_, + size_t num_threads_, + ThreadPool & pool_) + : WriteBuffer(nullptr, 0), out(out_), codec(codec_), buf_size(buf_size_), num_threads(num_threads_), pool(pool_) +{ + buffers.emplace_back(buf_size); + current_buffer = buffers.begin(); + BufferBase::set(current_buffer->uncompressed.data(), buf_size, 0); +} + +void ParallelCompressedWriteBuffer::nextImpl() +{ + if (!offset()) + return; + + std::unique_lock lock(mutex); + + /// The buffer will be compressed and processed in the thread. + current_buffer->busy = true; + current_buffer->sequence_num = current_sequence_num; + ++current_sequence_num; + current_buffer->uncompressed_size = offset(); + pool.scheduleOrThrowOnError([this, my_current_buffer = current_buffer, thread_group = CurrentThread::getGroup()] + { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachFromGroupIfNotDetached(); + ); + + if (thread_group) + CurrentThread::attachToGroupIfDetached(thread_group); + setThreadName("ParallelCompres"); + + compress(my_current_buffer); + }); + + BufferPair * previous_buffer = &*current_buffer; + ++current_buffer; + if (current_buffer == buffers.end()) + { + if (buffers.size() < num_threads) + { + /// If we didn't use all num_threads buffers yet, create a new one. + current_buffer = buffers.emplace(current_buffer, buf_size); + } + else + { + /// Otherwise, wrap around to the first buffer in the list. + current_buffer = buffers.begin(); + } + } + + /// Wait while the buffer becomes not busy + if (current_buffer->busy) + { + CurrentMetrics::Increment metric_increment(CurrentMetrics::ParallelCompressedWriteBufferWait); + cond.wait(lock, [&]{ return !current_buffer->busy; }); + } + + /// Now this buffer can be used. + current_buffer->previous = previous_buffer; + BufferBase::set(current_buffer->uncompressed.data(), buf_size, 0); +} + +void ParallelCompressedWriteBuffer::finalizeImpl() +{ + next(); + pool.wait(); +} + +void ParallelCompressedWriteBuffer::compress(Iterator buffer) +{ + CurrentMetrics::Increment metric_increment(CurrentMetrics::ParallelCompressedWriteBufferThreads); + + chassert(buffer->uncompressed_size <= INT_MAX); + UInt32 uncompressed_size = static_cast(buffer->uncompressed_size); + UInt32 compressed_reserve_size = codec->getCompressedReserveSize(uncompressed_size); + + /// If all previous buffers have been written, + /// and if the output buffer has the required capacity, + /// we can compress data directly into the output buffer. + size_t required_out_capacity = compressed_reserve_size + sizeof(CityHash_v1_0_2::uint128); + bool can_write_directly = false; + + if (!buffer->previous) + { + can_write_directly = out.available() >= required_out_capacity; + } + else + { + std::unique_lock lock(mutex); + can_write_directly = (!buffer->previous->busy || buffer->previous->sequence_num > buffer->sequence_num) + && out.available() >= required_out_capacity; + } + + if (can_write_directly) + { + char * out_compressed_ptr = out.position() + sizeof(CityHash_v1_0_2::uint128); + UInt32 compressed_size = codec->compress(buffer->uncompressed.data(), uncompressed_size, out_compressed_ptr); + + CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(out_compressed_ptr, compressed_size); + + writeBinaryLittleEndian(checksum.low64, out); + writeBinaryLittleEndian(checksum.high64, out); + + out.position() += compressed_size; + } + else + { + buffer->compressed.resize(compressed_reserve_size); + UInt32 compressed_size = codec->compress(buffer->uncompressed.data(), uncompressed_size, buffer->compressed.data()); + + CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(buffer->compressed.data(), compressed_size); + + /// Wait while all previous buffers have been written. + if (buffer->previous) + { + CurrentMetrics::Increment metric_wait_increment(CurrentMetrics::ParallelCompressedWriteBufferWait); + std::unique_lock lock(mutex); + cond.wait(lock, [&]{ return !buffer->previous->busy || buffer->previous->sequence_num > buffer->sequence_num; }); + } + + writeBinaryLittleEndian(checksum.low64, out); + writeBinaryLittleEndian(checksum.high64, out); + + out.write(buffer->compressed.data(), compressed_size); + } + + std::unique_lock lock(mutex); + buffer->busy = false; + cond.notify_all(); +} + +ParallelCompressedWriteBuffer::~ParallelCompressedWriteBuffer() +{ + if (!canceled) + finalize(); +} + +} diff --git a/src/Compression/ParallelCompressedWriteBuffer.h b/src/Compression/ParallelCompressedWriteBuffer.h new file mode 100644 index 00000000000..38a3a083e19 --- /dev/null +++ b/src/Compression/ParallelCompressedWriteBuffer.h @@ -0,0 +1,70 @@ +#pragma once + +#include +#include + +#include + +#include +#include +#include +#include +#include + + +namespace DB +{ + +/** Uses multi-buffering for parallel compression. + * When the buffer is filled, it will be compressed in the background, + * and a new buffer is created for the next input data. + */ +class ParallelCompressedWriteBuffer final : public WriteBuffer +{ +public: + explicit ParallelCompressedWriteBuffer( + WriteBuffer & out_, + CompressionCodecPtr codec_, + size_t buf_size_, + size_t num_threads_, + ThreadPool & pool_); + + ~ParallelCompressedWriteBuffer() override; + +private: + void nextImpl() override; + void finalizeImpl() override; + + WriteBuffer & out; + CompressionCodecPtr codec; + size_t buf_size; + size_t num_threads; + ThreadPool & pool; + + struct BufferPair + { + explicit BufferPair(size_t input_size) + : uncompressed(input_size) + { + } + + Memory<> uncompressed; + size_t uncompressed_size = 0; + PODArray compressed; + BufferPair * previous = nullptr; + size_t sequence_num = 0; + bool busy = false; + }; + + std::mutex mutex; + std::condition_variable cond; + std::list buffers; + + using Iterator = std::list::iterator; + Iterator current_buffer; + size_t current_sequence_num = 0; + + void compress(Iterator buffer); +}; + +} diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index 9648fdd4530..c56e698766a 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -330,7 +330,7 @@ TYPED_TEST(CoordinationTest, TestSummingRaft1) this->setLogDirectory("./logs"); this->setStateFileDirectory("."); - SummingRaftServer s1(1, "localhost", 44444, this->keeper_context); + SummingRaftServer s1(1, "localhost", 0, this->keeper_context); SCOPE_EXIT(if (std::filesystem::exists("./state")) std::filesystem::remove("./state");); /// Single node is leader diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index a41b7d9dbf4..9795812f15b 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -873,6 +873,12 @@ In CREATE TABLE statement allows specifying Variant type with similar variant ty )", 0) \ DECLARE(Bool, allow_suspicious_primary_key, false, R"( Allow suspicious `PRIMARY KEY`/`ORDER BY` for MergeTree (i.e. SimpleAggregateFunction). +)", 0) \ + DECLARE(Bool, allow_suspicious_types_in_group_by, false, R"( +Allows or restricts using [Variant](../../sql-reference/data-types/variant.md) and [Dynamic](../../sql-reference/data-types/dynamic.md) types in GROUP BY keys. +)", 0) \ + DECLARE(Bool, allow_suspicious_types_in_order_by, false, R"( +Allows or restricts using [Variant](../../sql-reference/data-types/variant.md) and [Dynamic](../../sql-reference/data-types/dynamic.md) types in ORDER BY keys. )", 0) \ DECLARE(Bool, compile_expressions, false, R"( Compile some scalar functions and operators to native code. Due to a bug in the LLVM compiler infrastructure, on AArch64 machines, it is known to lead to a nullptr dereference and, consequently, server crash. Do not enable this setting. @@ -2863,7 +2869,7 @@ Limit on size of multipart/form-data content. This setting cannot be parsed from DECLARE(Bool, calculate_text_stack_trace, true, R"( Calculate text stack trace in case of exceptions during query execution. This is the default. It requires symbol lookups that may slow down fuzzing tests when a huge amount of wrong queries are executed. In normal cases, you should not disable this option. )", 0) \ - DECLARE(Bool, enable_job_stack_trace, false, R"( + DECLARE(Bool, enable_job_stack_trace, true, R"( Output stack trace of a job creator when job results in exception )", 0) \ DECLARE(Bool, allow_ddl, true, R"( @@ -2886,6 +2892,9 @@ Possible values: **See Also** - [ORDER BY Clause](../../sql-reference/statements/select/order-by.md/#optimize_read_in_order) +)", 0) \ + DECLARE(Bool, read_in_order_use_virtual_row, false, R"( +Use virtual row while reading in order of primary key or its monotonic function fashion. It is useful when searching over multiple parts as only relevant ones are touched. )", 0) \ DECLARE(Bool, optimize_read_in_window_order, true, R"( Enable ORDER BY optimization in window clause for reading data in corresponding order in MergeTree tables. @@ -4236,7 +4245,7 @@ Rewrite aggregate functions with if expression as argument when logically equiva For example, `avg(if(cond, col, null))` can be rewritten to `avgOrNullIf(cond, col)`. It may improve performance. :::note -Supported only with experimental analyzer (`enable_analyzer = 1`). +Supported only with the analyzer (`enable_analyzer = 1`). ::: )", 0) \ DECLARE(Bool, optimize_rewrite_array_exists_to_has, false, R"( @@ -4551,7 +4560,7 @@ Possible values: - 0 - Disable - 1 - Enable )", 0) \ - DECLARE(Bool, query_plan_merge_filters, false, R"( + DECLARE(Bool, query_plan_merge_filters, true, R"( Allow to merge filters in the query plan )", 0) \ DECLARE(Bool, query_plan_filter_push_down, true, R"( @@ -4866,6 +4875,9 @@ Limit on size of a single batch of file segments that a read buffer can request )", 0) \ DECLARE(UInt64, filesystem_cache_reserve_space_wait_lock_timeout_milliseconds, 1000, R"( Wait time to lock cache for space reservation in filesystem cache +)", 0) \ + DECLARE(Bool, filesystem_cache_prefer_bigger_buffer_size, true, R"( +Prefer bigger buffer size if filesystem cache is enabled to avoid writing small file segments which deteriorate cache performance. On the other hand, enabling this setting might increase memory usage. )", 0) \ DECLARE(UInt64, temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds, (10 * 60 * 1000), R"( Wait time to lock cache for space reservation for temporary data in filesystem cache @@ -5129,6 +5141,12 @@ Only in ClickHouse Cloud. A window for sending ACK for DataPacket sequence in a )", 0) \ DECLARE(Bool, distributed_cache_discard_connection_if_unread_data, true, R"( Only in ClickHouse Cloud. Discard connection if some data is unread. +)", 0) \ + DECLARE(Bool, filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage, true, R"( +Only in ClickHouse Cloud. Wait time to lock cache for space reservation in filesystem cache +)", 0) \ + DECLARE(Bool, filesystem_cache_enable_background_download_during_fetch, true, R"( +Only in ClickHouse Cloud. Wait time to lock cache for space reservation in filesystem cache )", 0) \ \ DECLARE(Bool, parallelize_output_from_storages, true, R"( @@ -5139,6 +5157,7 @@ The setting allows a user to provide own deduplication semantic in MergeTree/Rep For example, by providing a unique value for the setting in each INSERT statement, user can avoid the same inserted data being deduplicated. + Possible values: - Any string @@ -5613,7 +5632,7 @@ If true, and JOIN can be executed with parallel replicas algorithm, and all stor DECLARE(UInt64, parallel_replicas_mark_segment_size, 0, R"( Parts virtually divided into segments to be distributed between replicas for parallel reading. This setting controls the size of these segments. Not recommended to change until you're absolutely sure in what you're doing. Value should be in range [128; 16384] )", BETA) \ - DECLARE(Bool, parallel_replicas_local_plan, false, R"( + DECLARE(Bool, parallel_replicas_local_plan, true, R"( Build local plan for local replica )", BETA) \ \ @@ -5695,6 +5714,8 @@ If enabled, MongoDB tables will return an error when a MongoDB query cannot be b )", 0) \ DECLARE(Bool, implicit_select, false, R"( Allow writing simple SELECT queries without the leading SELECT keyword, which makes it simple for calculator-style usage, e.g. `1 + 2` becomes a valid query. + +In `clickhouse-local` it is enabled by default and can be explicitly disabled. )", 0) \ \ \ @@ -5852,7 +5873,7 @@ Experimental data deduplication for SELECT queries based on part UUIDs // Please add settings related to formats in Core/FormatFactorySettings.h, move obsolete settings to OBSOLETE_SETTINGS and obsolete format settings to OBSOLETE_FORMAT_SETTINGS. #define OBSOLETE_SETTINGS(M, ALIAS) \ - /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ + /** Obsolete settings which are kept around for compatibility reasons. They have no effect anymore. */ \ MAKE_OBSOLETE(M, Bool, update_insert_deduplication_token_in_dependent_materialized_views, 0) \ MAKE_OBSOLETE(M, UInt64, max_memory_usage_for_all_queries, 0) \ MAKE_OBSOLETE(M, UInt64, multiple_joins_rewriter_version, 0) \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index ccd01f8d71f..80b04701133 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -64,7 +64,22 @@ static std::initializer_listsize(), perm); + } /// Set the cursor to the beginning of the new block. - void reset(const Columns & columns, const Block & block, IColumn::Permutation * perm = nullptr) + void reset(const Columns & columns, const Block & block, UInt64 num_rows, IColumn::Permutation * perm = nullptr) { all_columns.clear(); sort_columns.clear(); @@ -125,7 +136,7 @@ struct SortCursorImpl } pos = 0; - rows = all_columns[0]->size(); + rows = num_rows; permutation = perm; } diff --git a/src/Core/SortDescription.h b/src/Core/SortDescription.h index 5c6f3e3150a..7a7c92f3b53 100644 --- a/src/Core/SortDescription.h +++ b/src/Core/SortDescription.h @@ -33,9 +33,12 @@ struct FillColumnDescription DataTypePtr fill_to_type; Field fill_step; /// Default = +1 or -1 according to direction std::optional step_kind; + Field fill_staleness; /// Default = Null - should not be considered + std::optional staleness_kind; - using StepFunction = std::function; + using StepFunction = std::function; StepFunction step_func; + StepFunction staleness_step_func; }; /// Description of the sorting rule by one column. diff --git a/src/Core/fuzzers/CMakeLists.txt b/src/Core/fuzzers/CMakeLists.txt index c60ce0e097f..51db6fa0b53 100644 --- a/src/Core/fuzzers/CMakeLists.txt +++ b/src/Core/fuzzers/CMakeLists.txt @@ -1,2 +1,2 @@ clickhouse_add_executable (names_and_types_fuzzer names_and_types_fuzzer.cpp) -target_link_libraries (names_and_types_fuzzer PRIVATE) +target_link_libraries (names_and_types_fuzzer PRIVATE dbms) diff --git a/src/DataTypes/DataTypeObject.cpp b/src/DataTypes/DataTypeObject.cpp index 18bfed9c5c3..30bf470083d 100644 --- a/src/DataTypes/DataTypeObject.cpp +++ b/src/DataTypes/DataTypeObject.cpp @@ -1,6 +1,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -230,6 +233,15 @@ MutableColumnPtr DataTypeObject::createColumn() const return ColumnObject::create(std::move(typed_path_columns), max_dynamic_paths, max_dynamic_types); } +void DataTypeObject::forEachChild(const ChildCallback & callback) const +{ + for (const auto & [path, type] : typed_paths) + { + callback(*type); + type->forEachChild(callback); + } +} + namespace { @@ -522,6 +534,13 @@ static DataTypePtr createObject(const ASTPtr & arguments, const DataTypeObject:: return std::make_shared(schema_format, std::move(typed_paths), std::move(paths_to_skip), std::move(path_regexps_to_skip), max_dynamic_paths, max_dynamic_types); } +const DataTypePtr & DataTypeObject::getTypeOfSharedData() +{ + /// Array(Tuple(String, String)) + static const DataTypePtr type = std::make_shared(std::make_shared(DataTypes{std::make_shared(), std::make_shared()}, Names{"paths", "values"})); + return type; +} + static DataTypePtr createJSON(const ASTPtr & arguments) { auto context = CurrentThread::getQueryContext(); diff --git a/src/DataTypes/DataTypeObject.h b/src/DataTypes/DataTypeObject.h index 7eb2e7729de..70e2d4d177d 100644 --- a/src/DataTypes/DataTypeObject.h +++ b/src/DataTypes/DataTypeObject.h @@ -50,6 +50,8 @@ public: bool equals(const IDataType & rhs) const override; + void forEachChild(const ChildCallback &) const override; + bool hasDynamicSubcolumnsData() const override { return true; } std::unique_ptr getDynamicSubcolumnData(std::string_view subcolumn_name, const SubstreamData & data, bool throw_if_null) const override; @@ -63,6 +65,9 @@ public: size_t getMaxDynamicTypes() const { return max_dynamic_types; } size_t getMaxDynamicPaths() const { return max_dynamic_paths; } + /// Shared data has type Array(Tuple(String, String)). + static const DataTypePtr & getTypeOfSharedData(); + private: SchemaFormat schema_format; /// Set of paths with types that were specified in type declaration. diff --git a/src/DataTypes/Serializations/SerializationDynamic.cpp b/src/DataTypes/Serializations/SerializationDynamic.cpp index 109f14d49f0..91c8797d43f 100644 --- a/src/DataTypes/Serializations/SerializationDynamic.cpp +++ b/src/DataTypes/Serializations/SerializationDynamic.cpp @@ -26,8 +26,8 @@ namespace ErrorCodes struct SerializeBinaryBulkStateDynamic : public ISerialization::SerializeBinaryBulkState { - SerializationDynamic::DynamicStructureSerializationVersion structure_version; - size_t max_dynamic_types; + SerializationDynamic::DynamicSerializationVersion structure_version; + size_t num_dynamic_types; DataTypePtr variant_type; Names variant_names; SerializationPtr variant_serialization; @@ -81,15 +81,15 @@ void SerializationDynamic::enumerateStreams( settings.path.pop_back(); } -SerializationDynamic::DynamicStructureSerializationVersion::DynamicStructureSerializationVersion(UInt64 version) : value(static_cast(version)) +SerializationDynamic::DynamicSerializationVersion::DynamicSerializationVersion(UInt64 version) : value(static_cast(version)) { checkVersion(version); } -void SerializationDynamic::DynamicStructureSerializationVersion::checkVersion(UInt64 version) +void SerializationDynamic::DynamicSerializationVersion::checkVersion(UInt64 version) { - if (version != VariantTypeName) - throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid version for Dynamic structure serialization."); + if (version != V1 && version != V2) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid version for Dynamic structure serialization: {}", version); } void SerializationDynamic::serializeBinaryBulkStatePrefix( @@ -108,22 +108,17 @@ void SerializationDynamic::serializeBinaryBulkStatePrefix( throw Exception(ErrorCodes::LOGICAL_ERROR, "Missing stream for Dynamic column structure during serialization of binary bulk state prefix"); /// Write structure serialization version. - UInt64 structure_version = DynamicStructureSerializationVersion::Value::VariantTypeName; + UInt64 structure_version = DynamicSerializationVersion::Value::V2; writeBinaryLittleEndian(structure_version, *stream); auto dynamic_state = std::make_shared(structure_version); - dynamic_state->max_dynamic_types = column_dynamic.getMaxDynamicTypes(); - /// Write max_dynamic_types parameter, because it can differ from the max_dynamic_types - /// that is specified in the Dynamic type (we could decrease it before merge). - writeVarUInt(dynamic_state->max_dynamic_types, *stream); - dynamic_state->variant_type = variant_info.variant_type; dynamic_state->variant_names = variant_info.variant_names; const auto & variant_column = column_dynamic.getVariantColumn(); - /// Write information about variants. - size_t num_variants = dynamic_state->variant_names.size() - 1; /// Don't write shared variant, Dynamic column should always have it. - writeVarUInt(num_variants, *stream); + /// Write information about dynamic types. + dynamic_state->num_dynamic_types = dynamic_state->variant_names.size() - 1; /// -1 for SharedVariant + writeVarUInt(dynamic_state->num_dynamic_types, *stream); if (settings.data_types_binary_encoding) { const auto & variants = assert_cast(*dynamic_state->variant_type).getVariants(); @@ -251,22 +246,25 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationDynamic::deserializeD UInt64 structure_version; readBinaryLittleEndian(structure_version, *structure_stream); auto structure_state = std::make_shared(structure_version); - /// Read max_dynamic_types parameter. - readVarUInt(structure_state->max_dynamic_types, *structure_stream); + if (structure_state->structure_version.value == DynamicSerializationVersion::Value::V1) + { + /// Skip max_dynamic_types parameter in V1 serialization version. + size_t max_dynamic_types; + readVarUInt(max_dynamic_types, *structure_stream); + } /// Read information about variants. DataTypes variants; - size_t num_variants; - readVarUInt(num_variants, *structure_stream); - variants.reserve(num_variants + 1); /// +1 for shared variant. + readVarUInt(structure_state->num_dynamic_types, *structure_stream); + variants.reserve(structure_state->num_dynamic_types + 1); /// +1 for shared variant. if (settings.data_types_binary_encoding) { - for (size_t i = 0; i != num_variants; ++i) + for (size_t i = 0; i != structure_state->num_dynamic_types; ++i) variants.push_back(decodeDataType(*structure_stream)); } else { String data_type_name; - for (size_t i = 0; i != num_variants; ++i) + for (size_t i = 0; i != structure_state->num_dynamic_types; ++i) { readStringBinary(data_type_name, *structure_stream); variants.push_back(DataTypeFactory::instance().get(data_type_name)); @@ -364,9 +362,6 @@ void SerializationDynamic::serializeBinaryBulkWithMultipleStreamsAndCountTotalSi if (!variant_info.variant_type->equals(*dynamic_state->variant_type)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Mismatch of internal columns of Dynamic. Expected: {}, Got: {}", dynamic_state->variant_type->getName(), variant_info.variant_type->getName()); - if (column_dynamic.getMaxDynamicTypes() != dynamic_state->max_dynamic_types) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Mismatch of max_dynamic_types parameter of Dynamic. Expected: {}, Got: {}", dynamic_state->max_dynamic_types, column_dynamic.getMaxDynamicTypes()); - settings.path.push_back(Substream::DynamicData); assert_cast(*dynamic_state->variant_serialization) .serializeBinaryBulkWithMultipleStreamsAndUpdateVariantStatistics( @@ -424,7 +419,7 @@ void SerializationDynamic::deserializeBinaryBulkWithMultipleStreams( if (mutable_column->empty()) { - column_dynamic.setMaxDynamicPaths(structure_state->max_dynamic_types); + column_dynamic.setMaxDynamicPaths(structure_state->num_dynamic_types); column_dynamic.setVariantType(structure_state->variant_type); column_dynamic.setStatistics(structure_state->statistics); } diff --git a/src/DataTypes/Serializations/SerializationDynamic.h b/src/DataTypes/Serializations/SerializationDynamic.h index f34b5d0e770..ac98bbbc8b5 100644 --- a/src/DataTypes/Serializations/SerializationDynamic.h +++ b/src/DataTypes/Serializations/SerializationDynamic.h @@ -16,18 +16,28 @@ public: { } - struct DynamicStructureSerializationVersion + struct DynamicSerializationVersion { enum Value { - VariantTypeName = 1, + /// V1 serialization: + /// - DynamicStructure stream: + /// + /// + /// + /// (only in MergeTree serialization) + /// (only in MergeTree serialization) + /// - DynamicData stream: contains the data of nested Variant column. + V1 = 1, + /// V2 serialization: the same as V1 but without max_dynamic_types parameter in DynamicStructure stream. + V2 = 2, }; Value value; static void checkVersion(UInt64 version); - explicit DynamicStructureSerializationVersion(UInt64 version); + explicit DynamicSerializationVersion(UInt64 version); }; void enumerateStreams( @@ -113,9 +123,9 @@ private: struct DeserializeBinaryBulkStateDynamicStructure : public ISerialization::DeserializeBinaryBulkState { - DynamicStructureSerializationVersion structure_version; + DynamicSerializationVersion structure_version; DataTypePtr variant_type; - size_t max_dynamic_types; + size_t num_dynamic_types; ColumnDynamic::StatisticsPtr statistics; explicit DeserializeBinaryBulkStateDynamicStructure(UInt64 structure_version_) diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp index 0fbf8c54a22..1b95fddee9f 100644 --- a/src/DataTypes/Serializations/SerializationObject.cpp +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -25,7 +25,7 @@ SerializationObject::SerializationObject( : typed_path_serializations(std::move(typed_path_serializations_)) , paths_to_skip(paths_to_skip_) , dynamic_serialization(std::make_shared()) - , shared_data_serialization(getTypeOfSharedData()->getDefaultSerialization()) + , shared_data_serialization(DataTypeObject::getTypeOfSharedData()->getDefaultSerialization()) { /// We will need sorted order of typed paths to serialize them in order for consistency. sorted_typed_paths.reserve(typed_path_serializations.size()); @@ -38,13 +38,6 @@ SerializationObject::SerializationObject( path_regexps_to_skip.emplace_back(regexp_str); } -const DataTypePtr & SerializationObject::getTypeOfSharedData() -{ - /// Array(Tuple(String, String)) - static const DataTypePtr type = std::make_shared(std::make_shared(DataTypes{std::make_shared(), std::make_shared()}, Names{"paths", "values"})); - return type; -} - bool SerializationObject::shouldSkipPath(const String & path) const { if (paths_to_skip.contains(path)) @@ -70,14 +63,13 @@ SerializationObject::ObjectSerializationVersion::ObjectSerializationVersion(UInt void SerializationObject::ObjectSerializationVersion::checkVersion(UInt64 version) { - if (version != V1 && version != STRING) + if (version != V1 && version != V2 && version != STRING) throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid version for Object structure serialization."); } struct SerializeBinaryBulkStateObject: public ISerialization::SerializeBinaryBulkState { SerializationObject::ObjectSerializationVersion serialization_version; - size_t max_dynamic_paths; std::vector sorted_dynamic_paths; std::unordered_map typed_path_states; std::unordered_map dynamic_path_states; @@ -168,7 +160,7 @@ void SerializationObject::enumerateStreams(EnumerateStreamsSettings & settings, settings.path.push_back(Substream::ObjectSharedData); auto shared_data_substream_data = SubstreamData(shared_data_serialization) - .withType(getTypeOfSharedData()) + .withType(DataTypeObject::getTypeOfSharedData()) .withColumn(column_object ? column_object->getSharedDataPtr() : nullptr) .withSerializationInfo(data.serialization_info) .withDeserializeState(deserialize_state ? deserialize_state->shared_data_state : nullptr); @@ -195,7 +187,7 @@ void SerializationObject::serializeBinaryBulkStatePrefix( throw Exception(ErrorCodes::LOGICAL_ERROR, "Missing stream for Object column structure during serialization of binary bulk state prefix"); /// Write serialization version. - UInt64 serialization_version = settings.write_json_as_string ? ObjectSerializationVersion::Value::STRING : ObjectSerializationVersion::Value::V1; + UInt64 serialization_version = settings.write_json_as_string ? ObjectSerializationVersion::Value::STRING : ObjectSerializationVersion::Value::V2; writeBinaryLittleEndian(serialization_version, *stream); auto object_state = std::make_shared(serialization_version); @@ -205,9 +197,6 @@ void SerializationObject::serializeBinaryBulkStatePrefix( return; } - object_state->max_dynamic_paths = column_object.getMaxDynamicPaths(); - /// Write max_dynamic_paths parameter. - writeVarUInt(object_state->max_dynamic_paths, *stream); /// Write all dynamic paths in sorted order. object_state->sorted_dynamic_paths.reserve(dynamic_paths.size()); for (const auto & [path, _] : dynamic_paths) @@ -367,10 +356,15 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationObject::deserializeOb UInt64 serialization_version; readBinaryLittleEndian(serialization_version, *structure_stream); auto structure_state = std::make_shared(serialization_version); - if (structure_state->serialization_version.value == ObjectSerializationVersion::Value::V1) + if (structure_state->serialization_version.value == ObjectSerializationVersion::Value::V1 || structure_state->serialization_version.value == ObjectSerializationVersion::Value::V2) { - /// Read max_dynamic_paths parameter. - readVarUInt(structure_state->max_dynamic_paths, *structure_stream); + if (structure_state->serialization_version.value == ObjectSerializationVersion::Value::V1) + { + /// Skip max_dynamic_paths parameter in V1 serialization version. + size_t max_dynamic_paths; + readVarUInt(max_dynamic_paths, *structure_stream); + } + /// Read the sorted list of dynamic paths. size_t dynamic_paths_size; readVarUInt(dynamic_paths_size, *structure_stream); @@ -453,9 +447,6 @@ void SerializationObject::serializeBinaryBulkWithMultipleStreams( const auto & dynamic_paths = column_object.getDynamicPaths(); const auto & shared_data = column_object.getSharedDataPtr(); - if (column_object.getMaxDynamicPaths() != object_state->max_dynamic_paths) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Mismatch of max_dynamic_paths parameter of Object. Expected: {}, Got: {}", object_state->max_dynamic_paths, column_object.getMaxDynamicPaths()); - if (column_object.getDynamicPaths().size() != object_state->sorted_dynamic_paths.size()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Mismatch of number of dynamic paths in Object. Expected: {}, Got: {}", object_state->sorted_dynamic_paths.size(), column_object.getDynamicPaths().size()); @@ -604,7 +595,7 @@ void SerializationObject::deserializeBinaryBulkWithMultipleStreams( /// If it's a new object column, set dynamic paths and statistics. if (column_object.empty()) { - column_object.setMaxDynamicPaths(structure_state->max_dynamic_paths); + column_object.setMaxDynamicPaths(structure_state->sorted_dynamic_paths.size()); column_object.setDynamicPaths(structure_state->sorted_dynamic_paths); column_object.setStatistics(structure_state->statistics); } diff --git a/src/DataTypes/Serializations/SerializationObject.h b/src/DataTypes/Serializations/SerializationObject.h index 420293ba428..db772756a20 100644 --- a/src/DataTypes/Serializations/SerializationObject.h +++ b/src/DataTypes/Serializations/SerializationObject.h @@ -31,6 +31,8 @@ public: /// - ObjectDynamicPath stream for each column in dynamic paths /// - ObjectSharedData stream shared data column. V1 = 0, + /// V2 serialization: the same as V1 but without max_dynamic_paths parameter in ObjectStructure stream. + V2 = 2, /// String serialization: /// - ObjectData stream with single String column containing serialized JSON. STRING = 1, @@ -98,7 +100,6 @@ private: struct DeserializeBinaryBulkStateObjectStructure : public ISerialization::DeserializeBinaryBulkState { ObjectSerializationVersion serialization_version; - size_t max_dynamic_paths; std::vector sorted_dynamic_paths; std::unordered_set dynamic_paths; /// Paths statistics. Map (dynamic path) -> (number of non-null values in this path). @@ -111,9 +112,6 @@ private: DeserializeBinaryBulkSettings & settings, SubstreamsDeserializeStatesCache * cache); - /// Shared data has type Array(Tuple(String, String)). - static const DataTypePtr & getTypeOfSharedData(); - struct TypedPathSubcolumnCreator : public ISubcolumnCreator { String path; diff --git a/src/DataTypes/Serializations/SerializationObjectDynamicPath.cpp b/src/DataTypes/Serializations/SerializationObjectDynamicPath.cpp index 5323079c54b..c1f26eca792 100644 --- a/src/DataTypes/Serializations/SerializationObjectDynamicPath.cpp +++ b/src/DataTypes/Serializations/SerializationObjectDynamicPath.cpp @@ -18,7 +18,7 @@ SerializationObjectDynamicPath::SerializationObjectDynamicPath( , path(path_) , path_subcolumn(path_subcolumn_) , dynamic_serialization(std::make_shared()) - , shared_data_serialization(SerializationObject::getTypeOfSharedData()->getDefaultSerialization()) + , shared_data_serialization(DataTypeObject::getTypeOfSharedData()->getDefaultSerialization()) , max_dynamic_types(max_dynamic_types_) { } @@ -67,8 +67,8 @@ void SerializationObjectDynamicPath::enumerateStreams( { settings.path.push_back(Substream::ObjectSharedData); auto shared_data_substream_data = SubstreamData(shared_data_serialization) - .withType(data.type ? SerializationObject::getTypeOfSharedData() : nullptr) - .withColumn(data.column ? SerializationObject::getTypeOfSharedData()->createColumn() : nullptr) + .withType(data.type ? DataTypeObject::getTypeOfSharedData() : nullptr) + .withColumn(data.column ? DataTypeObject::getTypeOfSharedData()->createColumn() : nullptr) .withSerializationInfo(data.serialization_info) .withDeserializeState(deserialize_state->nested_state); settings.path.back().data = shared_data_substream_data; @@ -164,7 +164,7 @@ void SerializationObjectDynamicPath::deserializeBinaryBulkWithMultipleStreams( settings.path.push_back(Substream::ObjectSharedData); /// Initialize shared_data column if needed. if (result_column->empty()) - dynamic_path_state->shared_data = SerializationObject::getTypeOfSharedData()->createColumn(); + dynamic_path_state->shared_data = DataTypeObject::getTypeOfSharedData()->createColumn(); size_t prev_size = result_column->size(); shared_data_serialization->deserializeBinaryBulkWithMultipleStreams(dynamic_path_state->shared_data, limit, settings, dynamic_path_state->nested_state, cache); /// If we need to read a subcolumn from Dynamic column, create an empty Dynamic column, fill it and extract subcolumn. diff --git a/src/DataTypes/Serializations/SerializationSubObject.cpp b/src/DataTypes/Serializations/SerializationSubObject.cpp index 9084d46f9b2..ff61cb55572 100644 --- a/src/DataTypes/Serializations/SerializationSubObject.cpp +++ b/src/DataTypes/Serializations/SerializationSubObject.cpp @@ -17,7 +17,7 @@ SerializationSubObject::SerializationSubObject( : path_prefix(path_prefix_) , typed_paths_serializations(typed_paths_serializations_) , dynamic_serialization(std::make_shared()) - , shared_data_serialization(SerializationObject::getTypeOfSharedData()->getDefaultSerialization()) + , shared_data_serialization(DataTypeObject::getTypeOfSharedData()->getDefaultSerialization()) { } @@ -64,8 +64,8 @@ void SerializationSubObject::enumerateStreams( /// We will need to read shared data to find all paths with requested prefix. settings.path.push_back(Substream::ObjectSharedData); auto shared_data_substream_data = SubstreamData(shared_data_serialization) - .withType(data.type ? SerializationObject::getTypeOfSharedData() : nullptr) - .withColumn(data.column ? SerializationObject::getTypeOfSharedData()->createColumn() : nullptr) + .withType(data.type ? DataTypeObject::getTypeOfSharedData() : nullptr) + .withColumn(data.column ? DataTypeObject::getTypeOfSharedData()->createColumn() : nullptr) .withSerializationInfo(data.serialization_info) .withDeserializeState(deserialize_state ? deserialize_state->shared_data_state : nullptr); settings.path.back().data = shared_data_substream_data; @@ -208,7 +208,7 @@ void SerializationSubObject::deserializeBinaryBulkWithMultipleStreams( settings.path.push_back(Substream::ObjectSharedData); /// If it's a new object column, reinitialize column for shared data. if (result_column->empty()) - sub_object_state->shared_data = SerializationObject::getTypeOfSharedData()->createColumn(); + sub_object_state->shared_data = DataTypeObject::getTypeOfSharedData()->createColumn(); size_t prev_size = column_object.size(); shared_data_serialization->deserializeBinaryBulkWithMultipleStreams(sub_object_state->shared_data, limit, settings, sub_object_state->shared_data_state, cache); settings.path.pop_back(); diff --git a/src/DataTypes/fuzzers/CMakeLists.txt b/src/DataTypes/fuzzers/CMakeLists.txt index 9e5b1b3f673..8940586fc70 100644 --- a/src/DataTypes/fuzzers/CMakeLists.txt +++ b/src/DataTypes/fuzzers/CMakeLists.txt @@ -1,2 +1,3 @@ clickhouse_add_executable(data_type_deserialization_fuzzer data_type_deserialization_fuzzer.cpp ${SRCS}) -target_link_libraries(data_type_deserialization_fuzzer PRIVATE clickhouse_aggregate_functions) + +target_link_libraries(data_type_deserialization_fuzzer PRIVATE clickhouse_aggregate_functions dbms) diff --git a/src/DataTypes/fuzzers/data_type_deserialization_fuzzer.cpp b/src/DataTypes/fuzzers/data_type_deserialization_fuzzer.cpp index f9a733647e1..216b252ad0f 100644 --- a/src/DataTypes/fuzzers/data_type_deserialization_fuzzer.cpp +++ b/src/DataTypes/fuzzers/data_type_deserialization_fuzzer.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index 13230fdf545..bd077ccd7b5 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -19,6 +18,7 @@ #include #include + namespace fs = std::filesystem; namespace DB @@ -60,9 +60,6 @@ DatabaseAtomic::DatabaseAtomic(String name_, String metadata_path_, UUID uuid, c , db_uuid(uuid) { assert(db_uuid != UUIDHelpers::Nil); - fs::create_directories(fs::path(getContext()->getPath()) / "metadata"); - fs::create_directories(path_to_table_symlinks); - tryCreateMetadataSymlink(); } DatabaseAtomic::DatabaseAtomic(String name_, String metadata_path_, UUID uuid, ContextPtr context_) @@ -70,6 +67,20 @@ DatabaseAtomic::DatabaseAtomic(String name_, String metadata_path_, UUID uuid, C { } +void DatabaseAtomic::createDirectories() +{ + std::lock_guard lock(mutex); + createDirectoriesUnlocked(); +} + +void DatabaseAtomic::createDirectoriesUnlocked() +{ + DatabaseOnDisk::createDirectoriesUnlocked(); + fs::create_directories(fs::path(getContext()->getPath()) / "metadata"); + fs::create_directories(path_to_table_symlinks); + tryCreateMetadataSymlink(); +} + String DatabaseAtomic::getTableDataPath(const String & table_name) const { std::lock_guard lock(mutex); @@ -108,6 +119,7 @@ void DatabaseAtomic::attachTable(ContextPtr /* context_ */, const String & name, assert(relative_table_path != data_path && !relative_table_path.empty()); DetachedTables not_in_use; std::lock_guard lock(mutex); + createDirectoriesUnlocked(); not_in_use = cleanupDetachedTables(); auto table_id = table->getStorageID(); assertDetachedTableNotInUse(table_id.uuid); @@ -208,11 +220,15 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_ if (exchange && !supportsAtomicRename(&message)) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "RENAME EXCHANGE is not supported because exchanging files is not supported by the OS ({})", message); + createDirectories(); waitDatabaseStarted(); auto & other_db = dynamic_cast(to_database); bool inside_database = this == &other_db; + if (!inside_database) + other_db.createDirectories(); + String old_metadata_path = getObjectMetadataPath(table_name); String new_metadata_path = to_database.getObjectMetadataPath(to_table_name); @@ -333,6 +349,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora const String & table_metadata_tmp_path, const String & table_metadata_path, ContextPtr query_context) { + createDirectories(); DetachedTables not_in_use; auto table_data_path = getTableDataPath(query); try @@ -469,6 +486,9 @@ void DatabaseAtomic::beforeLoadingMetadata(ContextMutablePtr /*context*/, Loadin if (mode < LoadingStrictnessLevel::FORCE_RESTORE) return; + if (!fs::exists(path_to_table_symlinks)) + return; + /// Recreate symlinks to table data dirs in case of force restore, because some of them may be broken for (const auto & table_path : fs::directory_iterator(path_to_table_symlinks)) { @@ -611,6 +631,7 @@ void DatabaseAtomic::renameDatabase(ContextPtr query_context, const String & new { /// CREATE, ATTACH, DROP, DETACH and RENAME DATABASE must hold DDLGuard + createDirectories(); waitDatabaseStarted(); bool check_ref_deps = query_context->getSettingsRef()[Setting::check_referential_table_dependencies]; @@ -702,4 +723,5 @@ void registerDatabaseAtomic(DatabaseFactory & factory) }; factory.registerDatabase("Atomic", create_fn); } + } diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h index 07a1f9c7fa1..7e909128635 100644 --- a/src/Databases/DatabaseAtomic.h +++ b/src/Databases/DatabaseAtomic.h @@ -76,6 +76,9 @@ protected: using DetachedTables = std::unordered_map; [[nodiscard]] DetachedTables cleanupDetachedTables() TSA_REQUIRES(mutex); + void createDirectories(); + void createDirectoriesUnlocked() TSA_REQUIRES(mutex); + void tryCreateMetadataSymlink(); virtual bool allowMoveTableToOtherDatabaseEngine(IDatabase & /*to_database*/) const { return false; } diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index 0a4b02c4917..d63dea20120 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -47,6 +47,7 @@ DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_, : DatabaseOnDisk(name_, metadata_path_, std::filesystem::path("data") / escapeForFileName(name_) / "", "DatabaseLazy (" + name_ + ")", context_) , expiration_time(expiration_time_) { + createDirectories(); } diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index f914d9024e1..93ecf9cf11c 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -180,7 +180,18 @@ DatabaseOnDisk::DatabaseOnDisk( , metadata_path(metadata_path_) , data_path(data_path_) { - fs::create_directories(local_context->getPath() + data_path); +} + + +void DatabaseOnDisk::createDirectories() +{ + std::lock_guard lock(mutex); + createDirectoriesUnlocked(); +} + +void DatabaseOnDisk::createDirectoriesUnlocked() +{ + fs::create_directories(std::filesystem::path(getContext()->getPath()) / data_path); fs::create_directories(metadata_path); } @@ -198,6 +209,8 @@ void DatabaseOnDisk::createTable( const StoragePtr & table, const ASTPtr & query) { + createDirectories(); + const auto & settings = local_context->getSettingsRef(); const auto & create = query->as(); assert(table_name == create.getTable()); @@ -265,7 +278,6 @@ void DatabaseOnDisk::createTable( } commitCreateTable(create, table, table_metadata_tmp_path, table_metadata_path, local_context); - removeDetachedPermanentlyFlag(local_context, table_name, table_metadata_path, false); } @@ -293,6 +305,8 @@ void DatabaseOnDisk::commitCreateTable(const ASTCreateQuery & query, const Stora { try { + createDirectories(); + /// Add a table to the map of known tables. attachTable(query_context, query.getTable(), table, getTableDataPath(query)); @@ -426,6 +440,7 @@ void DatabaseOnDisk::renameTable( throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Moving tables between databases of different engines is not supported"); } + createDirectories(); waitDatabaseStarted(); auto table_data_relative_path = getTableDataPath(table_name); @@ -621,6 +636,9 @@ time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & object_n void DatabaseOnDisk::iterateMetadataFiles(const IteratingFunction & process_metadata_file) const { + if (!fs::exists(metadata_path)) + return; + auto process_tmp_drop_metadata_file = [&](const String & file_name) { assert(getUUID() == UUIDHelpers::Nil); diff --git a/src/Databases/DatabaseOnDisk.h b/src/Databases/DatabaseOnDisk.h index ffc95a7c128..1e11d21cc87 100644 --- a/src/Databases/DatabaseOnDisk.h +++ b/src/Databases/DatabaseOnDisk.h @@ -99,6 +99,9 @@ protected: virtual void removeDetachedPermanentlyFlag(ContextPtr context, const String & table_name, const String & table_metadata_path, bool attach); virtual void setDetachedTableNotInUseForce(const UUID & /*uuid*/) {} + void createDirectories(); + void createDirectoriesUnlocked() TSA_REQUIRES(mutex); + const String metadata_path; const String data_path; }; diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index f94326d220e..02418abb2b0 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -416,6 +416,7 @@ public: std::lock_guard lock{mutex}; return database_name; } + /// Get UUID of database. virtual UUID getUUID() const { return UUIDHelpers::Nil; } diff --git a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp index e097ffab0d7..3d5fcc153fd 100644 --- a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp @@ -62,6 +62,7 @@ DatabaseMaterializedMySQL::DatabaseMaterializedMySQL( , settings(std::move(settings_)) , materialize_thread(context_, database_name_, mysql_database_name_, std::move(pool_), std::move(client_), binlog_client_, settings.get()) { + createDirectories(); } DatabaseMaterializedMySQL::~DatabaseMaterializedMySQL() = default; diff --git a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp index 45fd52f27ab..5268dbcb59f 100644 --- a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp +++ b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp @@ -307,6 +307,13 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure( if (!columns.empty()) columns_part = fmt::format(" AND attname IN ('{}')", boost::algorithm::join(columns, "','")); + /// Bypassing the error of the missing column `attgenerated` in the system table `pg_attribute` for PostgreSQL versions below 12. + /// This trick involves executing a special query to the DBMS in advance to obtain the correct line with comment /// if column has GENERATED. + /// The result of the query will be the name of the column `attgenerated` or an empty string declaration for PostgreSQL version 11 and below. + /// This change does not degrade the function's performance but restores support for older versions and fix ERROR: column "attgenerated" does not exist. + pqxx::result gen_result{tx.exec("select case when current_setting('server_version_num')::int < 120000 then '''''' else 'attgenerated' end as generated")}; + std::string generated = gen_result[0][0].as(); + std::string query = fmt::format( "SELECT attname AS name, " /// column name "format_type(atttypid, atttypmod) AS type, " /// data type @@ -315,11 +322,11 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure( "atttypid as type_id, " "atttypmod as type_modifier, " "attnum as att_num, " - "attgenerated as generated " /// if column has GENERATED + "{} as generated " /// if column has GENERATED "FROM pg_attribute " "WHERE attrelid = (SELECT oid FROM pg_class WHERE {}) {}" "AND NOT attisdropped AND attnum > 0 " - "ORDER BY attnum ASC", where, columns_part); + "ORDER BY attnum ASC", generated, where, columns_part); /// Now we use variable `generated` to form query string. End of trick. auto postgres_table_with_schema = postgres_schema.empty() ? postgres_table : doubleQuoteString(postgres_schema) + '.' + doubleQuoteString(postgres_table); table.physical_columns = readNamesAndTypesList(tx, postgres_table_with_schema, query, use_nulls, false); diff --git a/src/Databases/enableAllExperimentalSettings.cpp b/src/Databases/enableAllExperimentalSettings.cpp index 6efbc429fd8..d51d2671992 100644 --- a/src/Databases/enableAllExperimentalSettings.cpp +++ b/src/Databases/enableAllExperimentalSettings.cpp @@ -32,6 +32,8 @@ void enableAllExperimentalSettings(ContextMutablePtr context) context->setSetting("allow_suspicious_low_cardinality_types", 1); context->setSetting("allow_suspicious_fixed_string_types", 1); + context->setSetting("allow_suspicious_types_in_group_by", 1); + context->setSetting("allow_suspicious_types_in_order_by", 1); context->setSetting("allow_suspicious_indices", 1); context->setSetting("allow_suspicious_codecs", 1); context->setSetting("allow_hyperscan", 1); diff --git a/src/Dictionaries/HashedDictionary.h b/src/Dictionaries/HashedDictionary.h index ef0b5b0ef2f..bb620127daf 100644 --- a/src/Dictionaries/HashedDictionary.h +++ b/src/Dictionaries/HashedDictionary.h @@ -334,22 +334,26 @@ HashedDictionary::~HashedDictionary() if (container.empty()) return; - pool.trySchedule([&container, thread_group = CurrentThread::getGroup()] - { - SCOPE_EXIT_SAFE( + if (!pool.trySchedule([&container, thread_group = CurrentThread::getGroup()] + { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachFromGroupIfNotDetached(); + ); + + /// Do not account memory that was occupied by the dictionaries for the query/user context. + MemoryTrackerBlockerInThread memory_blocker; + if (thread_group) - CurrentThread::detachFromGroupIfNotDetached(); - ); + CurrentThread::attachToGroupIfDetached(thread_group); + setThreadName("HashedDictDtor"); - /// Do not account memory that was occupied by the dictionaries for the query/user context. + clearContainer(container); + })) + { MemoryTrackerBlockerInThread memory_blocker; - - if (thread_group) - CurrentThread::attachToGroupIfDetached(thread_group); - setThreadName("HashedDictDtor"); - clearContainer(container); - }); + } ++hash_tables_count; }; diff --git a/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp b/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp index c405d296e60..01271d5342b 100644 --- a/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp +++ b/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp @@ -46,11 +46,13 @@ AsynchronousBoundedReadBuffer::AsynchronousBoundedReadBuffer( ImplPtr impl_, IAsynchronousReader & reader_, const ReadSettings & settings_, + size_t buffer_size_, AsyncReadCountersPtr async_read_counters_, FilesystemReadPrefetchesLogPtr prefetches_log_) : ReadBufferFromFileBase(0, nullptr, 0) , impl(std::move(impl_)) , read_settings(settings_) + , buffer_size(buffer_size_) , reader(reader_) , query_id(CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() != nullptr ? CurrentThread::getQueryId() : "") , current_reader_id(getRandomASCIIString(8)) @@ -112,7 +114,7 @@ void AsynchronousBoundedReadBuffer::prefetch(Priority priority) last_prefetch_info.submit_time = std::chrono::system_clock::now(); last_prefetch_info.priority = priority; - prefetch_buffer.resize(chooseBufferSizeForRemoteReading(read_settings, impl->getFileSize())); + prefetch_buffer.resize(buffer_size); prefetch_future = readAsync(prefetch_buffer.data(), prefetch_buffer.size(), priority); ProfileEvents::increment(ProfileEvents::RemoteFSPrefetches); } @@ -211,7 +213,7 @@ bool AsynchronousBoundedReadBuffer::nextImpl() } else { - memory.resize(chooseBufferSizeForRemoteReading(read_settings, impl->getFileSize())); + memory.resize(buffer_size); { ProfileEventTimeIncrement watch(ProfileEvents::SynchronousRemoteReadWaitMicroseconds); diff --git a/src/Disks/IO/AsynchronousBoundedReadBuffer.h b/src/Disks/IO/AsynchronousBoundedReadBuffer.h index 3dc8fcc39cb..7664cc4d386 100644 --- a/src/Disks/IO/AsynchronousBoundedReadBuffer.h +++ b/src/Disks/IO/AsynchronousBoundedReadBuffer.h @@ -27,6 +27,7 @@ public: ImplPtr impl_, IAsynchronousReader & reader_, const ReadSettings & settings_, + size_t buffer_size_, AsyncReadCountersPtr async_read_counters_ = nullptr, FilesystemReadPrefetchesLogPtr prefetches_log_ = nullptr); @@ -53,6 +54,7 @@ public: private: const ImplPtr impl; const ReadSettings read_settings; + const size_t buffer_size; IAsynchronousReader & reader; size_t file_offset_of_buffer_end = 0; diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 51c6045cb68..1f806e9c1e5 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -535,7 +535,7 @@ bool CachedOnDiskReadBufferFromFile::completeFileSegmentAndGetNext() chassert(file_offset_of_buffer_end > completed_range.right); cache_file_reader.reset(); - file_segments->popFront(); + file_segments->completeAndPopFront(settings.filesystem_cache_allow_background_download); if (file_segments->empty() && !nextFileSegmentsBatch()) return false; @@ -556,6 +556,12 @@ CachedOnDiskReadBufferFromFile::~CachedOnDiskReadBufferFromFile() { appendFilesystemCacheLog(file_segments->front(), read_type); } + + if (file_segments && !file_segments->empty() && !file_segments->front().isCompleted()) + { + file_segments->completeAndPopFront(settings.filesystem_cache_allow_background_download); + file_segments = {}; + } } void CachedOnDiskReadBufferFromFile::predownload(FileSegment & file_segment) @@ -784,6 +790,7 @@ bool CachedOnDiskReadBufferFromFile::writeCache(char * data, size_t size, size_t LOG_INFO(log, "Insert into cache is skipped due to insufficient disk space. ({})", e.displayText()); return false; } + chassert(file_segment.state() == FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); throw; } diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h index 119fa166214..4881b6a309d 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h @@ -41,6 +41,8 @@ public: ~CachedOnDiskReadBufferFromFile() override; + bool isCached() const override { return true; } + bool nextImpl() override; off_t seek(off_t off, int whence) override; diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp index 6aedc1f5d04..df6fb871772 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp @@ -196,7 +196,7 @@ void FileSegmentRangeWriter::completeFileSegment() if (file_segment.isDetached() || file_segment.isCompleted()) return; - file_segment.complete(); + file_segment.complete(false); appendFilesystemCacheLog(file_segment); } @@ -210,7 +210,7 @@ void FileSegmentRangeWriter::jumpToPosition(size_t position) if (position < current_write_offset) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot jump backwards: {} < {}", position, current_write_offset); - file_segment.complete(); + file_segment.complete(false); file_segments.reset(); } expected_write_offset = position; diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 8e4ec6f3dfb..8d3b9366261 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -18,24 +18,14 @@ namespace ErrorCodes extern const int CANNOT_SEEK_THROUGH_FILE; } -size_t chooseBufferSizeForRemoteReading(const DB::ReadSettings & settings, size_t file_size) -{ - /// Only when cache is used we could download bigger portions of FileSegments than what we actually gonna read within particular task. - if (!settings.enable_filesystem_cache && !settings.read_through_distributed_cache) - return settings.remote_fs_buffer_size; - - /// Buffers used for prefetch and pre-download better to have enough size, but not bigger than the whole file. - return std::min(std::max(settings.remote_fs_buffer_size, DBMS_DEFAULT_BUFFER_SIZE), file_size); -} - ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather( ReadBufferCreator && read_buffer_creator_, const StoredObjects & blobs_to_read_, const ReadSettings & settings_, std::shared_ptr cache_log_, - bool use_external_buffer_) - : ReadBufferFromFileBase(use_external_buffer_ ? 0 : chooseBufferSizeForRemoteReading( - settings_, getTotalSize(blobs_to_read_)), nullptr, 0) + bool use_external_buffer_, + size_t buffer_size) + : ReadBufferFromFileBase(use_external_buffer_ ? 0 : buffer_size, nullptr, 0) , settings(settings_) , blobs_to_read(blobs_to_read_) , read_buffer_creator(std::move(read_buffer_creator_)) diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h index 27f94a3e552..c5f1966dc38 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h @@ -28,7 +28,8 @@ public: const StoredObjects & blobs_to_read_, const ReadSettings & settings_, std::shared_ptr cache_log_, - bool use_external_buffer_); + bool use_external_buffer_, + size_t buffer_size); ~ReadBufferFromRemoteFSGather() override; @@ -84,6 +85,4 @@ private: LoggerPtr log; }; - -size_t chooseBufferSizeForRemoteReading(const DB::ReadSettings & settings, size_t file_size); } diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index cc8a873c544..fba45d5a0c9 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -641,19 +641,36 @@ std::unique_ptr DiskObjectStorage::readFile( return impl; }; + /// Avoid cache fragmentation by choosing bigger buffer size. + bool prefer_bigger_buffer_size = read_settings.filesystem_cache_prefer_bigger_buffer_size + && object_storage->supportsCache() + && read_settings.enable_filesystem_cache; + + size_t buffer_size = prefer_bigger_buffer_size + ? std::max(settings.remote_fs_buffer_size, DBMS_DEFAULT_BUFFER_SIZE) + : settings.remote_fs_buffer_size; + + size_t total_objects_size = file_size ? *file_size : getTotalSize(storage_objects); + if (total_objects_size) + buffer_size = std::min(buffer_size, total_objects_size); + const bool use_async_buffer = read_settings.remote_fs_method == RemoteFSReadMethod::threadpool; auto impl = std::make_unique( std::move(read_buffer_creator), storage_objects, read_settings, global_context->getFilesystemCacheLog(), - /* use_external_buffer */use_async_buffer); + /* use_external_buffer */use_async_buffer, + /* buffer_size */use_async_buffer ? 0 : buffer_size); if (use_async_buffer) { auto & reader = global_context->getThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER); return std::make_unique( - std::move(impl), reader, read_settings, + std::move(impl), + reader, + read_settings, + buffer_size, global_context->getAsyncReadCounters(), global_context->getFilesystemReadPrefetchesLog()); diff --git a/src/Disks/tests/gtest_asynchronous_bounded_read_buffer.cpp b/src/Disks/tests/gtest_asynchronous_bounded_read_buffer.cpp index 63a39fe39c7..11b4fc3118d 100644 --- a/src/Disks/tests/gtest_asynchronous_bounded_read_buffer.cpp +++ b/src/Disks/tests/gtest_asynchronous_bounded_read_buffer.cpp @@ -51,7 +51,7 @@ TEST_F(AsynchronousBoundedReadBufferTest, setReadUntilPosition) for (bool with_prefetch : {false, true}) { - AsynchronousBoundedReadBuffer read_buffer(createReadBufferFromFileBase(file_path, {}), remote_fs_reader, {}); + AsynchronousBoundedReadBuffer read_buffer(createReadBufferFromFileBase(file_path, {}), remote_fs_reader, {}, DBMS_DEFAULT_BUFFER_SIZE); read_buffer.setReadUntilPosition(20); auto try_read = [&](size_t count) diff --git a/src/Formats/PrettyFormatHelpers.cpp b/src/Formats/PrettyFormatHelpers.cpp new file mode 100644 index 00000000000..4ee4b49521d --- /dev/null +++ b/src/Formats/PrettyFormatHelpers.cpp @@ -0,0 +1,107 @@ +#include +#include +#include +#include +#include + + +static constexpr const char * GRAY_COLOR = "\033[90m"; +static constexpr const char * UNDERSCORE = "\033[4m"; +static constexpr const char * RESET_COLOR = "\033[0m"; + + +namespace DB +{ + +void writeReadableNumberTipIfSingleValue(WriteBuffer & out, const Chunk & chunk, const FormatSettings & settings, bool color) +{ + if (chunk.getNumRows() == 1 && chunk.getNumColumns() == 1) + writeReadableNumberTip(out, *chunk.getColumns()[0], 0, settings, color); +} + +void writeReadableNumberTip(WriteBuffer & out, const IColumn & column, size_t row, const FormatSettings & settings, bool color) +{ + if (column.isNullAt(row)) + return; + + auto value = column.getFloat64(row); + auto threshold = settings.pretty.output_format_pretty_single_large_number_tip_threshold; + + if (threshold && isFinite(value) && abs(value) > threshold) + { + if (color) + writeCString(GRAY_COLOR, out); + writeCString(" -- ", out); + formatReadableQuantity(value, out, 2); + if (color) + writeCString(RESET_COLOR, out); + } +} + + +String highlightDigitGroups(String source) +{ + if (source.size() <= 4) + return source; + + bool is_regular_number = true; + size_t num_digits_before_decimal = 0; + for (auto c : source) + { + if (c == '-' || c == ' ') + continue; + if (c == '.') + break; + if (c >= '0' && c <= '9') + { + ++num_digits_before_decimal; + } + else + { + is_regular_number = false; + break; + } + } + + if (!is_regular_number || num_digits_before_decimal <= 4) + return source; + + String result; + size_t size = source.size(); + result.reserve(2 * size); + + bool before_decimal = true; + size_t digit_num = 0; + for (size_t i = 0; i < size; ++i) + { + auto c = source[i]; + if (before_decimal && c >= '0' && c <= '9') + { + ++digit_num; + size_t offset = num_digits_before_decimal - digit_num; + if (offset && offset % 3 == 0) + { + result += UNDERSCORE; + result += c; + result += RESET_COLOR; + } + else + { + result += c; + } + } + else if (c == '.') + { + before_decimal = false; + result += c; + } + else + { + result += c; + } + } + + return result; +} + +} diff --git a/src/Formats/PrettyFormatHelpers.h b/src/Formats/PrettyFormatHelpers.h new file mode 100644 index 00000000000..b5d679c5a42 --- /dev/null +++ b/src/Formats/PrettyFormatHelpers.h @@ -0,0 +1,21 @@ +#pragma once + +#include + + +namespace DB +{ + +class Chunk; +class IColumn; +class WriteBuffer; +struct FormatSettings; + +/// Prints text describing the number in the form of: -- 12.34 million +void writeReadableNumberTip(WriteBuffer & out, const IColumn & column, size_t row, const FormatSettings & settings, bool color); +void writeReadableNumberTipIfSingleValue(WriteBuffer & out, const Chunk & chunk, const FormatSettings & settings, bool color); + +/// Underscores digit groups related to thousands using terminal ANSI escape sequences. +String highlightDigitGroups(String source); + +} diff --git a/src/Formats/fuzzers/CMakeLists.txt b/src/Formats/fuzzers/CMakeLists.txt index ee1a4fd4358..83aa5eb781a 100644 --- a/src/Formats/fuzzers/CMakeLists.txt +++ b/src/Formats/fuzzers/CMakeLists.txt @@ -1,2 +1,2 @@ clickhouse_add_executable(format_fuzzer format_fuzzer.cpp ${SRCS}) -target_link_libraries(format_fuzzer PRIVATE clickhouse_aggregate_functions) +target_link_libraries(format_fuzzer PRIVATE clickhouse_aggregate_functions dbms) diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 0f6311c9716..1e7cb8f1749 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -3921,7 +3921,7 @@ private: } } - WrapperType createTupleToObjectWrapper(const DataTypeTuple & from_tuple, bool has_nullable_subcolumns) const + WrapperType createTupleToObjectDeprecatedWrapper(const DataTypeTuple & from_tuple, bool has_nullable_subcolumns) const { if (!from_tuple.haveExplicitNames()) throw Exception(ErrorCodes::TYPE_MISMATCH, @@ -3968,7 +3968,7 @@ private: }; } - WrapperType createMapToObjectWrapper(const DataTypeMap & from_map, bool has_nullable_subcolumns) const + WrapperType createMapToObjectDeprecatedWrapper(const DataTypeMap & from_map, bool has_nullable_subcolumns) const { auto key_value_types = from_map.getKeyValueTypes(); @@ -4048,11 +4048,11 @@ private: { if (const auto * from_tuple = checkAndGetDataType(from_type.get())) { - return createTupleToObjectWrapper(*from_tuple, to_type->hasNullableSubcolumns()); + return createTupleToObjectDeprecatedWrapper(*from_tuple, to_type->hasNullableSubcolumns()); } else if (const auto * from_map = checkAndGetDataType(from_type.get())) { - return createMapToObjectWrapper(*from_map, to_type->hasNullableSubcolumns()); + return createMapToObjectDeprecatedWrapper(*from_map, to_type->hasNullableSubcolumns()); } else if (checkAndGetDataType(from_type.get())) { @@ -4081,23 +4081,43 @@ private: "Cast to Object can be performed only from flatten named Tuple, Map or String. Got: {}", from_type->getName()); } + WrapperType createObjectWrapper(const DataTypePtr & from_type, const DataTypeObject * to_object) const { if (checkAndGetDataType(from_type.get())) { return [this](ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable * nullable_source, size_t input_rows_count) { - auto res = ConvertImplGenericFromString::execute(arguments, result_type, nullable_source, input_rows_count, context)->assumeMutable(); - res->finalize(); - return res; + return ConvertImplGenericFromString::execute(arguments, result_type, nullable_source, input_rows_count, context); + }; + } + + /// Cast Tuple/Object/Map to JSON type through serializing into JSON string and parsing back into JSON column. + /// Potentially we can do smarter conversion Tuple -> JSON with type preservation, but it's questionable how exactly Tuple should be + /// converted to JSON (for example, should we recursively convert nested Array(Tuple) to Array(JSON) or not, should we infer types from String fields, etc). + if (checkAndGetDataType(from_type.get()) || checkAndGetDataType(from_type.get()) || checkAndGetDataType(from_type.get())) + { + return [this](ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable * nullable_source, size_t input_rows_count) + { + auto json_string = ColumnString::create(); + ColumnStringHelpers::WriteHelper write_helper(assert_cast(*json_string), input_rows_count); + auto & write_buffer = write_helper.getWriteBuffer(); + FormatSettings format_settings = context ? getFormatSettings(context) : FormatSettings{}; + auto serialization = arguments[0].type->getDefaultSerialization(); + for (size_t i = 0; i < input_rows_count; ++i) + { + serialization->serializeTextJSON(*arguments[0].column, i, write_buffer, format_settings); + write_helper.rowWritten(); + } + write_helper.finalize(); + + ColumnsWithTypeAndName args_with_json_string = {ColumnWithTypeAndName(json_string->getPtr(), std::make_shared(), "")}; + return ConvertImplGenericFromString::execute(args_with_json_string, result_type, nullable_source, input_rows_count, context); }; } /// TODO: support CAST between JSON types with different parameters - /// support CAST from Map to JSON - /// support CAST from Tuple to JSON - /// support CAST from Object('json') to JSON - throw Exception(ErrorCodes::TYPE_MISMATCH, "Cast to {} can be performed only from String. Got: {}", magic_enum::enum_name(to_object->getSchemaFormat()), from_type->getName()); + throw Exception(ErrorCodes::TYPE_MISMATCH, "Cast to {} can be performed only from String/Map/Object/Tuple. Got: {}", magic_enum::enum_name(to_object->getSchemaFormat()), from_type->getName()); } WrapperType createVariantToVariantWrapper(const DataTypeVariant & from_variant, const DataTypeVariant & to_variant) const @@ -4390,7 +4410,7 @@ private: variant_column = IColumn::mutate(column); /// Otherwise we should filter column. else - variant_column = column->filter(filter, variant_size_hint)->assumeMutable(); + variant_column = IColumn::mutate(column->filter(filter, variant_size_hint)); assert_cast(*variant_column).nestedRemoveNullable(); return createVariantFromDescriptorsAndOneNonEmptyVariant(variant_types, std::move(discriminators), std::move(variant_column), variant_discr); diff --git a/src/Functions/FunctionsMiscellaneous.h b/src/Functions/FunctionsMiscellaneous.h index fb5109eaa88..cea11cfe677 100644 --- a/src/Functions/FunctionsMiscellaneous.h +++ b/src/Functions/FunctionsMiscellaneous.h @@ -6,6 +6,7 @@ #include #include #include +#include #include @@ -112,6 +113,7 @@ public: NamesAndTypesList lambda_arguments; String return_name; DataTypePtr return_type; + bool allow_constant_folding; }; using CapturePtr = std::shared_ptr; @@ -122,6 +124,7 @@ public: String getName() const override { return "FunctionCapture"; } bool useDefaultImplementationForNulls() const override { return false; } + /// It's possible if expression_actions contains function that don't use /// default implementation for Nothing and one of captured columns can be Nothing /// Example: SELECT arrayMap(x -> [x, arrayElement(y, 0)], []), [] as y @@ -148,7 +151,26 @@ public: auto function = std::make_unique(expression_actions, types, names, capture->return_type, capture->return_name); - return ColumnFunction::create(input_rows_count, std::move(function), arguments); + /// If all the captured arguments are constant, let's also return ColumnConst (with ColumnFunction inside it). + /// Consequently, it allows to treat higher order functions with constant arrays and constant captured columns + /// as constant expressions. + /// Consequently, it allows its usage in contexts requiring constants, such as the right hand side of IN. + bool constant_folding = capture->allow_constant_folding + && std::all_of(arguments.begin(), arguments.end(), + [](const auto & arg) { return arg.column->isConst(); }); + + if (constant_folding) + { + ColumnsWithTypeAndName arguments_resized = arguments; + for (auto & elem : arguments_resized) + elem.column = elem.column->cloneResized(1); + + return ColumnConst::create(ColumnFunction::create(1, std::move(function), arguments_resized), input_rows_count); + } + else + { + return ColumnFunction::create(input_rows_count, std::move(function), arguments); + } } private: @@ -203,7 +225,8 @@ public: const Names & captured_names, const NamesAndTypesList & lambda_arguments, const DataTypePtr & function_return_type, - const String & expression_return_name) + const String & expression_return_name, + bool allow_constant_folding) : expression_actions(std::move(expression_actions_)) { /// Check that expression does not contain unusual actions that will break columns structure. @@ -246,6 +269,7 @@ public: .lambda_arguments = lambda_arguments, .return_name = expression_return_name, .return_type = function_return_type, + .allow_constant_folding = allow_constant_folding, }); } diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index c3ba4be7419..d0d6b02e69d 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -184,7 +184,7 @@ public: /** If function isSuitableForConstantFolding then, this method will be called during query analysis * if some arguments are constants. For example logical functions (AndFunction, OrFunction) can - * return they result based on some constant arguments. + * return the result based on some constant arguments. * Arguments are passed without modifications, useDefaultImplementationForNulls, useDefaultImplementationForNothing, * useDefaultImplementationForConstants, useDefaultImplementationForLowCardinality are not applied. */ diff --git a/src/Functions/UserDefined/UserDefinedSQLFunctionVisitor.cpp b/src/Functions/UserDefined/UserDefinedSQLFunctionVisitor.cpp index ebd65471449..a04b8d7b998 100644 --- a/src/Functions/UserDefined/UserDefinedSQLFunctionVisitor.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLFunctionVisitor.cpp @@ -24,92 +24,7 @@ namespace ErrorCodes void UserDefinedSQLFunctionVisitor::visit(ASTPtr & ast) { - if (!ast) - { - chassert(false); - return; - } - - /// FIXME: this helper should use updatePointerToChild(), but - /// forEachPointerToChild() is not implemented for ASTColumnDeclaration - /// (and also some members should be adjusted for this). - const auto visit_child_with_shared_ptr = [&](ASTPtr & child) - { - if (!child) - return; - - auto * old_value = child.get(); - visit(child); - - // child did not change - if (old_value == child.get()) - return; - - // child changed, we need to modify it in the list of children of the parent also - for (auto & current_child : ast->children) - { - if (current_child.get() == old_value) - current_child = child; - } - }; - - if (auto * col_decl = ast->as()) - { - visit_child_with_shared_ptr(col_decl->default_expression); - visit_child_with_shared_ptr(col_decl->ttl); - return; - } - - if (auto * storage = ast->as()) - { - const auto visit_child = [&](IAST * & child) - { - if (!child) - return; - - if (const auto * function = child->template as()) - { - std::unordered_set udf_in_replace_process; - auto replace_result = tryToReplaceFunction(*function, udf_in_replace_process); - if (replace_result) - ast->setOrReplace(child, replace_result); - } - - visit(child); - }; - - visit_child(storage->partition_by); - visit_child(storage->primary_key); - visit_child(storage->order_by); - visit_child(storage->sample_by); - visit_child(storage->ttl_table); - - return; - } - - if (auto * alter = ast->as()) - { - /// It is OK to use updatePointerToChild() because ASTAlterCommand implements forEachPointerToChild() - const auto visit_child_update_parent = [&](ASTPtr & child) - { - if (!child) - return; - - auto * old_ptr = child.get(); - visit(child); - auto * new_ptr = child.get(); - - /// Some AST classes have naked pointers to children elements as members. - /// We have to replace them if the child was replaced. - if (new_ptr != old_ptr) - ast->updatePointerToChild(old_ptr, new_ptr); - }; - - for (auto & children : alter->children) - visit_child_update_parent(children); - - return; - } + chassert(ast); if (const auto * function = ast->template as()) { @@ -120,7 +35,19 @@ void UserDefinedSQLFunctionVisitor::visit(ASTPtr & ast) } for (auto & child : ast->children) + { + if (!child) + return; + + auto * old_ptr = child.get(); visit(child); + auto * new_ptr = child.get(); + + /// Some AST classes have naked pointers to children elements as members. + /// We have to replace them if the child was replaced. + if (new_ptr != old_ptr) + ast->updatePointerToChild(old_ptr, new_ptr); + } } void UserDefinedSQLFunctionVisitor::visit(IAST * ast) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index f4832431f04..e51c465f883 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -282,7 +282,9 @@ public: if (!column_with_type_and_name.column) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be a function.", getName()); - const auto * column_function = typeid_cast(column_with_type_and_name.column.get()); + auto column_function_materialized = column_with_type_and_name.column->convertToFullColumnIfConst(); + + const auto * column_function = typeid_cast(column_function_materialized.get()); if (!column_function) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be a function.", getName()); diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp index 483a5d6404b..a9635f82db4 100644 --- a/src/Functions/array/arrayFold.cpp +++ b/src/Functions/array/arrayFold.cpp @@ -87,7 +87,9 @@ public: if (!lambda_function_with_type_and_name.column) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be a function", getName()); - const auto * lambda_function = typeid_cast(lambda_function_with_type_and_name.column.get()); + auto lambda_function_materialized = lambda_function_with_type_and_name.column->convertToFullColumnIfConst(); + + const auto * lambda_function = typeid_cast(lambda_function_materialized.get()); if (!lambda_function) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be a function", getName()); diff --git a/src/Functions/bitShiftLeft.cpp b/src/Functions/bitShiftLeft.cpp index 0eb0d82ef0f..7fd0f7cf631 100644 --- a/src/Functions/bitShiftLeft.cpp +++ b/src/Functions/bitShiftLeft.cpp @@ -25,8 +25,10 @@ struct BitShiftLeftImpl { if constexpr (is_big_int_v) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "BitShiftLeft is not implemented for big integers as second argument"); - else if (b < 0 || static_cast(b) > 8 * sizeof(A)) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift"); + else if (b < 0) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value"); + else if (static_cast(b) > 8 * sizeof(A)) + return static_cast(0); else if constexpr (is_big_int_v) return static_cast(a) << static_cast(b); else @@ -43,9 +45,10 @@ struct BitShiftLeftImpl const UInt8 word_size = 8 * sizeof(*pos); size_t n = end - pos; const UInt128 bit_limit = static_cast(word_size) * n; - if (b < 0 || static_cast(b) > bit_limit) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift"); - if (b == bit_limit) + if (b < 0) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value"); + + if (b == bit_limit || static_cast(b) > bit_limit) { // insert default value out_vec.push_back(0); @@ -111,9 +114,10 @@ struct BitShiftLeftImpl const UInt8 word_size = 8; size_t n = end - pos; const UInt128 bit_limit = static_cast(word_size) * n; - if (b < 0 || static_cast(b) > bit_limit) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift"); - if (b == bit_limit) + if (b < 0) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value"); + + if (b == bit_limit || static_cast(b) > bit_limit) { // insert default value out_vec.resize_fill(out_vec.size() + n); diff --git a/src/Functions/bitShiftRight.cpp b/src/Functions/bitShiftRight.cpp index 16032b32f68..19ea7b8c751 100644 --- a/src/Functions/bitShiftRight.cpp +++ b/src/Functions/bitShiftRight.cpp @@ -26,8 +26,10 @@ struct BitShiftRightImpl { if constexpr (is_big_int_v) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "BitShiftRight is not implemented for big integers as second argument"); - else if (b < 0 || static_cast(b) > 8 * sizeof(A)) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift"); + else if (b < 0) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value"); + else if (static_cast(b) > 8 * sizeof(A)) + return static_cast(0); else if constexpr (is_big_int_v) return static_cast(a) >> static_cast(b); else @@ -59,9 +61,10 @@ struct BitShiftRightImpl const UInt8 word_size = 8; size_t n = end - pos; const UInt128 bit_limit = static_cast(word_size) * n; - if (b < 0 || static_cast(b) > bit_limit) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift"); - if (b == bit_limit) + if (b < 0) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value"); + + if (b == bit_limit || static_cast(b) > bit_limit) { /// insert default value out_vec.push_back(0); @@ -99,9 +102,10 @@ struct BitShiftRightImpl const UInt8 word_size = 8; size_t n = end - pos; const UInt128 bit_limit = static_cast(word_size) * n; - if (b < 0 || static_cast(b) > bit_limit) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift"); - if (b == bit_limit) + if (b < 0) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value"); + + if (b == bit_limit || static_cast(b) > bit_limit) { // insert default value out_vec.resize_fill(out_vec.size() + n); diff --git a/src/Functions/nested.cpp b/src/Functions/nested.cpp index 85c342b5e7c..29d99b8a6df 100644 --- a/src/Functions/nested.cpp +++ b/src/Functions/nested.cpp @@ -108,27 +108,29 @@ public: { size_t arguments_size = arguments.size(); - const auto * lhs_array = assert_cast(arguments.at(1).column.get()); + ColumnPtr first_array_materialized = arguments[1].column->convertToFullColumnIfConst(); + const ColumnArray & first_array = assert_cast(*first_array_materialized); Columns data_columns; data_columns.reserve(arguments_size); - data_columns.push_back(lhs_array->getDataPtr()); + data_columns.push_back(first_array.getDataPtr()); for (size_t i = 2; i < arguments_size; ++i) { - const auto * rhs_array = assert_cast(arguments[i].column.get()); + ColumnPtr other_array_materialized = arguments[i].column->convertToFullColumnIfConst(); + const ColumnArray & other_array = assert_cast(*other_array_materialized); - if (!lhs_array->hasEqualOffsets(*rhs_array)) + if (!first_array.hasEqualOffsets(other_array)) throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "The argument 2 and argument {} of function {} have different array offsets", i + 1, getName()); - data_columns.push_back(rhs_array->getDataPtr()); + data_columns.push_back(other_array.getDataPtr()); } auto tuple_column = ColumnTuple::create(std::move(data_columns)); - auto array_column = ColumnArray::create(std::move(tuple_column), lhs_array->getOffsetsPtr()); + auto array_column = ColumnArray::create(std::move(tuple_column), first_array.getOffsetsPtr()); return array_column; } @@ -168,7 +170,12 @@ REGISTER_FUNCTION(Nested) { factory.registerFunction(FunctionDocumentation{ .description=R"( +This is a function used internally by the ClickHouse engine and not meant to be used directly. + Returns the array of tuples from multiple arrays. + +The first argument must be a constant array of Strings determining the names of the resulting Tuple. +The other arguments must be arrays of the same size. )", .examples{{"nested", "SELECT nested(['keys', 'values'], ['key_1', 'key_2'], ['value_1','value_2'])", ""}}, .categories{"OtherFunctions"} diff --git a/src/IO/ReadBufferFromFileBase.h b/src/IO/ReadBufferFromFileBase.h index c98dcd5a93e..c59a5c152b6 100644 --- a/src/IO/ReadBufferFromFileBase.h +++ b/src/IO/ReadBufferFromFileBase.h @@ -60,6 +60,8 @@ public: /// file offset and what getPosition() returns. virtual bool isRegularLocalFile(size_t * /*out_view_offsee*/) { return false; } + virtual bool isCached() const { return false; } + protected: std::optional file_size; ProfileCallback profile_callback; diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index aa52e00e6d7..c1747314c76 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -58,6 +58,10 @@ struct ReadSettings bool enable_filesystem_cache_log = false; size_t filesystem_cache_segments_batch_size = 20; size_t filesystem_cache_reserve_space_wait_lock_timeout_milliseconds = 1000; + bool filesystem_cache_allow_background_download = true; + bool filesystem_cache_allow_background_download_for_metadata_files_in_packed_storage = true; + bool filesystem_cache_allow_background_download_during_fetch = true; + bool filesystem_cache_prefer_bigger_buffer_size = true; bool use_page_cache_for_disks_without_file_cache = false; bool read_from_page_cache_if_exists_otherwise_bypass_cache = false; diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 9a0eccd8783..088087458c7 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -645,7 +645,7 @@ Client::doRequestWithRetryNetworkErrors(RequestType & request, RequestFn request try { /// S3 does retries network errors actually. - /// But it is matter when errors occur. + /// But it does matter when errors occur. /// This code retries a specific case when /// network error happens when XML document is being read from the response body. /// Hence, the response body is a stream, network errors are possible at reading. @@ -656,8 +656,9 @@ Client::doRequestWithRetryNetworkErrors(RequestType & request, RequestFn request /// Requests that expose the response stream as an answer are not retried with that code. E.g. GetObject. return request_fn_(request_); } - catch (Poco::Net::ConnectionResetException &) + catch (Poco::Net::NetException &) { + /// This includes "connection reset", "malformed message", and possibly other exceptions. if constexpr (IsReadMethod) { diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 65c3fe8cfcf..696021b418c 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -1308,7 +1308,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & String lambda_name = data.getUniqueName("__lambda"); auto function_capture = std::make_shared( - lambda_actions, captured, lambda_arguments, result_type, result_name); + lambda_actions, captured, lambda_arguments, result_type, result_name, false); data.addFunction(function_capture, captured, lambda_name); argument_types[i] = std::make_shared(lambda_type->getArgumentTypes(), result_type); diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index 08776ad5aee..f5d5fdec6ba 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -83,7 +83,8 @@ void EvictionCandidates::removeQueueEntries(const CachePriorityGuard::Lock & loc queue_iterator->invalidate(); chassert(candidate->releasable()); - candidate->file_segment->resetQueueIterator(); + candidate->file_segment->markDelayedRemovalAndResetQueueIterator(); + /// We need to set removed flag in file segment metadata, /// because in dynamic cache resize we first remove queue entries, /// then evict which also removes file segment metadata, diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index f7b7ffc5aea..7de3f7af78d 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -37,6 +37,11 @@ namespace ProfileEvents extern const Event FilesystemCacheFailToReserveSpaceBecauseOfCacheResize; } +namespace CurrentMetrics +{ + extern const Metric FilesystemCacheDownloadQueueElements; +} + namespace DB { @@ -918,7 +923,13 @@ bool FileCache::tryReserve( if (!query_priority->collectCandidatesForEviction( size, required_elements_num, reserve_stat, eviction_candidates, {}, user.user_id, cache_lock)) { - failure_reason = "cannot evict enough space for query limit"; + const auto & stat = reserve_stat.total_stat; + failure_reason = fmt::format( + "cannot evict enough space for query limit " + "(non-releasable count: {}, non-releasable size: {}, " + "releasable count: {}, releasable size: {}, background download elements: {})", + stat.non_releasable_count, stat.non_releasable_size, stat.releasable_count, stat.releasable_size, + CurrentMetrics::get(CurrentMetrics::FilesystemCacheDownloadQueueElements)); return false; } @@ -933,7 +944,13 @@ bool FileCache::tryReserve( if (!main_priority->collectCandidatesForEviction( size, required_elements_num, reserve_stat, eviction_candidates, queue_iterator, user.user_id, cache_lock)) { - failure_reason = "cannot evict enough space"; + const auto & stat = reserve_stat.total_stat; + failure_reason = fmt::format( + "cannot evict enough space " + "(non-releasable count: {}, non-releasable size: {}, " + "releasable count: {}, releasable size: {}, background download elements: {})", + stat.non_releasable_count, stat.non_releasable_size, stat.releasable_count, stat.releasable_size, + CurrentMetrics::get(CurrentMetrics::FilesystemCacheDownloadQueueElements)); return false; } diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index c356800fa57..541f0f5607a 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -28,6 +28,7 @@ namespace ProfileEvents extern const Event FileSegmentFailToIncreasePriority; extern const Event FilesystemCacheHoldFileSegments; extern const Event FilesystemCacheUnusedHoldFileSegments; + extern const Event FilesystemCacheBackgroundDownloadQueuePush; } namespace CurrentMetrics @@ -171,10 +172,11 @@ void FileSegment::setQueueIterator(Priority::IteratorPtr iterator) queue_iterator = iterator; } -void FileSegment::resetQueueIterator() +void FileSegment::markDelayedRemovalAndResetQueueIterator() { auto lk = lock(); - queue_iterator.reset(); + on_delayed_removal = true; + queue_iterator = {}; } size_t FileSegment::getCurrentWriteOffset() const @@ -627,7 +629,7 @@ void FileSegment::completePartAndResetDownloader() LOG_TEST(log, "Complete batch. ({})", getInfoForLogUnlocked(lk)); } -void FileSegment::complete() +void FileSegment::complete(bool allow_background_download) { ProfileEventTimeIncrement watch(ProfileEvents::FileSegmentCompleteMicroseconds); @@ -700,12 +702,15 @@ void FileSegment::complete() case State::PARTIALLY_DOWNLOADED: { chassert(current_downloaded_size > 0); + chassert(fs::exists(getPath())); + chassert(fs::file_size(getPath()) > 0); if (is_last_holder) { bool added_to_download_queue = false; - if (background_download_enabled && remote_file_reader) + if (allow_background_download && background_download_enabled && remote_file_reader) { + ProfileEvents::increment(ProfileEvents::FilesystemCacheBackgroundDownloadQueuePush); added_to_download_queue = locked_key->addToDownloadQueue(offset(), segment_lock); /// Finish download in background. } @@ -841,29 +846,60 @@ bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock & lock) } } - if (download_state == State::DOWNLOADED) + switch (download_state.load()) { - chassert(downloader_id.empty()); - chassert(downloaded_size == reserved_size); - chassert(downloaded_size == range().size()); - chassert(downloaded_size > 0); - chassert(std::filesystem::file_size(getPath()) > 0); - check_iterator(queue_iterator); - } - else - { - if (download_state == State::DOWNLOADING) - { - chassert(!downloader_id.empty()); - } - else if (download_state == State::PARTIALLY_DOWNLOADED - || download_state == State::EMPTY) + case State::EMPTY: { chassert(downloader_id.empty()); + chassert(!fs::exists(getPath())); + chassert(!queue_iterator); + break; } + case State::DOWNLOADED: + { + chassert(downloader_id.empty()); - chassert(reserved_size >= downloaded_size); - check_iterator(queue_iterator); + chassert(downloaded_size == reserved_size); + chassert(downloaded_size == range().size()); + chassert(downloaded_size > 0); + chassert(fs::file_size(getPath()) > 0); + + chassert(queue_iterator || on_delayed_removal); + check_iterator(queue_iterator); + break; + } + case State::DOWNLOADING: + { + chassert(!downloader_id.empty()); + if (downloaded_size) + { + chassert(queue_iterator); + chassert(fs::file_size(getPath()) > 0); + } + break; + } + case State::PARTIALLY_DOWNLOADED: + { + chassert(downloader_id.empty()); + + chassert(reserved_size >= downloaded_size); + chassert(downloaded_size > 0); + chassert(fs::file_size(getPath()) > 0); + + chassert(queue_iterator); + check_iterator(queue_iterator); + break; + } + case State::PARTIALLY_DOWNLOADED_NO_CONTINUATION: + { + chassert(reserved_size >= downloaded_size); + check_iterator(queue_iterator); + break; + } + case State::DETACHED: + { + break; + } } return true; @@ -991,7 +1027,12 @@ FileSegmentsHolder::FileSegmentsHolder(FileSegments && file_segments_) FileSegmentPtr FileSegmentsHolder::getSingleFileSegment() const { if (file_segments.size() != 1) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected single file segment, got: {} in holder {}", file_segments.size(), toString()); + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Expected single file segment, got: {} in holder {}", + file_segments.size(), toString()); + } return file_segments.front(); } @@ -1001,7 +1042,23 @@ void FileSegmentsHolder::reset() ProfileEvents::increment(ProfileEvents::FilesystemCacheUnusedHoldFileSegments, file_segments.size()); for (auto file_segment_it = file_segments.begin(); file_segment_it != file_segments.end();) - file_segment_it = completeAndPopFrontImpl(); + { + try + { + /// One might think it would have been more correct to do `false` here, + /// not to allow background download for file segments that we actually did not start reading. + /// But actually we would only do that, if those file segments were already read partially by some other thread/query + /// but they were not put to the download queue, because current thread was holding them in Holder. + /// So as a culprit, we need to allow to happen what would have happened if we did not exist. + file_segment_it = completeAndPopFrontImpl(true); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + chassert(false); + continue; + } + } file_segments.clear(); } @@ -1010,9 +1067,9 @@ FileSegmentsHolder::~FileSegmentsHolder() reset(); } -FileSegments::iterator FileSegmentsHolder::completeAndPopFrontImpl() +FileSegments::iterator FileSegmentsHolder::completeAndPopFrontImpl(bool allow_background_download) { - front().complete(); + front().complete(allow_background_download); CurrentMetrics::sub(CurrentMetrics::FilesystemCacheHoldFileSegments); return file_segments.erase(file_segments.begin()); } diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index ee9aee1e354..21d5f9dab5f 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -177,7 +177,7 @@ public: void setQueueIterator(Priority::IteratorPtr iterator); - void resetQueueIterator(); + void markDelayedRemovalAndResetQueueIterator(); KeyMetadataPtr tryGetKeyMetadata() const; @@ -189,7 +189,7 @@ public: * ========== Methods that must do cv.notify() ================== */ - void complete(); + void complete(bool allow_background_download); void completePartAndResetDownloader(); @@ -249,12 +249,13 @@ private: String tryGetPath() const; - Key file_key; + const Key file_key; Range segment_range; const FileSegmentKind segment_kind; /// Size of the segment is not known until it is downloaded and /// can be bigger than max_file_segment_size. - const bool is_unbound = false; + /// is_unbound == true for temporary data in cache. + const bool is_unbound; const bool background_download_enabled; std::atomic download_state; @@ -279,6 +280,8 @@ private: std::atomic hits_count = 0; /// cache hits. std::atomic ref_count = 0; /// Used for getting snapshot state + bool on_delayed_removal = false; + CurrentMetrics::Increment metric_increment{CurrentMetrics::CacheFileSegments}; }; @@ -297,7 +300,7 @@ struct FileSegmentsHolder final : private boost::noncopyable String toString(bool with_state = false) const; - void popFront() { completeAndPopFrontImpl(); } + void completeAndPopFront(bool allow_background_download) { completeAndPopFrontImpl(allow_background_download); } FileSegment & front() { return *file_segments.front(); } const FileSegment & front() const { return *file_segments.front(); } @@ -319,7 +322,7 @@ struct FileSegmentsHolder final : private boost::noncopyable private: FileSegments file_segments{}; - FileSegments::iterator completeAndPopFrontImpl(); + FileSegments::iterator completeAndPopFrontImpl(bool allow_background_download); }; using FileSegmentsHolderPtr = std::unique_ptr; diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 99ea01aa4f1..231545212cd 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -940,7 +940,16 @@ KeyMetadata::iterator LockedKey::removeFileSegmentImpl( if (file_segment->queue_iterator && invalidate_queue_entry) file_segment->queue_iterator->invalidate(); - file_segment->detach(segment_lock, *this); + try + { + file_segment->detach(segment_lock, *this); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + chassert(false); + /// Do not rethrow, we must delete the file below. + } try { @@ -990,8 +999,8 @@ void LockedKey::shrinkFileSegmentToDownloadedSize( * because of no space left in cache, we need to be able to cut file segment's size to downloaded_size. */ - auto metadata = getByOffset(offset); - const auto & file_segment = metadata->file_segment; + auto file_segment_metadata = getByOffset(offset); + const auto & file_segment = file_segment_metadata->file_segment; chassert(file_segment->assertCorrectnessUnlocked(segment_lock)); const size_t downloaded_size = file_segment->getDownloadedSize(); @@ -1006,15 +1015,15 @@ void LockedKey::shrinkFileSegmentToDownloadedSize( chassert(file_segment->reserved_size >= downloaded_size); int64_t diff = file_segment->reserved_size - downloaded_size; - metadata->file_segment = std::make_shared( + file_segment_metadata->file_segment = std::make_shared( getKey(), offset, downloaded_size, FileSegment::State::DOWNLOADED, CreateFileSegmentSettings(file_segment->getKind()), false, file_segment->cache, key_metadata, file_segment->queue_iterator); if (diff) - metadata->getQueueIterator()->decrementSize(diff); + file_segment_metadata->getQueueIterator()->decrementSize(diff); - chassert(file_segment->assertCorrectnessUnlocked(segment_lock)); + chassert(file_segment_metadata->file_segment->assertCorrectnessUnlocked(segment_lock)); } bool LockedKey::addToDownloadQueue(size_t offset, const FileSegmentGuard::Lock &) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index e232b5c2b6e..ebcc30b6a81 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -194,6 +194,9 @@ namespace Setting extern const SettingsUInt64 filesystem_cache_max_download_size; extern const SettingsUInt64 filesystem_cache_reserve_space_wait_lock_timeout_milliseconds; extern const SettingsUInt64 filesystem_cache_segments_batch_size; + extern const SettingsBool filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage; + extern const SettingsBool filesystem_cache_enable_background_download_during_fetch; + extern const SettingsBool filesystem_cache_prefer_bigger_buffer_size; extern const SettingsBool http_make_head_request; extern const SettingsUInt64 http_max_fields; extern const SettingsUInt64 http_max_field_name_size; @@ -5754,6 +5757,10 @@ ReadSettings Context::getReadSettings() const res.filesystem_cache_segments_batch_size = settings_ref[Setting::filesystem_cache_segments_batch_size]; res.filesystem_cache_reserve_space_wait_lock_timeout_milliseconds = settings_ref[Setting::filesystem_cache_reserve_space_wait_lock_timeout_milliseconds]; + res.filesystem_cache_allow_background_download_for_metadata_files_in_packed_storage + = settings_ref[Setting::filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage]; + res.filesystem_cache_allow_background_download_during_fetch = settings_ref[Setting::filesystem_cache_enable_background_download_during_fetch]; + res.filesystem_cache_prefer_bigger_buffer_size = settings_ref[Setting::filesystem_cache_prefer_bigger_buffer_size]; res.filesystem_cache_max_download_size = settings_ref[Setting::filesystem_cache_max_download_size]; res.skip_download_if_exceeds_query_cache = settings_ref[Setting::skip_download_if_exceeds_query_cache]; diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index d4da038c089..afab06a13ba 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -5,9 +5,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -19,9 +21,8 @@ #include #include #include -#include #include -#include +#include namespace ProfileEvents { @@ -67,6 +68,18 @@ bool ExecuteScalarSubqueriesMatcher::needChildVisit(ASTPtr & node, const ASTPtr return false; } + if (auto * tables = node->as()) + { + /// Contrary to what's said in the code block above, ARRAY JOIN needs to resolve the subquery if possible + /// and assign an alias for 02367_optimize_trivial_count_with_array_join to pass. Otherwise it will fail in + /// ArrayJoinedColumnsVisitor (`No alias for non-trivial value in ARRAY JOIN: _a`) + /// This looks 100% as a incomplete code working on top of a bug, but this code has already been made obsolete + /// by the new analyzer, so it's an inconvenience we can live with until we deprecate it. + if (child == tables->array_join) + return true; + return false; + } + return true; } @@ -246,6 +259,8 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr if (tmp_block.rows() != 0) throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, "Scalar subquery returned more than one row"); + + logProcessorProfile(data.getContext(), io.pipeline.getProcessors()); } block = materializeBlock(block); diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 4e5cf7d2549..58224239723 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -105,6 +105,8 @@ namespace Setting extern const SettingsBool query_plan_aggregation_in_order; extern const SettingsBool query_plan_read_in_order; extern const SettingsUInt64 use_index_for_in_with_subqueries_max_values; + extern const SettingsBool allow_suspicious_types_in_group_by; + extern const SettingsBool allow_suspicious_types_in_order_by; } @@ -118,6 +120,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; extern const int UNKNOWN_IDENTIFIER; extern const int UNKNOWN_TYPE_OF_AST_NODE; + extern const int ILLEGAL_COLUMN; } namespace @@ -1368,6 +1371,7 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain ExpressionActionsChain::Step & step = chain.lastStep(columns_after_join); ASTs asts = select_query->groupBy()->children; + NameSet group_by_keys; if (select_query->group_by_with_grouping_sets) { for (const auto & ast : asts) @@ -1375,6 +1379,7 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain for (const auto & ast_element : ast->children) { step.addRequiredOutput(ast_element->getColumnName()); + group_by_keys.insert(ast_element->getColumnName()); getRootActions(ast_element, only_types, step.actions()->dag); } } @@ -1384,10 +1389,17 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain for (const auto & ast : asts) { step.addRequiredOutput(ast->getColumnName()); + group_by_keys.insert(ast->getColumnName()); getRootActions(ast, only_types, step.actions()->dag); } } + for (const auto & result_column : step.getResultColumns()) + { + if (group_by_keys.contains(result_column.name)) + validateGroupByKeyType(result_column.type); + } + if (optimize_aggregation_in_order) { for (auto & child : asts) @@ -1402,6 +1414,26 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain return true; } +void SelectQueryExpressionAnalyzer::validateGroupByKeyType(const DB::DataTypePtr & key_type) const +{ + if (getContext()->getSettingsRef()[Setting::allow_suspicious_types_in_group_by]) + return; + + auto check = [](const IDataType & type) + { + if (isDynamic(type) || isVariant(type)) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Data types Variant/Dynamic are not allowed in GROUP BY keys, because it can lead to unexpected results. " + "Consider using a subcolumn with a specific data type instead (for example 'column.Int64' or 'json.some.path.:Int64' if " + "its a JSON path subcolumn) or casting this column to a specific data type. " + "Set setting allow_suspicious_types_in_group_by = 1 in order to allow it"); + }; + + check(*key_type); + key_type->forEachChild(check); +} + void SelectQueryExpressionAnalyzer::appendAggregateFunctionsArguments(ExpressionActionsChain & chain, bool only_types) { const auto * select_query = getAggregatingQuery(); @@ -1599,6 +1631,12 @@ ActionsAndProjectInputsFlagPtr SelectQueryExpressionAnalyzer::appendOrderBy( with_fill = true; } + for (const auto & result_column : step.getResultColumns()) + { + if (order_by_keys.contains(result_column.name)) + validateOrderByKeyType(result_column.type); + } + if (auto interpolate_list = select_query->interpolate()) { @@ -1664,6 +1702,26 @@ ActionsAndProjectInputsFlagPtr SelectQueryExpressionAnalyzer::appendOrderBy( return actions; } +void SelectQueryExpressionAnalyzer::validateOrderByKeyType(const DataTypePtr & key_type) const +{ + if (getContext()->getSettingsRef()[Setting::allow_suspicious_types_in_order_by]) + return; + + auto check = [](const IDataType & type) + { + if (isDynamic(type) || isVariant(type)) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Data types Variant/Dynamic are not allowed in ORDER BY keys, because it can lead to unexpected results. " + "Consider using a subcolumn with a specific data type instead (for example 'column.Int64' or 'json.some.path.:Int64' if " + "its a JSON path subcolumn) or casting this column to a specific data type. " + "Set setting allow_suspicious_types_in_order_by = 1 in order to allow it"); + }; + + check(*key_type); + key_type->forEachChild(check); +} + bool SelectQueryExpressionAnalyzer::appendLimitBy(ExpressionActionsChain & chain, bool only_types) { const auto * select_query = getSelectQuery(); @@ -1981,7 +2039,9 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( Block before_prewhere_sample = source_header; if (sanitizeBlock(before_prewhere_sample)) { - before_prewhere_sample = prewhere_dag_and_flags->dag.updateHeader(before_prewhere_sample); + ExpressionActions( + prewhere_dag_and_flags->dag.clone(), + ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_prewhere_sample); auto & column_elem = before_prewhere_sample.getByName(query.prewhere()->getColumnName()); /// If the filter column is a constant, record it. if (column_elem.column) @@ -2013,7 +2073,9 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( before_where_sample = source_header; if (sanitizeBlock(before_where_sample)) { - before_where_sample = before_where->dag.updateHeader(before_where_sample); + ExpressionActions( + before_where->dag.clone(), + ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_where_sample); auto & column_elem = before_where_sample.getByName(query.where()->getColumnName()); diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index d4ee8832c1c..5e3e1f81ca1 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -396,6 +396,7 @@ private: ActionsAndProjectInputsFlagPtr appendPrewhere(ExpressionActionsChain & chain, bool only_types); bool appendWhere(ExpressionActionsChain & chain, bool only_types); bool appendGroupBy(ExpressionActionsChain & chain, bool only_types, bool optimize_aggregation_in_order, ManyExpressionActions &); + void validateGroupByKeyType(const DataTypePtr & key_type) const; void appendAggregateFunctionsArguments(ExpressionActionsChain & chain, bool only_types); void appendWindowFunctionsArguments(ExpressionActionsChain & chain, bool only_types); @@ -408,6 +409,7 @@ private: bool appendHaving(ExpressionActionsChain & chain, bool only_types); /// appendSelect ActionsAndProjectInputsFlagPtr appendOrderBy(ExpressionActionsChain & chain, bool only_types, bool optimize_read_in_order, ManyExpressionActions &); + void validateOrderByKeyType(const DataTypePtr & key_type) const; bool appendLimitBy(ExpressionActionsChain & chain, bool only_types); /// appendProjectResult }; diff --git a/src/Interpreters/FillingRow.cpp b/src/Interpreters/FillingRow.cpp index 21b5b04bca3..384ad669206 100644 --- a/src/Interpreters/FillingRow.cpp +++ b/src/Interpreters/FillingRow.cpp @@ -1,11 +1,24 @@ -#include -#include +#include + #include +#include +#include +#include +#include namespace DB { +constexpr static bool debug_logging_enabled = false; + +template +inline static void logDebug(const char * fmt_str, Args&&... args) +{ + if constexpr (debug_logging_enabled) + LOG_DEBUG(getLogger("FillingRow"), "{}", fmt::format(fmt::runtime(fmt_str), std::forward(args)...)); +} + bool less(const Field & lhs, const Field & rhs, int direction) { if (direction == -1) @@ -28,6 +41,10 @@ FillingRow::FillingRow(const SortDescription & sort_description_) : sort_description(sort_description_) { row.resize(sort_description.size()); + + constraints.reserve(sort_description.size()); + for (size_t i = 0; i < size(); ++i) + constraints.push_back(getFillDescription(i).fill_to); } bool FillingRow::operator<(const FillingRow & other) const @@ -63,71 +80,254 @@ bool FillingRow::isNull() const return true; } -std::pair FillingRow::next(const FillingRow & to_row) +std::optional FillingRow::doLongJump(const FillColumnDescription & descr, size_t column_ind, const Field & to) { + Field shifted_value = row[column_ind]; + + if (less(to, shifted_value, getDirection(column_ind))) + return std::nullopt; + + for (int32_t step_len = 1, step_no = 0; step_no < 100 && step_len > 0; ++step_no) + { + Field next_value = shifted_value; + descr.step_func(next_value, step_len); + + if (less(to, next_value, getDirection(0))) + { + step_len /= 2; + } + else + { + shifted_value = std::move(next_value); + step_len *= 2; + } + } + + return shifted_value; +} + +bool FillingRow::hasSomeConstraints(size_t pos) const +{ + return !constraints[pos].isNull(); +} + +bool FillingRow::isConstraintsSatisfied(size_t pos) const +{ + chassert(!row[pos].isNull()); + chassert(hasSomeConstraints(pos)); + + int direction = getDirection(pos); + logDebug("constraint: {}, row: {}, direction: {}", constraints[pos], row[pos], direction); + + return less(row[pos], constraints[pos], direction); +} + +static const Field & findBorder(const Field & constraint, const Field & next_original, int direction) +{ + if (constraint.isNull()) + return next_original; + + if (next_original.isNull()) + return constraint; + + if (less(constraint, next_original, direction)) + return constraint; + + return next_original; +} + +bool FillingRow::next(const FillingRow & next_original_row, bool& value_changed) +{ + const size_t row_size = size(); size_t pos = 0; /// Find position we need to increment for generating next row. for (; pos < row_size; ++pos) - if (!row[pos].isNull() && !to_row.row[pos].isNull() && !equals(row[pos], to_row.row[pos])) + { + if (row[pos].isNull()) + continue; + + const Field & border = findBorder(constraints[pos], next_original_row[pos], getDirection(pos)); + logDebug("border: {}", border); + + if (!border.isNull() && !equals(row[pos], border)) break; + } - if (pos == row_size || less(to_row.row[pos], row[pos], getDirection(pos))) - return {false, false}; + logDebug("pos: {}", pos); - /// If we have any 'fill_to' value at position greater than 'pos', - /// we need to generate rows up to 'fill_to' value. + if (pos == row_size) + return false; + + if (!next_original_row[pos].isNull() && less(next_original_row[pos], row[pos], getDirection(pos))) + return false; + + if (!constraints[pos].isNull() && !less(row[pos], constraints[pos], getDirection(pos))) + return false; + + /// If we have any 'fill_to' value at position greater than 'pos' or configured staleness, + /// we need to generate rows up to one of this borders. for (size_t i = row_size - 1; i > pos; --i) { auto & fill_column_desc = getFillDescription(i); - if (fill_column_desc.fill_to.isNull() || row[i].isNull()) + if (row[i].isNull()) + continue; + + if (constraints[i].isNull()) continue; Field next_value = row[i]; - fill_column_desc.step_func(next_value); - if (less(next_value, fill_column_desc.fill_to, getDirection(i))) - { - row[i] = next_value; - initFromDefaults(i + 1); - return {true, true}; - } + fill_column_desc.step_func(next_value, 1); + + if (!less(next_value, constraints[i], getDirection(i))) + continue; + + row[i] = next_value; + initUsingFrom(i + 1); + + value_changed = true; + return true; } auto next_value = row[pos]; - getFillDescription(pos).step_func(next_value); + getFillDescription(pos).step_func(next_value, 1); - if (less(to_row.row[pos], next_value, getDirection(pos)) || equals(next_value, getFillDescription(pos).fill_to)) - return {false, false}; + if (!next_original_row[pos].isNull() && less(next_original_row[pos], next_value, getDirection(pos))) + return false; + + if (!constraints[pos].isNull() && !less(next_value, constraints[pos], getDirection(pos))) + return false; row[pos] = next_value; - if (equals(row[pos], to_row.row[pos])) + if (equals(row[pos], next_original_row[pos])) { bool is_less = false; for (size_t i = pos + 1; i < row_size; ++i) { - const auto & fill_from = getFillDescription(i).fill_from; - if (!fill_from.isNull()) - row[i] = fill_from; + const auto & descr = getFillDescription(i); + if (!descr.fill_from.isNull()) + row[i] = descr.fill_from; else - row[i] = to_row.row[i]; - is_less |= less(row[i], to_row.row[i], getDirection(i)); + row[i] = next_original_row[i]; + + is_less |= ( + (next_original_row[i].isNull() || less(row[i], next_original_row[i], getDirection(i))) && + (constraints[i].isNull() || less(row[i], constraints[i], getDirection(i))) + ); } - return {is_less, true}; + value_changed = true; + return is_less; } - initFromDefaults(pos + 1); - return {true, true}; + initUsingFrom(pos + 1); + + value_changed = true; + return true; } -void FillingRow::initFromDefaults(size_t from_pos) +bool FillingRow::shift(const FillingRow & next_original_row, bool& value_changed) +{ + logDebug("next_original_row: {}, current: {}", next_original_row, *this); + + for (size_t pos = 0; pos < size(); ++pos) + { + if (row[pos].isNull() || next_original_row[pos].isNull() || equals(row[pos], next_original_row[pos])) + continue; + + if (less(next_original_row[pos], row[pos], getDirection(pos))) + return false; + + std::optional next_value = doLongJump(getFillDescription(pos), pos, next_original_row[pos]); + logDebug("jumped to next value: {}", next_value.value_or("Did not complete")); + + row[pos] = std::move(next_value.value()); + + if (equals(row[pos], next_original_row[pos])) + { + bool is_less = false; + for (size_t i = pos + 1; i < size(); ++i) + { + const auto & descr = getFillDescription(i); + if (!descr.fill_from.isNull()) + row[i] = descr.fill_from; + else + row[i] = next_original_row[i]; + + is_less |= ( + (next_original_row[i].isNull() || less(row[i], next_original_row[i], getDirection(i))) && + (constraints[i].isNull() || less(row[i], constraints[i], getDirection(i))) + ); + } + + logDebug("is less: {}", is_less); + + value_changed = true; + return is_less; + } + else + { + initUsingTo(/*from_pos=*/pos + 1); + + value_changed = false; + return false; + } + } + + return false; +} + +bool FillingRow::hasSomeConstraints() const +{ + for (size_t pos = 0; pos < size(); ++pos) + if (hasSomeConstraints(pos)) + return true; + + return false; +} + +bool FillingRow::isConstraintsSatisfied() const +{ + for (size_t pos = 0; pos < size(); ++pos) + { + if (row[pos].isNull() || !hasSomeConstraints(pos)) + continue; + + return isConstraintsSatisfied(pos); + } + + return true; +} + +void FillingRow::initUsingFrom(size_t from_pos) { for (size_t i = from_pos; i < sort_description.size(); ++i) row[i] = getFillDescription(i).fill_from; } +void FillingRow::initUsingTo(size_t from_pos) +{ + for (size_t i = from_pos; i < sort_description.size(); ++i) + row[i] = getFillDescription(i).fill_to; +} + +void FillingRow::updateConstraintsWithStalenessRow(const Columns& base_row, size_t row_ind) +{ + for (size_t i = 0; i < size(); ++i) + { + const auto& descr = getFillDescription(i); + + if (!descr.fill_staleness.isNull()) + { + Field staleness_border = (*base_row[i])[row_ind]; + descr.staleness_step_func(staleness_border, 1); + constraints[i] = findBorder(descr.fill_to, staleness_border, getDirection(i)); + } + } +} + String FillingRow::dump() const { WriteBufferFromOwnString out; @@ -147,3 +347,12 @@ WriteBuffer & operator<<(WriteBuffer & out, const FillingRow & row) } } + +template <> +struct fmt::formatter : fmt::formatter +{ + constexpr auto format(const DB::FillingRow & row, format_context & ctx) const + { + return fmt::format_to(ctx.out(), "{}", row.dump()); + } +}; diff --git a/src/Interpreters/FillingRow.h b/src/Interpreters/FillingRow.h index 004b417542c..08d624a2405 100644 --- a/src/Interpreters/FillingRow.h +++ b/src/Interpreters/FillingRow.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace DB { @@ -15,16 +15,28 @@ bool equals(const Field & lhs, const Field & rhs); */ class FillingRow { + /// finds last value <= to + std::optional doLongJump(const FillColumnDescription & descr, size_t column_ind, const Field & to); + + bool hasSomeConstraints(size_t pos) const; + bool isConstraintsSatisfied(size_t pos) const; + public: explicit FillingRow(const SortDescription & sort_description); /// Generates next row according to fill 'from', 'to' and 'step' values. - /// Return pair of boolean - /// apply - true if filling values should be inserted into result set - /// value_changed - true if filling row value was changed - std::pair next(const FillingRow & to_row); + /// Returns true if filling values should be inserted into result set + bool next(const FillingRow & next_original_row, bool& value_changed); - void initFromDefaults(size_t from_pos = 0); + /// Returns true if need to generate some prefix for to_row + bool shift(const FillingRow & next_original_row, bool& value_changed); + + bool hasSomeConstraints() const; + bool isConstraintsSatisfied() const; + + void initUsingFrom(size_t from_pos = 0); + void initUsingTo(size_t from_pos = 0); + void updateConstraintsWithStalenessRow(const Columns& base_row, size_t row_ind); Field & operator[](size_t index) { return row[index]; } const Field & operator[](size_t index) const { return row[index]; } @@ -42,6 +54,7 @@ public: private: Row row; + Row constraints; SortDescription sort_description; }; diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index a38a7ab45d1..f6586f8bfc2 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1467,7 +1467,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) bool is_secondary_query = getContext()->getZooKeeperMetadataTransaction() && !getContext()->getZooKeeperMetadataTransaction()->isInitialQuery(); auto mode = getLoadingStrictnessLevel(create.attach, /*force_attach*/ false, /*has_force_restore_data_flag*/ false, is_secondary_query || is_restore_from_backup); - if (!create.sql_security && create.supportSQLSecurity() && !getContext()->getServerSettings()[ServerSetting::ignore_empty_sql_security_in_create_view_query]) + if (!create.sql_security && create.supportSQLSecurity() && (create.refresh_strategy || !getContext()->getServerSettings()[ServerSetting::ignore_empty_sql_security_in_create_view_query])) create.sql_security = std::make_shared(); if (create.sql_security) diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 80b9d91a248..797895e4a93 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -121,8 +121,7 @@ StoragePtr InterpreterInsertQuery::getTable(ASTInsertQuery & query) if (current_context->getSettingsRef()[Setting::allow_experimental_analyzer]) { - InterpreterSelectQueryAnalyzer interpreter_select(query.select, current_context, select_query_options); - header_block = interpreter_select.getSampleBlock(); + header_block = InterpreterSelectQueryAnalyzer::getSampleBlock(query.select, current_context, select_query_options); } else { diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 45636ab40b9..b651bfb245e 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -795,9 +795,9 @@ BlockIO InterpreterSystemQuery::execute() case Type::WAIT_FAILPOINT: { getContext()->checkAccess(AccessType::SYSTEM_FAILPOINT); - LOG_TRACE(log, "waiting for failpoint {}", query.fail_point_name); + LOG_TRACE(log, "Waiting for failpoint {}", query.fail_point_name); FailPointInjection::pauseFailPoint(query.fail_point_name); - LOG_TRACE(log, "finished failpoint {}", query.fail_point_name); + LOG_TRACE(log, "Finished waiting for failpoint {}", query.fail_point_name); break; } case Type::RESET_COVERAGE: @@ -1310,7 +1310,7 @@ RefreshTaskList InterpreterSystemQuery::getRefreshTasks() void InterpreterSystemQuery::prewarmMarkCache() { if (table_id.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table is not specified for prewarming marks cache"); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table is not specified for PREWARM MARK CACHE command"); getContext()->checkAccess(AccessType::SYSTEM_PREWARM_MARK_CACHE, table_id); diff --git a/src/Interpreters/PreparedSets.cpp b/src/Interpreters/PreparedSets.cpp index 538108165fb..c69e2f84d42 100644 --- a/src/Interpreters/PreparedSets.cpp +++ b/src/Interpreters/PreparedSets.cpp @@ -1,21 +1,22 @@ #include #include -#include -#include -#include -#include -#include +#include #include -#include +#include +#include +#include +#include +#include #include #include +#include #include +#include #include #include -#include -#include #include #include +#include namespace DB { @@ -239,6 +240,8 @@ SetPtr FutureSetFromSubquery::buildOrderedSetInplace(const ContextPtr & context) if (!set_and_key->set->isCreated()) return nullptr; + logProcessorProfile(context, pipeline.getProcessors()); + return set_and_key->set; } diff --git a/src/Interpreters/ProcessorsProfileLog.cpp b/src/Interpreters/ProcessorsProfileLog.cpp index 8a646b5d0e7..d7811e5e9e2 100644 --- a/src/Interpreters/ProcessorsProfileLog.cpp +++ b/src/Interpreters/ProcessorsProfileLog.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -8,16 +9,19 @@ #include #include #include +#include #include #include #include -#include - -#include namespace DB { +namespace Setting +{ +extern const SettingsBool log_processors_profiles; +} + ColumnsDescription ProcessorProfileLogElement::getColumnsDescription() { return ColumnsDescription @@ -81,5 +85,57 @@ void ProcessorProfileLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(output_bytes); } +void logProcessorProfile(ContextPtr context, const Processors & processors) +{ + const Settings & settings = context->getSettingsRef(); + if (settings[Setting::log_processors_profiles]) + { + if (auto processors_profile_log = context->getProcessorsProfileLog()) + { + ProcessorProfileLogElement processor_elem; + const auto time_now = std::chrono::system_clock::now(); + processor_elem.event_time = timeInSeconds(time_now); + processor_elem.event_time_microseconds = timeInMicroseconds(time_now); + processor_elem.initial_query_id = context->getInitialQueryId(); + processor_elem.query_id = context->getCurrentQueryId(); + + auto get_proc_id = [](const IProcessor & proc) -> UInt64 { return reinterpret_cast(&proc); }; + + for (const auto & processor : processors) + { + std::vector parents; + for (const auto & port : processor->getOutputs()) + { + if (!port.isConnected()) + continue; + const IProcessor & next = port.getInputPort().getProcessor(); + parents.push_back(get_proc_id(next)); + } + + processor_elem.id = get_proc_id(*processor); + processor_elem.parent_ids = std::move(parents); + + processor_elem.plan_step = reinterpret_cast(processor->getQueryPlanStep()); + processor_elem.plan_step_name = processor->getPlanStepName(); + processor_elem.plan_step_description = processor->getPlanStepDescription(); + processor_elem.plan_group = processor->getQueryPlanStepGroup(); + + processor_elem.processor_name = processor->getName(); + + processor_elem.elapsed_us = static_cast(processor->getElapsedNs() / 1000U); + processor_elem.input_wait_elapsed_us = static_cast(processor->getInputWaitElapsedNs() / 1000U); + processor_elem.output_wait_elapsed_us = static_cast(processor->getOutputWaitElapsedNs() / 1000U); + + auto stats = processor->getProcessorDataStats(); + processor_elem.input_rows = stats.input_rows; + processor_elem.input_bytes = stats.input_bytes; + processor_elem.output_rows = stats.output_rows; + processor_elem.output_bytes = stats.output_bytes; + + processors_profile_log->add(processor_elem); + } + } + } +} } diff --git a/src/Interpreters/ProcessorsProfileLog.h b/src/Interpreters/ProcessorsProfileLog.h index fbf52f45f56..9cc2ab6c7f0 100644 --- a/src/Interpreters/ProcessorsProfileLog.h +++ b/src/Interpreters/ProcessorsProfileLog.h @@ -50,4 +50,5 @@ public: using SystemLog::SystemLog; }; +void logProcessorProfile(ContextPtr context, const Processors & processors); } diff --git a/src/Interpreters/QueryMetricLog.cpp b/src/Interpreters/QueryMetricLog.cpp index 5ab3fe590e0..52d773b7d1b 100644 --- a/src/Interpreters/QueryMetricLog.cpp +++ b/src/Interpreters/QueryMetricLog.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -16,7 +17,6 @@ #include #include -#include namespace DB @@ -24,6 +24,15 @@ namespace DB static auto logger = getLogger("QueryMetricLog"); +String timePointToString(QueryMetricLog::TimePoint time) +{ + /// fmtlib supports subsecond formatting in 10.0.0. We're in 9.1.0, so we need to add the milliseconds ourselves. + auto seconds = std::chrono::time_point_cast(time); + auto microseconds = std::chrono::duration_cast(time - seconds).count(); + + return fmt::format("{:%Y.%m.%d %H:%M:%S}.{:06}", seconds, microseconds); +} + ColumnsDescription QueryMetricLogElement::getColumnsDescription() { ColumnsDescription result; @@ -87,36 +96,73 @@ void QueryMetricLog::shutdown() Base::shutdown(); } -void QueryMetricLog::startQuery(const String & query_id, TimePoint start_time, UInt64 interval_milliseconds) +void QueryMetricLog::collectMetric(const ProcessList & process_list, String query_id) { - QueryMetricLogStatus status; - status.interval_milliseconds = interval_milliseconds; - status.next_collect_time = start_time + std::chrono::milliseconds(interval_milliseconds); + auto current_time = std::chrono::system_clock::now(); + const auto query_info = process_list.getQueryInfo(query_id, false, true, false); + if (!query_info) + { + /// TODO: remove trace before 24.11 release after checking everything is fine on the CI + LOG_TRACE(logger, "Query {} is not running anymore, so we couldn't get its QueryStatusInfo", query_id); + return; + } + + LockGuard global_lock(queries_mutex); + auto it = queries.find(query_id); + + /// The query might have finished while the scheduled task is running. + if (it == queries.end()) + { + global_lock.unlock(); + /// TODO: remove trace before 24.11 release after checking everything is fine on the CI + LOG_TRACE(logger, "Query {} not found in the list. Finished while this collecting task was running", query_id); + return; + } + + auto & query_status = it->second; + if (!query_status.mutex) + { + global_lock.unlock(); + /// TODO: remove trace before 24.11 release after checking everything is fine on the CI + LOG_TRACE(logger, "Query {} finished while this collecting task was running", query_id); + return; + } + + LockGuard query_lock(query_status.getMutex()); + global_lock.unlock(); + + auto elem = query_status.createLogMetricElement(query_id, *query_info, current_time); + if (elem) + add(std::move(elem.value())); +} + +/// We use TSA_NO_THREAD_SAFETY_ANALYSIS to prevent TSA complaining that we're modifying the query_status fields +/// without locking the mutex. Since we're building it from scratch, there's no harm in not holding it. +/// If we locked it to make TSA happy, TSAN build would falsely complain about +/// lock-order-inversion (potential deadlock) +/// which is not a real issue since QueryMetricLogStatus's mutex cannot be locked by anything else +/// until we add it to the queries map. +void QueryMetricLog::startQuery(const String & query_id, TimePoint start_time, UInt64 interval_milliseconds) TSA_NO_THREAD_SAFETY_ANALYSIS +{ + QueryMetricLogStatus query_status; + QueryMetricLogStatusInfo & info = query_status.info; + info.interval_milliseconds = interval_milliseconds; + info.next_collect_time = start_time; auto context = getContext(); const auto & process_list = context->getProcessList(); - status.task = context->getSchedulePool().createTask("QueryMetricLog", [this, &process_list, query_id] { - auto current_time = std::chrono::system_clock::now(); - const auto query_info = process_list.getQueryInfo(query_id, false, true, false); - if (!query_info) - { - LOG_TRACE(logger, "Query {} is not running anymore, so we couldn't get its QueryStatusInfo", query_id); - return; - } - - auto elem = createLogMetricElement(query_id, *query_info, current_time); - if (elem) - add(std::move(elem.value())); + info.task = context->getSchedulePool().createTask("QueryMetricLog", [this, &process_list, query_id] { + collectMetric(process_list, query_id); }); - std::lock_guard lock(queries_mutex); - status.task->scheduleAfter(interval_milliseconds); - queries.emplace(query_id, std::move(status)); + LockGuard global_lock(queries_mutex); + query_status.scheduleNext(query_id); + queries.emplace(query_id, std::move(query_status)); } void QueryMetricLog::finishQuery(const String & query_id, TimePoint finish_time, QueryStatusInfoPtr query_info) { - std::unique_lock lock(queries_mutex); + LockGuard global_lock(queries_mutex); auto it = queries.find(query_id); /// finishQuery may be called from logExceptionBeforeStart when the query has not even started @@ -124,9 +170,19 @@ void QueryMetricLog::finishQuery(const String & query_id, TimePoint finish_time, if (it == queries.end()) return; + auto & query_status = it->second; + decltype(query_status.mutex) query_mutex; + LockGuard query_lock(query_status.getMutex()); + + /// Move the query mutex here so that we hold it until the end, after removing the query from queries. + query_mutex = std::move(query_status.mutex); + query_status.mutex = {}; + + global_lock.unlock(); + if (query_info) { - auto elem = createLogMetricElement(query_id, *query_info, finish_time, false); + auto elem = query_status.createLogMetricElement(query_id, *query_info, finish_time, false); if (elem) add(std::move(elem.value())); } @@ -139,51 +195,58 @@ void QueryMetricLog::finishQuery(const String & query_id, TimePoint finish_time, /// that order. { /// Take ownership of the task so that we can destroy it in this scope after unlocking `queries_mutex`. - auto task = std::move(it->second.task); + auto task = std::move(query_status.info.task); /// Build an empty task for the old task to make sure it does not lock any mutex on its destruction. - it->second.task = {}; + query_status.info.task = {}; + query_lock.unlock(); + global_lock.lock(); queries.erase(query_id); /// Ensure `queries_mutex` is unlocked before calling task's destructor at the end of this /// scope which will lock `exec_mutex`. - lock.unlock(); + global_lock.unlock(); } } -std::optional QueryMetricLog::createLogMetricElement(const String & query_id, const QueryStatusInfo & query_info, TimePoint query_info_time, bool schedule_next) +void QueryMetricLogStatus::scheduleNext(String query_id) { - /// fmtlib supports subsecond formatting in 10.0.0. We're in 9.1.0, so we need to add the milliseconds ourselves. - auto seconds = std::chrono::time_point_cast(query_info_time); - auto microseconds = std::chrono::duration_cast(query_info_time - seconds).count(); - LOG_DEBUG(logger, "Collecting query_metric_log for query {} with QueryStatusInfo from {:%Y.%m.%d %H:%M:%S}.{:06}. Schedule next: {}", query_id, seconds, microseconds, schedule_next); - - std::unique_lock lock(queries_mutex); - auto query_status_it = queries.find(query_id); - - /// The query might have finished while the scheduled task is running. - if (query_status_it == queries.end()) + info.next_collect_time += std::chrono::milliseconds(info.interval_milliseconds); + const auto now = std::chrono::system_clock::now(); + if (info.next_collect_time > now) { - lock.unlock(); - LOG_TRACE(logger, "Query {} finished already while this collecting task was running", query_id); - return {}; + const auto wait_time = std::chrono::duration_cast(info.next_collect_time - now).count(); + info.task->scheduleAfter(wait_time); } - - auto & query_status = query_status_it->second; - if (query_info_time <= query_status.last_collect_time) + else { - lock.unlock(); + LOG_TRACE(logger, "The next collecting task for query {} should have already run at {}. Scheduling it right now", + query_id, timePointToString(info.next_collect_time)); + info.task->schedule(); + } +} + +std::optional QueryMetricLogStatus::createLogMetricElement(const String & query_id, const QueryStatusInfo & query_info, TimePoint query_info_time, bool schedule_next) +{ + /// TODO: remove trace before 24.11 release after checking everything is fine on the CI + LOG_TRACE(logger, "Collecting query_metric_log for query {} and interval {} ms with QueryStatusInfo from {}. Next collection time: {}", + query_id, info.interval_milliseconds, timePointToString(query_info_time), + schedule_next ? timePointToString(info.next_collect_time + std::chrono::milliseconds(info.interval_milliseconds)) : "finished"); + + if (query_info_time <= info.last_collect_time) + { + /// TODO: remove trace before 24.11 release after checking everything is fine on the CI LOG_TRACE(logger, "Query {} has a more recent metrics collected. Skipping this one", query_id); return {}; } - query_status.last_collect_time = query_info_time; + info.last_collect_time = query_info_time; QueryMetricLogElement elem; elem.event_time = timeInSeconds(query_info_time); elem.event_time_microseconds = timeInMicroseconds(query_info_time); - elem.query_id = query_status_it->first; + elem.query_id = query_id; elem.memory_usage = query_info.memory_usage > 0 ? query_info.memory_usage : 0; elem.peak_memory_usage = query_info.peak_memory_usage > 0 ? query_info.peak_memory_usage : 0; @@ -192,7 +255,7 @@ std::optional QueryMetricLog::createLogMetricElement(cons for (ProfileEvents::Event i = ProfileEvents::Event(0), end = ProfileEvents::end(); i < end; ++i) { const auto & new_value = (*(query_info.profile_counters))[i]; - auto & old_value = query_status.last_profile_events[i]; + auto & old_value = info.last_profile_events[i]; /// Profile event counters are supposed to be monotonic. However, at least the `NetworkReceiveBytes` can be inaccurate. /// So, since in the future the counter should always have a bigger value than in the past, we skip this event. @@ -208,16 +271,13 @@ std::optional QueryMetricLog::createLogMetricElement(cons } else { - LOG_TRACE(logger, "Query {} has no profile counters", query_id); + /// TODO: remove trace before 24.11 release after checking everything is fine on the CI + LOG_DEBUG(logger, "Query {} has no profile counters", query_id); elem.profile_events = std::vector(ProfileEvents::end()); } if (schedule_next) - { - query_status.next_collect_time += std::chrono::milliseconds(query_status.interval_milliseconds); - const auto wait_time = std::chrono::duration_cast(query_status.next_collect_time - std::chrono::system_clock::now()).count(); - query_status.task->scheduleAfter(wait_time); - } + scheduleNext(query_id); return elem; } diff --git a/src/Interpreters/QueryMetricLog.h b/src/Interpreters/QueryMetricLog.h index 802cee7bf26..9371dfbb6b5 100644 --- a/src/Interpreters/QueryMetricLog.h +++ b/src/Interpreters/QueryMetricLog.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -11,11 +12,17 @@ #include #include +#include namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +}; + /** QueryMetricLogElement is a log of query metric values measured at regular time interval. */ @@ -34,7 +41,7 @@ struct QueryMetricLogElement void appendToBlock(MutableColumns & columns) const; }; -struct QueryMetricLogStatus +struct QueryMetricLogStatusInfo { UInt64 interval_milliseconds; std::chrono::system_clock::time_point last_collect_time; @@ -43,24 +50,47 @@ struct QueryMetricLogStatus BackgroundSchedulePool::TaskHolder task; }; +struct QueryMetricLogStatus +{ + using TimePoint = std::chrono::system_clock::time_point; + using Mutex = std::mutex; + + QueryMetricLogStatusInfo info TSA_GUARDED_BY(getMutex()); + + /// We need to be able to move it for the hash map, so we need to add an indirection here. + std::unique_ptr mutex = std::make_unique(); + + /// Return a reference to the mutex, used for Thread Sanitizer annotations. + Mutex & getMutex() const TSA_RETURN_CAPABILITY(mutex) + { + if (!mutex) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Mutex cannot be NULL"); + return *mutex; + } + + void scheduleNext(String query_id) TSA_REQUIRES(getMutex()); + std::optional createLogMetricElement(const String & query_id, const QueryStatusInfo & query_info, TimePoint query_info_time, bool schedule_next = true) TSA_REQUIRES(getMutex()); +}; + class QueryMetricLog : public SystemLog { using SystemLog::SystemLog; - using TimePoint = std::chrono::system_clock::time_point; using Base = SystemLog; public: + using TimePoint = std::chrono::system_clock::time_point; + void shutdown() final; - // Both startQuery and finishQuery are called from the thread that executes the query + /// Both startQuery and finishQuery are called from the thread that executes the query. void startQuery(const String & query_id, TimePoint start_time, UInt64 interval_milliseconds); void finishQuery(const String & query_id, TimePoint finish_time, QueryStatusInfoPtr query_info = nullptr); private: - std::optional createLogMetricElement(const String & query_id, const QueryStatusInfo & query_info, TimePoint query_info_time, bool schedule_next = true); + void collectMetric(const ProcessList & process_list, String query_id); - std::recursive_mutex queries_mutex; - std::unordered_map queries; + std::mutex queries_mutex; + std::unordered_map queries TSA_GUARDED_BY(queries_mutex); }; } diff --git a/src/Interpreters/QueryNormalizer.cpp b/src/Interpreters/QueryNormalizer.cpp index a8639906aad..bba30fb5194 100644 --- a/src/Interpreters/QueryNormalizer.cpp +++ b/src/Interpreters/QueryNormalizer.cpp @@ -161,7 +161,13 @@ void QueryNormalizer::visit(ASTTablesInSelectQueryElement & node, const ASTPtr & { auto & join = node.table_join->as(); if (join.on_expression) + { + ASTPtr original_on_expression = join.on_expression; visit(join.on_expression, data); + if (join.on_expression != original_on_expression) + join.children = { join.on_expression }; + } + } } diff --git a/src/Interpreters/RewriteArrayExistsFunctionVisitor.cpp b/src/Interpreters/RewriteArrayExistsFunctionVisitor.cpp index 22ce91d8c67..2890357494d 100644 --- a/src/Interpreters/RewriteArrayExistsFunctionVisitor.cpp +++ b/src/Interpreters/RewriteArrayExistsFunctionVisitor.cpp @@ -6,6 +6,12 @@ namespace DB { + +namespace ErrorCode +{ +extern const int LOGICAL_ERROR; +} + void RewriteArrayExistsFunctionMatcher::visit(ASTPtr & ast, Data & data) { if (auto * func = ast->as()) @@ -20,21 +26,21 @@ void RewriteArrayExistsFunctionMatcher::visit(ASTPtr & ast, Data & data) if (join->using_expression_list) { auto * it = std::find(join->children.begin(), join->children.end(), join->using_expression_list); + if (it == join->children.end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not find join->using_expression_list in '{}'", join->formatForLogging()); visit(join->using_expression_list, data); - - if (it && *it != join->using_expression_list) - *it = join->using_expression_list; + *it = join->using_expression_list; } if (join->on_expression) { auto * it = std::find(join->children.begin(), join->children.end(), join->on_expression); + if (it == join->children.end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not find join->on_expression in '{}'", join->formatForLogging()); visit(join->on_expression, data); - - if (it && *it != join->on_expression) - *it = join->on_expression; + *it = join->on_expression; } } } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 9250c069283..0bc1d4956a1 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -117,7 +117,6 @@ namespace Setting extern const SettingsOverflowMode join_overflow_mode; extern const SettingsString log_comment; extern const SettingsBool log_formatted_queries; - extern const SettingsBool log_processors_profiles; extern const SettingsBool log_profile_events; extern const SettingsUInt64 log_queries_cut_to_length; extern const SettingsBool log_queries; @@ -506,6 +505,7 @@ void logQueryFinish( auto time_now = std::chrono::system_clock::now(); QueryStatusInfo info = process_list_elem->getInfo(true, settings[Setting::log_profile_events]); + logQueryMetricLogFinish(context, internal, elem.client_info.current_query_id, time_now, std::make_shared(info)); elem.type = QueryLogElementType::QUERY_FINISH; addStatusInfoToQueryLogElement(elem, info, query_ast, context); @@ -551,53 +551,8 @@ void logQueryFinish( if (auto query_log = context->getQueryLog()) query_log->add(elem); } - if (settings[Setting::log_processors_profiles]) - { - if (auto processors_profile_log = context->getProcessorsProfileLog()) - { - ProcessorProfileLogElement processor_elem; - processor_elem.event_time = elem.event_time; - processor_elem.event_time_microseconds = elem.event_time_microseconds; - processor_elem.initial_query_id = elem.client_info.initial_query_id; - processor_elem.query_id = elem.client_info.current_query_id; - auto get_proc_id = [](const IProcessor & proc) -> UInt64 { return reinterpret_cast(&proc); }; - - for (const auto & processor : query_pipeline.getProcessors()) - { - std::vector parents; - for (const auto & port : processor->getOutputs()) - { - if (!port.isConnected()) - continue; - const IProcessor & next = port.getInputPort().getProcessor(); - parents.push_back(get_proc_id(next)); - } - - processor_elem.id = get_proc_id(*processor); - processor_elem.parent_ids = std::move(parents); - - processor_elem.plan_step = reinterpret_cast(processor->getQueryPlanStep()); - processor_elem.plan_step_name = processor->getPlanStepName(); - processor_elem.plan_step_description = processor->getPlanStepDescription(); - processor_elem.plan_group = processor->getQueryPlanStepGroup(); - - processor_elem.processor_name = processor->getName(); - - processor_elem.elapsed_us = static_cast(processor->getElapsedNs() / 1000U); - processor_elem.input_wait_elapsed_us = static_cast(processor->getInputWaitElapsedNs() / 1000U); - processor_elem.output_wait_elapsed_us = static_cast(processor->getOutputWaitElapsedNs() / 1000U); - - auto stats = processor->getProcessorDataStats(); - processor_elem.input_rows = stats.input_rows; - processor_elem.input_bytes = stats.input_bytes; - processor_elem.output_rows = stats.output_rows; - processor_elem.output_bytes = stats.output_bytes; - - processors_profile_log->add(processor_elem); - } - } - } + logProcessorProfile(context, query_pipeline.getProcessors()); logQueryMetricLogFinish(context, internal, elem.client_info.current_query_id, time_now, std::make_shared(info)); } @@ -669,6 +624,7 @@ void logQueryException( { elem.query_duration_ms = start_watch.elapsedMilliseconds(); } + logQueryMetricLogFinish(context, internal, elem.client_info.current_query_id, time_now, info); elem.query_cache_usage = QueryCache::Usage::None; @@ -698,8 +654,6 @@ void logQueryException( query_span->addAttribute("clickhouse.exception_code", elem.exception_code); query_span->finish(); } - - logQueryMetricLogFinish(context, internal, elem.client_info.current_query_id, time_now, info); } void logExceptionBeforeStart( @@ -753,6 +707,8 @@ void logExceptionBeforeStart( elem.client_info = context->getClientInfo(); + logQueryMetricLogFinish(context, false, elem.client_info.current_query_id, std::chrono::system_clock::now(), nullptr); + elem.log_comment = settings[Setting::log_comment]; if (elem.log_comment.size() > settings[Setting::max_query_size]) elem.log_comment.resize(settings[Setting::max_query_size]); @@ -797,8 +753,6 @@ void logExceptionBeforeStart( ProfileEvents::increment(ProfileEvents::FailedInsertQuery); } } - - logQueryMetricLogFinish(context, false, elem.client_info.current_query_id, std::chrono::system_clock::now(), nullptr); } void validateAnalyzerSettings(ASTPtr ast, bool context_value) diff --git a/src/Interpreters/fuzzers/CMakeLists.txt b/src/Interpreters/fuzzers/CMakeLists.txt index 3317bba7e30..174fae299b7 100644 --- a/src/Interpreters/fuzzers/CMakeLists.txt +++ b/src/Interpreters/fuzzers/CMakeLists.txt @@ -3,5 +3,6 @@ target_link_libraries(execute_query_fuzzer PRIVATE dbms clickhouse_table_functions clickhouse_aggregate_functions + clickhouse_functions clickhouse_dictionaries clickhouse_dictionaries_embedded) diff --git a/src/Interpreters/tests/gtest_filecache.cpp b/src/Interpreters/tests/gtest_filecache.cpp index 60436604f70..caf4fa8402f 100644 --- a/src/Interpreters/tests/gtest_filecache.cpp +++ b/src/Interpreters/tests/gtest_filecache.cpp @@ -253,7 +253,7 @@ void download(FileSegment & file_segment) download(cache_base_path, file_segment); ASSERT_EQ(file_segment.state(), State::DOWNLOADING); - file_segment.complete(); + file_segment.complete(false); ASSERT_EQ(file_segment.state(), State::DOWNLOADED); } @@ -263,7 +263,7 @@ void assertDownloadFails(FileSegment & file_segment) ASSERT_EQ(file_segment.getDownloadedSize(), 0); std::string failure_reason; ASSERT_FALSE(file_segment.reserve(file_segment.range().size(), 1000, failure_reason)); - file_segment.complete(); + file_segment.complete(false); } void download(const HolderPtr & holder) @@ -972,7 +972,7 @@ try ASSERT_TRUE(segment->getOrSetDownloader() == DB::FileSegment::getCallerId()); ASSERT_TRUE(segment->reserve(segment->range().size(), 1000, failure_reason)); download(*segment); - segment->complete(); + segment->complete(false); } } diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index e7c3fdbb548..1c7d72bafcc 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -128,4 +128,14 @@ void ASTColumnDeclaration::formatImpl(const FormatSettings & format_settings, Fo } } +void ASTColumnDeclaration::forEachPointerToChild(std::function f) +{ + f(reinterpret_cast(&default_expression)); + f(reinterpret_cast(&comment)); + f(reinterpret_cast(&codec)); + f(reinterpret_cast(&statistics_desc)); + f(reinterpret_cast(&ttl)); + f(reinterpret_cast(&collation)); + f(reinterpret_cast(&settings)); +} } diff --git a/src/Parsers/ASTColumnDeclaration.h b/src/Parsers/ASTColumnDeclaration.h index 914916d5074..0c5076f0201 100644 --- a/src/Parsers/ASTColumnDeclaration.h +++ b/src/Parsers/ASTColumnDeclaration.h @@ -29,6 +29,9 @@ public: ASTPtr clone() const override; void formatImpl(const FormatSettings & format_settings, FormatState & state, FormatStateStacked frame) const override; + +protected: + void forEachPointerToChild(std::function f) override; }; } diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 53d44e2f325..11cfe2e584e 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -724,7 +724,10 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format { if (secret_arguments.are_named) { - assert_cast(argument.get())->arguments->children[0]->formatImpl(settings, state, nested_dont_need_parens); + if (const auto * func_ast = typeid_cast(argument.get())) + func_ast->arguments->children[0]->formatImpl(settings, state, nested_dont_need_parens); + else + argument->formatImpl(settings, state, nested_dont_need_parens); settings.ostr << (settings.hilite ? hilite_operator : "") << " = " << (settings.hilite ? hilite_none : ""); } if (!secret_arguments.replacement.empty()) diff --git a/src/Parsers/ASTOrderByElement.cpp b/src/Parsers/ASTOrderByElement.cpp index 09193a8b5e1..d87c296d398 100644 --- a/src/Parsers/ASTOrderByElement.cpp +++ b/src/Parsers/ASTOrderByElement.cpp @@ -54,6 +54,11 @@ void ASTOrderByElement::formatImpl(const FormatSettings & settings, FormatState settings.ostr << (settings.hilite ? hilite_keyword : "") << " STEP " << (settings.hilite ? hilite_none : ""); fill_step->formatImpl(settings, state, frame); } + if (auto fill_staleness = getFillStaleness()) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " STALENESS " << (settings.hilite ? hilite_none : ""); + fill_staleness->formatImpl(settings, state, frame); + } } } diff --git a/src/Parsers/ASTOrderByElement.h b/src/Parsers/ASTOrderByElement.h index 6edf84d7bde..4dc35dac217 100644 --- a/src/Parsers/ASTOrderByElement.h +++ b/src/Parsers/ASTOrderByElement.h @@ -18,6 +18,7 @@ private: FILL_FROM, FILL_TO, FILL_STEP, + FILL_STALENESS, }; public: @@ -32,12 +33,14 @@ public: void setFillFrom(ASTPtr node) { setChild(Child::FILL_FROM, node); } void setFillTo(ASTPtr node) { setChild(Child::FILL_TO, node); } void setFillStep(ASTPtr node) { setChild(Child::FILL_STEP, node); } + void setFillStaleness(ASTPtr node) { setChild(Child::FILL_STALENESS, node); } /** Collation for locale-specific string comparison. If empty, then sorting done by bytes. */ ASTPtr getCollation() const { return getChild(Child::COLLATION); } ASTPtr getFillFrom() const { return getChild(Child::FILL_FROM); } ASTPtr getFillTo() const { return getChild(Child::FILL_TO); } ASTPtr getFillStep() const { return getChild(Child::FILL_STEP); } + ASTPtr getFillStaleness() const { return getChild(Child::FILL_STALENESS); } String getID(char) const override { return "OrderByElement"; } diff --git a/src/Parsers/Access/ParserGrantQuery.cpp b/src/Parsers/Access/ParserGrantQuery.cpp index e29cf11273b..4a0d24559a3 100644 --- a/src/Parsers/Access/ParserGrantQuery.cpp +++ b/src/Parsers/Access/ParserGrantQuery.cpp @@ -155,6 +155,9 @@ namespace for (auto & [access_flags, columns] : access_and_columns) { + if (wildcard && !columns.empty()) + return false; + AccessRightsElement element; element.access_flags = access_flags; element.columns = std::move(columns); diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index dd0ba91d428..c02f8d06323 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -546,6 +546,7 @@ namespace DB MR_MACROS(YY, "YY") \ MR_MACROS(YYYY, "YYYY") \ MR_MACROS(ZKPATH, "ZKPATH") \ + MR_MACROS(STALENESS, "STALENESS") \ /// The list of keywords where underscore is intentional #define APPLY_FOR_PARSER_KEYWORDS_WITH_UNDERSCORES(MR_MACROS) \ diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 31efcb16f02..ad062d27a37 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -2178,6 +2178,7 @@ bool ParserOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expect ParserKeyword from(Keyword::FROM); ParserKeyword to(Keyword::TO); ParserKeyword step(Keyword::STEP); + ParserKeyword staleness(Keyword::STALENESS); ParserStringLiteral collate_locale_parser; ParserExpressionWithOptionalAlias exp_parser(false); @@ -2219,6 +2220,7 @@ bool ParserOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expect ASTPtr fill_from; ASTPtr fill_to; ASTPtr fill_step; + ASTPtr fill_staleness; if (with_fill.ignore(pos, expected)) { has_with_fill = true; @@ -2230,6 +2232,9 @@ bool ParserOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expect if (step.ignore(pos, expected) && !exp_parser.parse(pos, fill_step, expected)) return false; + + if (staleness.ignore(pos, expected) && !exp_parser.parse(pos, fill_staleness, expected)) + return false; } auto elem = std::make_shared(); @@ -2244,6 +2249,7 @@ bool ParserOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expect elem->setFillFrom(fill_from); elem->setFillTo(fill_to); elem->setFillStep(fill_step); + elem->setFillStaleness(fill_staleness); node = elem; diff --git a/src/Parsers/fuzzers/CMakeLists.txt b/src/Parsers/fuzzers/CMakeLists.txt index 903319d733c..c829c26a805 100644 --- a/src/Parsers/fuzzers/CMakeLists.txt +++ b/src/Parsers/fuzzers/CMakeLists.txt @@ -2,10 +2,10 @@ clickhouse_add_executable(lexer_fuzzer lexer_fuzzer.cpp ${SRCS}) target_link_libraries(lexer_fuzzer PRIVATE clickhouse_parsers) clickhouse_add_executable(select_parser_fuzzer select_parser_fuzzer.cpp ${SRCS}) -target_link_libraries(select_parser_fuzzer PRIVATE clickhouse_parsers dbms) +target_link_libraries(select_parser_fuzzer PRIVATE clickhouse_parsers clickhouse_functions dbms) clickhouse_add_executable(create_parser_fuzzer create_parser_fuzzer.cpp ${SRCS}) -target_link_libraries(create_parser_fuzzer PRIVATE clickhouse_parsers dbms) +target_link_libraries(create_parser_fuzzer PRIVATE clickhouse_parsers clickhouse_functions dbms) add_subdirectory(codegen_fuzzer) diff --git a/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt b/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt index 74fdcff79f7..ee17e03fce2 100644 --- a/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt +++ b/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt @@ -47,4 +47,4 @@ target_compile_options (codegen_select_fuzzer PRIVATE -Wno-newline-eof) target_link_libraries(protoc ch_contrib::fuzzer) target_include_directories(codegen_select_fuzzer SYSTEM BEFORE PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") -target_link_libraries(codegen_select_fuzzer PRIVATE ch_contrib::protobuf_mutator ch_contrib::protoc dbms) +target_link_libraries(codegen_select_fuzzer PRIVATE ch_contrib::protobuf_mutator ch_contrib::protoc clickhouse_functions dbms) diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index aea304e0ecc..2cb2a242c35 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -391,6 +391,9 @@ public: if (sort_node.hasFillStep()) buffer << " STEP " << calculateActionNodeName(sort_node.getFillStep()); + + if (sort_node.hasFillStaleness()) + buffer << " STALENESS " << calculateActionNodeName(sort_node.getFillStaleness()); } if (i + 1 != order_by_nodes_size) @@ -801,7 +804,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi auto lambda_node_name = calculateActionNodeName(node, *planner_context); auto function_capture = std::make_shared( - lambda_actions, captured_column_names, lambda_arguments_names_and_types, lambda_node.getExpression()->getResultType(), lambda_expression_node_name); + lambda_actions, captured_column_names, lambda_arguments_names_and_types, lambda_node.getExpression()->getResultType(), lambda_expression_node_name, true); // TODO: Pass IFunctionBase here not FunctionCaptureOverloadResolver. const auto * actions_node = actions_stack[level].addFunctionIfNecessary(lambda_node_name, std::move(lambda_children), function_capture); diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 39c1352c9cf..5c153f6db39 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -1555,10 +1555,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ SortingStep::Settings sort_settings(*query_context); auto sorting_step = std::make_unique( - plan.getCurrentHeader(), - std::move(sort_description), - 0 /*limit*/, - sort_settings); + plan.getCurrentHeader(), std::move(sort_description), 0 /*limit*/, sort_settings, true /*is_sorting_for_merge_join*/); sorting_step->setStepDescription(fmt::format("Sort {} before JOIN", join_table_side)); plan.addStep(std::move(sorting_step)); }; diff --git a/src/Planner/PlannerSorting.cpp b/src/Planner/PlannerSorting.cpp index af51afdef13..9476ae348c5 100644 --- a/src/Planner/PlannerSorting.cpp +++ b/src/Planner/PlannerSorting.cpp @@ -43,7 +43,7 @@ std::pair extractWithFillValue(const QueryTreeNodePtr & node return result; } -std::pair> extractWithFillStepValue(const QueryTreeNodePtr & node) +std::pair> extractWithFillValueWithIntervalKind(const QueryTreeNodePtr & node) { const auto & constant_node = node->as(); @@ -77,7 +77,7 @@ FillColumnDescription extractWithFillDescription(const SortNode & sort_node) if (sort_node.hasFillStep()) { - auto extract_result = extractWithFillStepValue(sort_node.getFillStep()); + auto extract_result = extractWithFillValueWithIntervalKind(sort_node.getFillStep()); fill_column_description.fill_step = std::move(extract_result.first); fill_column_description.step_kind = std::move(extract_result.second); } @@ -87,16 +87,36 @@ FillColumnDescription extractWithFillDescription(const SortNode & sort_node) fill_column_description.fill_step = Field(direction_value); } + if (sort_node.getFillStaleness()) + { + auto extract_result = extractWithFillValueWithIntervalKind(sort_node.getFillStaleness()); + fill_column_description.fill_staleness = std::move(extract_result.first); + fill_column_description.staleness_kind = std::move(extract_result.second); + } + + /////////////////////////////////// + if (applyVisitor(FieldVisitorAccurateEquals(), fill_column_description.fill_step, Field{0})) throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, "WITH FILL STEP value cannot be zero"); + if (sort_node.hasFillStaleness()) + { + if (sort_node.hasFillFrom()) + throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, + "WITH FILL STALENESS cannot be used together with WITH FILL FROM"); + } + if (sort_node.getSortDirection() == SortDirection::ASCENDING) { if (applyVisitor(FieldVisitorAccurateLess(), fill_column_description.fill_step, Field{0})) throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, "WITH FILL STEP value cannot be negative for sorting in ascending direction"); + if (applyVisitor(FieldVisitorAccurateLess(), fill_column_description.fill_staleness, Field{0})) + throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, + "WITH FILL STALENESS value cannot be negative for sorting in ascending direction"); + if (!fill_column_description.fill_from.isNull() && !fill_column_description.fill_to.isNull() && applyVisitor(FieldVisitorAccurateLess(), fill_column_description.fill_to, fill_column_description.fill_from)) { @@ -110,6 +130,10 @@ FillColumnDescription extractWithFillDescription(const SortNode & sort_node) throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, "WITH FILL STEP value cannot be positive for sorting in descending direction"); + if (applyVisitor(FieldVisitorAccurateLess(), Field{0}, fill_column_description.fill_staleness)) + throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, + "WITH FILL STALENESS value cannot be positive for sorting in descending direction"); + if (!fill_column_description.fill_from.isNull() && !fill_column_description.fill_to.isNull() && applyVisitor(FieldVisitorAccurateLess(), fill_column_description.fill_from, fill_column_description.fill_to)) { diff --git a/src/Planner/findParallelReplicasQuery.cpp b/src/Planner/findParallelReplicasQuery.cpp index b97a9a36381..fce86a6cda0 100644 --- a/src/Planner/findParallelReplicasQuery.cpp +++ b/src/Planner/findParallelReplicasQuery.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -170,12 +171,25 @@ const QueryNode * findQueryForParallelReplicas( const std::unordered_map & mapping, const Settings & settings) { - const QueryPlan::Node * prev_checked_node = nullptr; + struct Frame + { + const QueryPlan::Node * node = nullptr; + /// Below we will check subqueries from `stack` to find outermost subquery that could be executed remotely. + /// Currently traversal algorithm considers only steps with 0 or 1 children and JOIN specifically. + /// When we found some step that requires finalization on the initiator (e.g. GROUP BY) there are two options: + /// 1. If plan looks like a single path (e.g. AggregatingStep -> ExpressionStep -> Reading) we can execute + /// current subquery as a whole with replicas. + /// 2. If we were inside JOIN we cannot offload the whole subquery to replicas because at least one side + /// of the JOIN needs to be finalized on the initiator. + /// So this flag is used to track what subquery to return once we hit a step that needs finalization. + bool inside_join = false; + }; + const QueryNode * res = nullptr; while (!stack.empty()) { - const QueryNode * subquery_node = stack.top(); + const QueryNode * const subquery_node = stack.top(); stack.pop(); auto it = mapping.find(subquery_node); @@ -183,22 +197,21 @@ const QueryNode * findQueryForParallelReplicas( if (it == mapping.end()) break; - const QueryPlan::Node * curr_node = it->second; - const QueryPlan::Node * next_node_to_check = curr_node; + std::stack nodes_to_check; + nodes_to_check.push({.node = it->second, .inside_join = false}); bool can_distribute_full_node = true; + bool currently_inside_join = false; - while (next_node_to_check && next_node_to_check != prev_checked_node) + while (!nodes_to_check.empty()) { + const auto & [next_node_to_check, inside_join] = nodes_to_check.top(); + nodes_to_check.pop(); const auto & children = next_node_to_check->children; auto * step = next_node_to_check->step.get(); if (children.empty()) { - /// Found a source step. This should be possible only in the first iteration. - if (prev_checked_node) - return nullptr; - - next_node_to_check = nullptr; + /// Found a source step. } else if (children.size() == 1) { @@ -206,12 +219,19 @@ const QueryNode * findQueryForParallelReplicas( const auto * filter = typeid_cast(step); const auto * creating_sets = typeid_cast(step); - bool allowed_creating_sets = settings[Setting::parallel_replicas_allow_in_with_subquery] && creating_sets; + const bool allowed_creating_sets = settings[Setting::parallel_replicas_allow_in_with_subquery] && creating_sets; - if (!expression && !filter && !allowed_creating_sets) + const auto * sorting = typeid_cast(step); + /// Sorting for merge join is supposed to be done locally before join itself, so it doesn't need finalization. + const bool allowed_sorting = sorting && sorting->isSortingForMergeJoin(); + + if (!expression && !filter && !allowed_creating_sets && !allowed_sorting) + { can_distribute_full_node = false; + currently_inside_join = inside_join; + } - next_node_to_check = children.front(); + nodes_to_check.push({.node = children.front(), .inside_join = inside_join}); } else { @@ -221,12 +241,11 @@ const QueryNode * findQueryForParallelReplicas( if (!join) return res; - next_node_to_check = children.front(); + for (const auto & child : children) + nodes_to_check.push({.node = child, .inside_join = true}); } } - /// Current node contains steps like GROUP BY / DISTINCT - /// Will try to execute query up to WithMergableStage if (!can_distribute_full_node) { /// Current query node does not contain subqueries. @@ -234,12 +253,11 @@ const QueryNode * findQueryForParallelReplicas( if (!res) return nullptr; - return subquery_node; + return currently_inside_join ? res : subquery_node; } /// Query is simple enough to be fully distributed. res = subquery_node; - prev_checked_node = curr_node; } return res; diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp index 81df330ffb5..26aa3555c2b 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp @@ -1534,15 +1534,23 @@ static ColumnWithTypeAndName readColumnWithDateData( for (size_t i = 0; i < orc_int_column->numElements; ++i) { - Int32 days_num = static_cast(orc_int_column->data[i]); - if (check_date_range && (days_num > DATE_LUT_MAX_EXTEND_DAY_NUM || days_num < -DAYNUM_OFFSET_EPOCH)) - throw Exception( - ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, - "Input value {} of a column \"{}\" exceeds the range of type Date32", - days_num, - column_name); + if (!orc_int_column->hasNulls || orc_int_column->notNull[i]) + { + Int32 days_num = static_cast(orc_int_column->data[i]); + if (check_date_range && (days_num > DATE_LUT_MAX_EXTEND_DAY_NUM || days_num < -DAYNUM_OFFSET_EPOCH)) + throw Exception( + ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, + "Input value {} of a column \"{}\" exceeds the range of type Date32", + days_num, + column_name); - column_data.push_back(days_num); + column_data.push_back(days_num); + } + else + { + /// ORC library doesn't guarantee that orc_int_column->data[i] is initialized to zero when orc_int_column->notNull[i] is false since https://github.com/ClickHouse/ClickHouse/pull/69473 + column_data.push_back(0); + } } return {std::move(internal_column), internal_type, column_name}; diff --git a/src/Processors/Formats/Impl/ParquetMetadataInputFormat.cpp b/src/Processors/Formats/Impl/ParquetMetadataInputFormat.cpp index 7fd6e93dd80..8264b565e39 100644 --- a/src/Processors/Formats/Impl/ParquetMetadataInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetMetadataInputFormat.cpp @@ -92,8 +92,9 @@ static NamesAndTypesList getHeaderForParquetMetadata() std::make_shared(std::make_shared()), std::make_shared(std::make_shared())}, Names{"num_values", "null_count", "distinct_count", "min", "max"}), + DataTypeFactory::instance().get("Bool"), }, - Names{"name", "path", "total_compressed_size", "total_uncompressed_size", "have_statistics", "statistics"}))}, + Names{"name", "path", "total_compressed_size", "total_uncompressed_size", "have_statistics", "statistics", "have_bloom_filter"}))}, Names{"num_columns", "num_rows", "total_uncompressed_size", "total_compressed_size", "columns"}))}, }; return names_and_types; @@ -350,6 +351,8 @@ void ParquetMetadataInputFormat::fillColumnChunksMetadata(const std::unique_ptr< fillColumnStatistics(column_chunk_metadata->statistics(), tuple_column.getColumn(5), row_group_metadata->schema()->Column(column_i)->type_length()); else tuple_column.getColumn(5).insertDefault(); + bool have_bloom_filter = column_chunk_metadata->bloom_filter_offset().has_value(); + assert_cast(tuple_column.getColumn(6)).insertValue(have_bloom_filter); } array_column.getOffsets().push_back(tuple_column.size()); } diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp index ff1a048029d..e8b55ea423b 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -352,7 +353,8 @@ void PrettyBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port_kind } writeCString(grid_symbols.bar, out); - writeReadableNumberTip(chunk); + if (readable_number_tip) + writeReadableNumberTipIfSingleValue(out, chunk, format_settings, color); writeCString("\n", out); } @@ -392,72 +394,6 @@ void PrettyBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port_kind } -static String highlightDigitGroups(String source) -{ - if (source.size() <= 4) - return source; - - bool is_regular_number = true; - size_t num_digits_before_decimal = 0; - for (auto c : source) - { - if (c == '-' || c == ' ') - continue; - if (c == '.') - break; - if (c >= '0' && c <= '9') - { - ++num_digits_before_decimal; - } - else - { - is_regular_number = false; - break; - } - } - - if (!is_regular_number || num_digits_before_decimal <= 4) - return source; - - String result; - size_t size = source.size(); - result.reserve(2 * size); - - bool before_decimal = true; - size_t digit_num = 0; - for (size_t i = 0; i < size; ++i) - { - auto c = source[i]; - if (before_decimal && c >= '0' && c <= '9') - { - ++digit_num; - size_t offset = num_digits_before_decimal - digit_num; - if (offset && offset % 3 == 0) - { - result += "\033[4m"; - result += c; - result += "\033[0m"; - } - else - { - result += c; - } - } - else if (c == '.') - { - before_decimal = false; - result += c; - } - else - { - result += c; - } - } - - return result; -} - - void PrettyBlockOutputFormat::writeValueWithPadding( const IColumn & column, const ISerialization & serialization, size_t row_num, size_t value_width, size_t pad_to_width, size_t cut_to_width, bool align_right, bool is_number) @@ -553,30 +489,6 @@ void PrettyBlockOutputFormat::writeSuffix() } } -void PrettyBlockOutputFormat::writeReadableNumberTip(const Chunk & chunk) -{ - const auto & columns = chunk.getColumns(); - auto is_single_number = readable_number_tip && chunk.getNumRows() == 1 && chunk.getNumColumns() == 1; - if (!is_single_number) - return; - - if (columns[0]->isNullAt(0)) - return; - - auto value = columns[0]->getFloat64(0); - auto threshold = format_settings.pretty.output_format_pretty_single_large_number_tip_threshold; - - if (threshold && isFinite(value) && abs(value) > threshold) - { - if (color) - writeCString("\033[90m", out); - writeCString(" -- ", out); - formatReadableQuantity(value, out, 2); - if (color) - writeCString("\033[0m", out); - } -} - void registerOutputFormatPretty(FormatFactory & factory) { registerPrettyFormatWithNoEscapesAndMonoBlock(factory, "Pretty"); diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h index 698efecd4b2..81bd0e6632d 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h @@ -38,7 +38,6 @@ protected: virtual void writeChunk(const Chunk & chunk, PortKind port_kind); void writeMonoChunkIfNeeded(); void writeSuffix() override; - void writeReadableNumberTip(const Chunk & chunk); void onRowsReadBeforeUpdate() override { total_rows = getRowsReadBefore(); } @@ -56,9 +55,9 @@ protected: } bool color; + bool readable_number_tip = false; private: - bool readable_number_tip = false; bool mono_block; /// For mono_block == true only Chunk mono_chunk; diff --git a/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp index 57ec23e7e3b..1e4f784bc71 100644 --- a/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -189,7 +190,8 @@ void PrettyCompactBlockOutputFormat::writeRow( } writeCString(grid_symbols.bar, out); - writeReadableNumberTip(chunk); + if (readable_number_tip) + writeReadableNumberTipIfSingleValue(out, chunk, format_settings, color); writeCString("\n", out); } diff --git a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp index 0a594b54b12..5b481099e41 100644 --- a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -102,7 +103,8 @@ void PrettySpaceBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port writeValueWithPadding( *columns[column], *serializations[column], row, cur_width, max_widths[column], cut_to_width, type.shouldAlignRightInPrettyFormats(), isNumber(type)); } - writeReadableNumberTip(chunk); + if (readable_number_tip) + writeReadableNumberTipIfSingleValue(out, chunk, format_settings, color); writeChar('\n', out); } diff --git a/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp b/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp index 4852af9f0c8..7b0135b3ae4 100644 --- a/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp @@ -4,7 +4,10 @@ #include #include #include +#include #include +#include +#include namespace DB @@ -14,6 +17,8 @@ VerticalRowOutputFormat::VerticalRowOutputFormat( WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_) : IRowOutputFormat(header_, out_), format_settings(format_settings_) { + color = format_settings.pretty.color == 1 || (format_settings.pretty.color == 2 && format_settings.is_writing_to_terminal); + const auto & sample = getPort(PortKind::Main).getHeader(); size_t columns = sample.columns(); @@ -31,6 +36,7 @@ VerticalRowOutputFormat::VerticalRowOutputFormat( } names_and_paddings.resize(columns); + is_number.resize(columns); for (size_t i = 0; i < columns; ++i) { WriteBufferFromString buf(names_and_paddings[i]); @@ -42,6 +48,7 @@ VerticalRowOutputFormat::VerticalRowOutputFormat( { size_t new_size = max_name_width - name_widths[i] + names_and_paddings[i].size(); names_and_paddings[i].resize(new_size, ' '); + is_number[i] = isNumber(removeNullable(recursiveRemoveLowCardinality(sample.getByPosition(i).type))); } } @@ -61,7 +68,26 @@ void VerticalRowOutputFormat::writeField(const IColumn & column, const ISerializ void VerticalRowOutputFormat::writeValue(const IColumn & column, const ISerialization & serialization, size_t row_num) const { - serialization.serializeText(column, row_num, out, format_settings); + if (color && format_settings.pretty.highlight_digit_groups && is_number[field_number]) + { + String serialized_value; + { + WriteBufferFromString buf(serialized_value); + serialization.serializeText(column, row_num, buf, format_settings); + } + + /// Highlight groups of thousands. + serialized_value = highlightDigitGroups(serialized_value); + out.write(serialized_value.data(), serialized_value.size()); + } + else + { + serialization.serializeText(column, row_num, out, format_settings); + } + + /// Write a tip. + if (is_number[field_number]) + writeReadableNumberTip(out, column, row_num, format_settings, color); } diff --git a/src/Processors/Formats/Impl/VerticalRowOutputFormat.h b/src/Processors/Formats/Impl/VerticalRowOutputFormat.h index 5870c3503fc..6fe79adc9be 100644 --- a/src/Processors/Formats/Impl/VerticalRowOutputFormat.h +++ b/src/Processors/Formats/Impl/VerticalRowOutputFormat.h @@ -56,6 +56,9 @@ private: using NamesAndPaddings = std::vector; NamesAndPaddings names_and_paddings; + + std::vector is_number; + bool color; }; } diff --git a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp index cbad6813fbc..0e950094499 100644 --- a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp +++ b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp @@ -1,5 +1,5 @@ #include -#include +#include namespace DB @@ -24,7 +24,12 @@ void IMergingAlgorithmWithDelayedChunk::initializeQueue(Inputs inputs) continue; cursors[source_num] = SortCursorImpl( - header, current_inputs[source_num].chunk.getColumns(), description, source_num, current_inputs[source_num].permutation); + header, + current_inputs[source_num].chunk.getColumns(), + current_inputs[source_num].chunk.getNumRows(), + description, + source_num, + current_inputs[source_num].permutation); inputs_origin_merge_tree_part_level[source_num] = getPartLevelFromChunk(current_inputs[source_num].chunk); } @@ -41,7 +46,7 @@ void IMergingAlgorithmWithDelayedChunk::updateCursor(Input & input, size_t sourc last_chunk_sort_columns = std::move(cursors[source_num].sort_columns); current_input.swap(input); - cursors[source_num].reset(current_input.chunk.getColumns(), header, current_input.permutation); + cursors[source_num].reset(current_input.chunk.getColumns(), header, current_input.chunk.getNumRows(), current_input.permutation); inputs_origin_merge_tree_part_level[source_num] = getPartLevelFromChunk(current_input.chunk); diff --git a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp index 47b7ddf38dc..e47549e4a76 100644 --- a/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp +++ b/src/Processors/Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp @@ -1,5 +1,5 @@ #include -#include +#include namespace DB { @@ -31,7 +31,8 @@ void IMergingAlgorithmWithSharedChunks::initialize(Inputs inputs) source.skip_last_row = inputs[source_num].skip_last_row; source.chunk = chunk_allocator.alloc(inputs[source_num].chunk); - cursors[source_num] = SortCursorImpl(header, source.chunk->getColumns(), description, source_num, inputs[source_num].permutation); + cursors[source_num] = SortCursorImpl( + header, source.chunk->getColumns(), source.chunk->getNumRows(), description, source_num, inputs[source_num].permutation); source.chunk->all_columns = cursors[source_num].all_columns; source.chunk->sort_columns = cursors[source_num].sort_columns; @@ -49,7 +50,7 @@ void IMergingAlgorithmWithSharedChunks::consume(Input & input, size_t source_num auto & source = sources[source_num]; source.skip_last_row = input.skip_last_row; source.chunk = chunk_allocator.alloc(input.chunk); - cursors[source_num].reset(source.chunk->getColumns(), header, input.permutation); + cursors[source_num].reset(source.chunk->getColumns(), header, source.chunk->getNumRows(), input.permutation); source.chunk->all_columns = cursors[source_num].all_columns; source.chunk->sort_columns = cursors[source_num].sort_columns; diff --git a/src/Processors/Merges/Algorithms/MergeTreePartLevelInfo.h b/src/Processors/Merges/Algorithms/MergeTreePartLevelInfo.h deleted file mode 100644 index e4f22deec8d..00000000000 --- a/src/Processors/Merges/Algorithms/MergeTreePartLevelInfo.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include - -namespace DB -{ - -/// To carry part level if chunk is produced by a merge tree source -class MergeTreePartLevelInfo : public ChunkInfoCloneable -{ -public: - MergeTreePartLevelInfo() = delete; - explicit MergeTreePartLevelInfo(ssize_t part_level) - : origin_merge_tree_part_level(part_level) - { } - MergeTreePartLevelInfo(const MergeTreePartLevelInfo & other) = default; - - size_t origin_merge_tree_part_level = 0; -}; - -inline size_t getPartLevelFromChunk(const Chunk & chunk) -{ - const auto part_level_info = chunk.getChunkInfos().get(); - if (part_level_info) - return part_level_info->origin_merge_tree_part_level; - return 0; -} - -} diff --git a/src/Processors/Merges/Algorithms/MergeTreeReadInfo.h b/src/Processors/Merges/Algorithms/MergeTreeReadInfo.h new file mode 100644 index 00000000000..253d008c21d --- /dev/null +++ b/src/Processors/Merges/Algorithms/MergeTreeReadInfo.h @@ -0,0 +1,86 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +/// To carry part level and virtual row if chunk is produced by a merge tree source +class MergeTreeReadInfo : public ChunkInfoCloneable +{ +public: + MergeTreeReadInfo() = delete; + explicit MergeTreeReadInfo(size_t part_level) : + origin_merge_tree_part_level(part_level) {} + explicit MergeTreeReadInfo(size_t part_level, const Block & pk_block_, ExpressionActionsPtr virtual_row_conversions_) : + origin_merge_tree_part_level(part_level), pk_block(pk_block_), virtual_row_conversions(std::move(virtual_row_conversions_)) {} + MergeTreeReadInfo(const MergeTreeReadInfo & other) = default; + + size_t origin_merge_tree_part_level = 0; + + /// If is virtual_row, block should not be empty. + Block pk_block; + ExpressionActionsPtr virtual_row_conversions; +}; + +inline size_t getPartLevelFromChunk(const Chunk & chunk) +{ + const auto read_info = chunk.getChunkInfos().get(); + if (read_info) + return read_info->origin_merge_tree_part_level; + return 0; +} + +inline bool isVirtualRow(const Chunk & chunk) +{ + const auto read_info = chunk.getChunkInfos().get(); + if (read_info) + return read_info->pk_block.columns() > 0; + return false; +} + +inline void setVirtualRow(Chunk & chunk, const Block & header, bool apply_virtual_row_conversions) +{ + auto read_info = chunk.getChunkInfos().get(); + chassert(read_info); + + Block & pk_block = read_info->pk_block; + + // std::cerr << apply_virtual_row_conversions << std::endl; + // std::cerr << read_info->virtual_row_conversions->dumpActions() << std::endl; + + if (apply_virtual_row_conversions) + read_info->virtual_row_conversions->execute(pk_block); + + // std::cerr << "++++" << pk_block.dumpStructure() << std::endl; + + Columns ordered_columns; + ordered_columns.reserve(pk_block.columns()); + + for (size_t i = 0; i < header.columns(); ++i) + { + const ColumnWithTypeAndName & col = header.getByPosition(i); + if (const auto * pk_col = pk_block.findByName(col.name)) + { + if (!col.type->equals(*pk_col->type)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Virtual row has different type for {}. Expected {}, got {}", + col.name, col.dumpStructure(), pk_col->dumpStructure()); + + ordered_columns.push_back(pk_col->column); + } + else + ordered_columns.push_back(col.type->createColumnConstWithDefaultValue(1)); + } + + chunk.setColumns(ordered_columns, 1); +} + +} diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp index 3a9cf7ee141..c4338a73520 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -16,12 +17,14 @@ MergingSortedAlgorithm::MergingSortedAlgorithm( SortingQueueStrategy sorting_queue_strategy_, UInt64 limit_, WriteBuffer * out_row_sources_buf_, - bool use_average_block_sizes) + bool use_average_block_sizes, + bool apply_virtual_row_conversions_) : header(std::move(header_)) , merged_data(use_average_block_sizes, max_block_size_, max_block_size_bytes_) , description(description_) , limit(limit_) , out_row_sources_buf(out_row_sources_buf_) + , apply_virtual_row_conversions(apply_virtual_row_conversions_) , current_inputs(num_inputs) , sorting_queue_strategy(sorting_queue_strategy_) , cursors(num_inputs) @@ -49,6 +52,15 @@ void MergingSortedAlgorithm::addInput() void MergingSortedAlgorithm::initialize(Inputs inputs) { + for (auto & input : inputs) + { + if (!isVirtualRow(input.chunk)) + continue; + + setVirtualRow(input.chunk, header, apply_virtual_row_conversions); + input.skip_last_row = true; + } + removeConstAndSparse(inputs); merged_data.initialize(header, inputs); current_inputs = std::move(inputs); @@ -59,7 +71,7 @@ void MergingSortedAlgorithm::initialize(Inputs inputs) if (!chunk) continue; - cursors[source_num] = SortCursorImpl(header, chunk.getColumns(), description, source_num); + cursors[source_num] = SortCursorImpl(header, chunk.getColumns(), chunk.getNumRows(), description, source_num); } if (sorting_queue_strategy == SortingQueueStrategy::Default) @@ -84,7 +96,7 @@ void MergingSortedAlgorithm::consume(Input & input, size_t source_num) { removeConstAndSparse(input); current_inputs[source_num].swap(input); - cursors[source_num].reset(current_inputs[source_num].chunk.getColumns(), header); + cursors[source_num].reset(current_inputs[source_num].chunk.getColumns(), header, current_inputs[source_num].chunk.getNumRows()); if (sorting_queue_strategy == SortingQueueStrategy::Default) { diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h index c889668a38e..0a99b1bd8a6 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h @@ -22,7 +22,8 @@ public: SortingQueueStrategy sorting_queue_strategy_, UInt64 limit_ = 0, WriteBuffer * out_row_sources_buf_ = nullptr, - bool use_average_block_sizes = false); + bool use_average_block_sizes = false, + bool apply_virtual_row_conversions_ = true); void addInput(); @@ -47,6 +48,8 @@ private: /// If it is not nullptr then it should be populated during execution WriteBuffer * out_row_sources_buf = nullptr; + bool apply_virtual_row_conversions; + /// Chunks currently being merged. Inputs current_inputs; diff --git a/src/Processors/Merges/IMergingTransform.cpp b/src/Processors/Merges/IMergingTransform.cpp index b1b0182a113..68957cd55f9 100644 --- a/src/Processors/Merges/IMergingTransform.cpp +++ b/src/Processors/Merges/IMergingTransform.cpp @@ -1,3 +1,4 @@ +#include #include namespace DB @@ -101,11 +102,16 @@ IProcessor::Status IMergingTransformBase::prepareInitializeInputs() /// setNotNeeded after reading first chunk, because in optimismtic case /// (e.g. with optimized 'ORDER BY primary_key LIMIT n' and small 'n') /// we won't have to read any chunks anymore; - auto chunk = input.pull(limit_hint != 0); - if ((limit_hint && chunk.getNumRows() < limit_hint) || always_read_till_end) + /// If virtual row exists, let it pass through, so don't read more chunks. + auto chunk = input.pull(true); + bool virtual_row = isVirtualRow(chunk); + if (limit_hint == 0 && !virtual_row) input.setNeeded(); - if (!chunk.hasRows()) + if (!virtual_row && ((limit_hint && chunk.getNumRows() < limit_hint) || always_read_till_end)) + input.setNeeded(); + + if (!virtual_row && !chunk.hasRows()) { if (!input.isFinished()) { diff --git a/src/Processors/Merges/MergingSortedTransform.cpp b/src/Processors/Merges/MergingSortedTransform.cpp index d2895a2a2e9..760108facb6 100644 --- a/src/Processors/Merges/MergingSortedTransform.cpp +++ b/src/Processors/Merges/MergingSortedTransform.cpp @@ -22,6 +22,7 @@ MergingSortedTransform::MergingSortedTransform( bool always_read_till_end_, WriteBuffer * out_row_sources_buf_, bool use_average_block_sizes, + bool apply_virtual_row_conversions, bool have_all_inputs_) : IMergingTransform( num_inputs, @@ -38,7 +39,8 @@ MergingSortedTransform::MergingSortedTransform( sorting_queue_strategy, limit_, out_row_sources_buf_, - use_average_block_sizes) + use_average_block_sizes, + apply_virtual_row_conversions) { } diff --git a/src/Processors/Merges/MergingSortedTransform.h b/src/Processors/Merges/MergingSortedTransform.h index 6e52450efa7..220ecf0902a 100644 --- a/src/Processors/Merges/MergingSortedTransform.h +++ b/src/Processors/Merges/MergingSortedTransform.h @@ -22,6 +22,7 @@ public: bool always_read_till_end_ = false, WriteBuffer * out_row_sources_buf_ = nullptr, bool use_average_block_sizes = false, + bool apply_virtual_row_conversions = true, bool have_all_inputs_ = true); String getName() const override { return "MergingSortedTransform"; } diff --git a/src/Processors/QueryPlan/BufferChunksTransform.cpp b/src/Processors/QueryPlan/BufferChunksTransform.cpp index 3601a68d36e..75f5f91d981 100644 --- a/src/Processors/QueryPlan/BufferChunksTransform.cpp +++ b/src/Processors/QueryPlan/BufferChunksTransform.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB { @@ -48,14 +49,27 @@ IProcessor::Status BufferChunksTransform::prepare() } else if (input.hasData()) { - auto chunk = pullChunk(); + bool virtual_row; + auto chunk = pullChunk(virtual_row); output.push(std::move(chunk)); + if (virtual_row) + { + input.setNotNeeded(); + return Status::PortFull; + } } } if (input.hasData() && (num_buffered_rows < max_rows_to_buffer || num_buffered_bytes < max_bytes_to_buffer)) { - auto chunk = pullChunk(); + bool virtual_row; + auto chunk = pullChunk(virtual_row); + if (virtual_row) + { + output.push(std::move(chunk)); + input.setNotNeeded(); + return Status::PortFull; + } num_buffered_rows += chunk.getNumRows(); num_buffered_bytes += chunk.bytes(); chunks.push(std::move(chunk)); @@ -71,10 +85,12 @@ IProcessor::Status BufferChunksTransform::prepare() return Status::NeedData; } -Chunk BufferChunksTransform::pullChunk() +Chunk BufferChunksTransform::pullChunk(bool & virtual_row) { auto chunk = input.pull(); - num_processed_rows += chunk.getNumRows(); + virtual_row = isVirtualRow(chunk); + if (!virtual_row) + num_processed_rows += chunk.getNumRows(); if (limit && num_processed_rows >= limit) input.close(); diff --git a/src/Processors/QueryPlan/BufferChunksTransform.h b/src/Processors/QueryPlan/BufferChunksTransform.h index 752f9910734..fce79eeaef3 100644 --- a/src/Processors/QueryPlan/BufferChunksTransform.h +++ b/src/Processors/QueryPlan/BufferChunksTransform.h @@ -24,7 +24,7 @@ public: String getName() const override { return "BufferChunks"; } private: - Chunk pullChunk(); + Chunk pullChunk(bool & virtual_row); InputPort & input; OutputPort & output; diff --git a/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp b/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp index fb3ed7f80fc..1832cc2ad42 100644 --- a/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp +++ b/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp @@ -6,12 +6,23 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool query_plan_merge_filters; +} + BuildQueryPipelineSettings BuildQueryPipelineSettings::fromContext(ContextPtr from) { + const auto & query_settings = from->getSettingsRef(); BuildQueryPipelineSettings settings; - settings.actions_settings = ExpressionActionsSettings::fromSettings(from->getSettingsRef(), CompileExpressions::yes); + settings.actions_settings = ExpressionActionsSettings::fromSettings(query_settings, CompileExpressions::yes); settings.process_list_element = from->getProcessListElement(); settings.progress_callback = from->getProgressCallback(); + + /// Setting query_plan_merge_filters is enabled by default. + /// But it can brake short-circuit without splitting filter step into smaller steps. + /// So, enable and disable this optimizations together. + settings.enable_multiple_filters_transforms_for_and_chain = query_settings[Setting::query_plan_merge_filters]; return settings; } diff --git a/src/Processors/QueryPlan/BuildQueryPipelineSettings.h b/src/Processors/QueryPlan/BuildQueryPipelineSettings.h index d99f9a7d1f1..6219e37db58 100644 --- a/src/Processors/QueryPlan/BuildQueryPipelineSettings.h +++ b/src/Processors/QueryPlan/BuildQueryPipelineSettings.h @@ -17,6 +17,8 @@ using TemporaryFileLookupPtr = std::shared_ptr; struct BuildQueryPipelineSettings { + bool enable_multiple_filters_transforms_for_and_chain = true; + ExpressionActionsSettings actions_settings; QueryStatusPtr process_list_element; ProgressCallback progress_callback = nullptr; diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp index 862e03d74f2..af9e3f0c515 100644 --- a/src/Processors/QueryPlan/FilterStep.cpp +++ b/src/Processors/QueryPlan/FilterStep.cpp @@ -5,6 +5,11 @@ #include #include #include +#include +#include +#include +#include +#include namespace DB { @@ -24,6 +29,92 @@ static ITransformingStep::Traits getTraits() }; } +static bool isTrivialSubtree(const ActionsDAG::Node * node) +{ + while (node->type == ActionsDAG::ActionType::ALIAS) + node = node->children.at(0); + + return node->type != ActionsDAG::ActionType::FUNCTION && node->type != ActionsDAG::ActionType::ARRAY_JOIN; +} + +struct ActionsAndName +{ + ActionsDAG dag; + std::string name; +}; + +static ActionsAndName splitSingleAndFilter(ActionsDAG & dag, const ActionsDAG::Node * filter_node) +{ + auto split_result = dag.split({filter_node}, true); + dag = std::move(split_result.second); + + const auto * split_filter_node = split_result.split_nodes_mapping[filter_node]; + auto filter_type = removeLowCardinality(split_filter_node->result_type); + if (!filter_type->onlyNull() && !isUInt8(removeNullable(filter_type))) + { + DataTypePtr cast_type = std::make_shared(); + if (filter_type->isNullable()) + cast_type = std::make_shared(std::move(cast_type)); + + split_filter_node = &split_result.first.addCast(*split_filter_node, cast_type, {}); + } + + split_result.first.getOutputs().emplace(split_result.first.getOutputs().begin(), split_filter_node); + auto name = split_filter_node->result_name; + return ActionsAndName{std::move(split_result.first), std::move(name)}; +} + +/// Try to split the left most AND atom to a separate DAG. +static std::optional trySplitSingleAndFilter(ActionsDAG & dag, const std::string & filter_name) +{ + const auto * filter = &dag.findInOutputs(filter_name); + while (filter->type == ActionsDAG::ActionType::ALIAS) + filter = filter->children.at(0); + + if (filter->type != ActionsDAG::ActionType::FUNCTION || filter->function_base->getName() != "and") + return {}; + + const ActionsDAG::Node * condition_to_split = nullptr; + std::stack nodes; + nodes.push(filter); + while (!nodes.empty()) + { + const auto * node = nodes.top(); + nodes.pop(); + + if (node->type == ActionsDAG::ActionType::FUNCTION && node->function_base->getName() == "and") + { + /// The order is important. We should take the left-most atom, so put conditions on stack in reverse order. + for (const auto * child : node->children | std::ranges::views::reverse) + nodes.push(child); + + continue; + } + + if (isTrivialSubtree(node)) + continue; + + /// Do not split subtree if it's the last non-trivial one. + /// So, split the first found condition only when there is a another one found. + if (condition_to_split) + return splitSingleAndFilter(dag, condition_to_split); + + condition_to_split = node; + } + + return {}; +} + +std::vector splitAndChainIntoMultipleFilters(ActionsDAG & dag, const std::string & filter_name) +{ + std::vector res; + + while (auto condition = trySplitSingleAndFilter(dag, filter_name)) + res.push_back(std::move(*condition)); + + return res; +} + FilterStep::FilterStep( const Header & input_header_, ActionsDAG actions_dag_, @@ -50,6 +141,23 @@ FilterStep::FilterStep( void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) { + std::vector and_atoms; + + /// Splitting AND filter condition to steps under the setting, which is enabled with merge_filters optimization. + /// This is needed to support short-circuit properly. + if (settings.enable_multiple_filters_transforms_for_and_chain && !actions_dag.hasStatefulFunctions()) + and_atoms = splitAndChainIntoMultipleFilters(actions_dag, filter_column_name); + + for (auto & and_atom : and_atoms) + { + auto expression = std::make_shared(std::move(and_atom.dag), settings.getActionsSettings()); + pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) + { + bool on_totals = stream_type == QueryPipelineBuilder::StreamType::Totals; + return std::make_shared(header, expression, and_atom.name, true, on_totals); + }); + } + auto expression = std::make_shared(std::move(actions_dag), settings.getActionsSettings()); pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) @@ -76,18 +184,45 @@ void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQ void FilterStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, settings.indent_char); + + auto cloned_dag = actions_dag.clone(); + + std::vector and_atoms; + if (!actions_dag.hasStatefulFunctions()) + and_atoms = splitAndChainIntoMultipleFilters(cloned_dag, filter_column_name); + + for (auto & and_atom : and_atoms) + { + auto expression = std::make_shared(std::move(and_atom.dag)); + settings.out << prefix << "AND column: " << and_atom.name << '\n'; + expression->describeActions(settings.out, prefix); + } + settings.out << prefix << "Filter column: " << filter_column_name; if (remove_filter_column) settings.out << " (removed)"; settings.out << '\n'; - auto expression = std::make_shared(actions_dag.clone()); + auto expression = std::make_shared(std::move(cloned_dag)); expression->describeActions(settings.out, prefix); } void FilterStep::describeActions(JSONBuilder::JSONMap & map) const { + auto cloned_dag = actions_dag.clone(); + + std::vector and_atoms; + if (!actions_dag.hasStatefulFunctions()) + and_atoms = splitAndChainIntoMultipleFilters(cloned_dag, filter_column_name); + + for (auto & and_atom : and_atoms) + { + auto expression = std::make_shared(std::move(and_atom.dag)); + map.add("AND column", and_atom.name); + map.add("Expression", expression->toTree()); + } + map.add("Filter Column", filter_column_name); map.add("Removes Filter", remove_filter_column); diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h index 6232fc7f54f..55a9d18f063 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h @@ -32,7 +32,7 @@ struct QueryPlanOptimizationSettings bool merge_expressions = true; /// If merge-filters optimization is enabled. - bool merge_filters = false; + bool merge_filters = true; /// If filter push down optimization is enabled. bool filter_push_down = true; diff --git a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp index eddcbab76d2..18171daf67c 100644 --- a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp +++ b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp @@ -211,6 +211,8 @@ MatchedTrees::Matches matchTrees(const ActionsDAG::NodeRawConstPtrs & inner_dag, MatchedTrees::Monotonicity monotonicity; monotonicity.direction *= info.is_positive ? 1 : -1; monotonicity.strict = info.is_strict; + monotonicity.child_match = &child_match; + monotonicity.child_node = monotonic_child; if (child_match.monotonicity) { diff --git a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.h b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.h index 6a33e7d3dd6..50b28c773df 100644 --- a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.h +++ b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.h @@ -22,12 +22,16 @@ namespace DB /// DAG for PK does not contain aliases and ambiguous nodes. struct MatchedTrees { + struct Match; + /// Monotonicity is calculated for monotonic functions chain. /// Chain is not strict if there is any non-strict monotonic function. struct Monotonicity { int direction = 1; bool strict = true; + const Match * child_match = nullptr; + const ActionsDAG::Node * child_node = nullptr; }; struct Match diff --git a/src/Processors/QueryPlan/Optimizations/applyOrder.cpp b/src/Processors/QueryPlan/Optimizations/applyOrder.cpp index 8695f29c26b..51a5aa099ac 100644 --- a/src/Processors/QueryPlan/Optimizations/applyOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/applyOrder.cpp @@ -124,7 +124,7 @@ SortingProperty applyOrder(QueryPlan::Node * parent, SortingProperty * propertie auto common_prefix = commonPrefix(properties->sort_description, sorting_step->getSortDescription()); if (!common_prefix.empty()) /// Buffering is useful for reading from MergeTree, and it is applied in optimizeReadInOrder only. - sorting_step->convertToFinishSorting(common_prefix, /*use_buffering*/ false); + sorting_step->convertToFinishSorting(common_prefix, /*use_buffering*/ false, false); } auto scope = sorting_step->hasPartitions() ? SortingProperty::SortScope::Stream : SortingProperty::SortScope::Global; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index e64a88de62e..9cb9db8eebe 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -324,12 +324,43 @@ void enrichFixedColumns(const ActionsDAG & dag, FixedColumns & fixed_columns) } } +const ActionsDAG::Node * addMonotonicChain(ActionsDAG & dag, const ActionsDAG::Node * node, const MatchedTrees::Match * match, const std::string & input_name) +{ + if (!match->monotonicity) + return &dag.addInput(input_name, node->result_type); + + if (node->type == ActionsDAG::ActionType::ALIAS) + return &dag.addAlias(*addMonotonicChain(dag, node->children.front(), match, input_name), node->result_name); + + ActionsDAG::NodeRawConstPtrs args; + args.reserve(node->children.size()); + for (const auto * child : node->children) + { + if (child == match->monotonicity->child_node) + args.push_back(addMonotonicChain(dag, match->monotonicity->child_node, match->monotonicity->child_match, input_name)); + else + args.push_back(&dag.addColumn({child->column, child->result_type, child->result_name})); + } + + return &dag.addFunction(node->function_base, std::move(args), {}); +} + +struct SortingInputOrder +{ + InputOrderInfoPtr input_order{}; + /// This is needed for virtual row optimization. + /// Convert the PR values to ORDER BY key. + /// If empty, the optimization cannot be applied. + std::optional virtual_row_conversion{}; +}; + /// For the case when the order of keys is important (ORDER BY keys). -InputOrderInfoPtr buildInputOrderFromSortDescription( +SortingInputOrder buildInputOrderFromSortDescription( const FixedColumns & fixed_columns, const std::optional & dag, const SortDescription & description, const KeyDescription & sorting_key, + const Names & pk_column_names, size_t limit) { //std::cerr << "------- buildInputOrderInfo " << std::endl; @@ -369,6 +400,18 @@ InputOrderInfoPtr buildInputOrderFromSortDescription( size_t next_description_column = 0; size_t next_sort_key = 0; + bool can_optimize_virtual_row = true; + + struct MatchInfo + { + const ActionsDAG::Node * source = nullptr; + const ActionsDAG::Node * fixed_column = nullptr; + const MatchedTrees::Match * monotonic = nullptr; + }; + + std::vector match_infos; + match_infos.reserve(description.size()); + while (next_description_column < description.size() && next_sort_key < sorting_key.column_names.size()) { const auto & sorting_key_column = sorting_key.column_names[next_sort_key]; @@ -410,6 +453,7 @@ InputOrderInfoPtr buildInputOrderFromSortDescription( //std::cerr << "====== (no dag) Found direct match" << std::endl; + match_infos.push_back({.source = sort_column_node}); ++next_description_column; ++next_sort_key; } @@ -438,24 +482,46 @@ InputOrderInfoPtr buildInputOrderFromSortDescription( { current_direction *= match.monotonicity->direction; strict_monotonic = match.monotonicity->strict; + match_infos.push_back({.source = sort_node, .monotonic = &match}); } + else + match_infos.push_back({.source = sort_node}); ++next_description_column; ++next_sort_key; } else if (fixed_key_columns.contains(sort_column_node)) { + + if (next_sort_key == 0) + { + // Disable virtual row optimization. + // For example, when pk is (a,b), a = 1, order by b, virtual row should be + // disabled in the following case: + // 1st part (0, 100), (1, 2), (1, 3), (1, 4) + // 2nd part (0, 100), (1, 2), (1, 3), (1, 4). + + can_optimize_virtual_row = false; + } + //std::cerr << "+++++++++ Found fixed key by match" << std::endl; ++next_sort_key; } else { - //std::cerr << "====== Check for fixed const : " << bool(sort_node->column) << " fixed : " << fixed_columns.contains(sort_node) << std::endl; bool is_fixed_column = sort_node->column || fixed_columns.contains(sort_node); if (!is_fixed_column) break; + if (!sort_node->column) + /// Virtual row for fixed column from order by is not supported now. + /// TODO: we can do it for the simple case, + /// But it's better to remove fixed columns from ORDER BY completely, e.g: + /// WHERE x = 42 ORDER BY x, y => WHERE x = 42 ORDER BY y + can_optimize_virtual_row = false; + + match_infos.push_back({.source = sort_node, .fixed_column = sort_node}); order_key_prefix_descr.push_back(sort_column_description); ++next_description_column; } @@ -477,9 +543,46 @@ InputOrderInfoPtr buildInputOrderFromSortDescription( } if (read_direction == 0 || order_key_prefix_descr.empty()) - return nullptr; + return {}; - return std::make_shared(order_key_prefix_descr, next_sort_key, read_direction, limit); + /// If the prefix description is used, we can't restore the full description from PK value. + /// TODO: partial sort description can be used as well. Implement support later. + if (order_key_prefix_descr.size() < description.size() || pk_column_names.size() < next_sort_key) + can_optimize_virtual_row = false; + + auto order_info = std::make_shared(order_key_prefix_descr, next_sort_key, read_direction, limit); + + std::optional virtual_row_conversion; + if (can_optimize_virtual_row) + { + ActionsDAG virtual_row_dag; + virtual_row_dag.getOutputs().reserve(match_infos.size()); + size_t next_pk_name = 0; + for (const auto & info : match_infos) + { + const ActionsDAG::Node * output; + if (info.fixed_column) + output = &virtual_row_dag.addColumn({info.fixed_column->column, info.fixed_column->result_type, info.fixed_column->result_name}); + else + { + if (info.monotonic) + output = addMonotonicChain(virtual_row_dag, info.source, info.monotonic, pk_column_names[next_pk_name]); + else + { + output = &virtual_row_dag.addInput(pk_column_names[next_pk_name], info.source->result_type); + if (pk_column_names[next_pk_name] != info.source->result_name) + output = &virtual_row_dag.addAlias(*output, info.source->result_name); + } + + ++next_pk_name; + } + + virtual_row_dag.getOutputs().push_back(output); + } + virtual_row_conversion = std::move(virtual_row_dag); + } + + return {std::move(order_info), std::move(virtual_row_conversion)}; } /// We may need a few different sort descriptions here. @@ -689,7 +792,7 @@ InputOrder buildInputOrderFromUnorderedKeys( return { std::move(input_order), std::move(sort_description) }; // std::move(group_by_sort_description) }; } -InputOrderInfoPtr buildInputOrderFromSortDescription( +SortingInputOrder buildInputOrderFromSortDescription( const ReadFromMergeTree * reading, const FixedColumns & fixed_columns, const std::optional & dag, @@ -697,15 +800,17 @@ InputOrderInfoPtr buildInputOrderFromSortDescription( size_t limit) { const auto & sorting_key = reading->getStorageMetadata()->getSortingKey(); + const auto & pk_column_names = reading->getStorageMetadata()->getPrimaryKey().column_names; return buildInputOrderFromSortDescription( fixed_columns, dag, description, sorting_key, + pk_column_names, limit); } -InputOrderInfoPtr buildInputOrderFromSortDescription( +SortingInputOrder buildInputOrderFromSortDescription( ReadFromMerge * merge, const FixedColumns & fixed_columns, const std::optional & dag, @@ -714,28 +819,31 @@ InputOrderInfoPtr buildInputOrderFromSortDescription( { const auto & tables = merge->getSelectedTables(); - InputOrderInfoPtr order_info; + SortingInputOrder order_info; for (const auto & table : tables) { auto storage = std::get(table); - const auto & sorting_key = storage->getInMemoryMetadataPtr()->getSortingKey(); + auto metadata = storage->getInMemoryMetadataPtr(); + const auto & sorting_key = metadata->getSortingKey(); + // const auto & pk_column_names = metadata->getPrimaryKey().column_names; if (sorting_key.column_names.empty()) - return nullptr; + return {}; auto table_order_info = buildInputOrderFromSortDescription( fixed_columns, dag, description, sorting_key, + {}, limit); - if (!table_order_info) - return nullptr; + if (!table_order_info.input_order) + return {}; - if (!order_info) - order_info = table_order_info; - else if (*order_info != *table_order_info) - return nullptr; + if (!order_info.input_order) + order_info = std::move(table_order_info); + else if (*order_info.input_order != *table_order_info.input_order) + return {}; } return order_info; @@ -791,7 +899,7 @@ InputOrder buildInputOrderFromUnorderedKeys( return order_info; } -InputOrderInfoPtr buildInputOrderInfo(SortingStep & sorting, QueryPlan::Node & node) +InputOrderInfoPtr buildInputOrderInfo(SortingStep & sorting, bool & apply_virtual_row, QueryPlan::Node & node) { QueryPlan::Node * reading_node = findReadingStep(node, /*allow_existing_order=*/ false); if (!reading_node) @@ -815,14 +923,21 @@ InputOrderInfoPtr buildInputOrderInfo(SortingStep & sorting, QueryPlan::Node & n dag, description, limit); - if (order_info) + if (order_info.input_order) { - bool can_read = reading->requestReadingInOrder(order_info->used_prefix_of_sorting_key_size, order_info->direction, order_info->limit); + apply_virtual_row = order_info.virtual_row_conversion != std::nullopt; + + bool can_read = reading->requestReadingInOrder( + order_info.input_order->used_prefix_of_sorting_key_size, + order_info.input_order->direction, + order_info.input_order->limit, + std::move(order_info.virtual_row_conversion)); + if (!can_read) return nullptr; } - return order_info; + return order_info.input_order; } if (auto * merge = typeid_cast(reading_node->step.get())) { @@ -832,14 +947,14 @@ InputOrderInfoPtr buildInputOrderInfo(SortingStep & sorting, QueryPlan::Node & n dag, description, limit); - if (order_info) + if (order_info.input_order) { - bool can_read = merge->requestReadingInOrder(order_info); + bool can_read = merge->requestReadingInOrder(order_info.input_order); if (!can_read) return nullptr; } - return order_info; + return order_info.input_order; } return nullptr; @@ -873,7 +988,8 @@ InputOrder buildInputOrderInfo(AggregatingStep & aggregating, QueryPlan::Node & bool can_read = reading->requestReadingInOrder( order_info.input_order->used_prefix_of_sorting_key_size, order_info.input_order->direction, - order_info.input_order->limit); + order_info.input_order->limit, + {}); if (!can_read) return {}; } @@ -962,7 +1078,7 @@ InputOrder buildInputOrderInfo(DistinctStep & distinct, QueryPlan::Node & node) if (!reading->requestReadingInOrder( order_info.input_order->used_prefix_of_sorting_key_size, order_info.input_order->direction, - order_info.input_order->limit)) + order_info.input_order->limit, {})) return {}; return order_info; @@ -1014,6 +1130,8 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes) if (sorting->getType() != SortingStep::Type::Full) return; + bool apply_virtual_row = false; + if (typeid_cast(node.children.front()->step.get())) { auto & union_node = node.children.front(); @@ -1036,7 +1154,7 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes) for (auto * child : union_node->children) { - infos.push_back(buildInputOrderInfo(*sorting, *child)); + infos.push_back(buildInputOrderInfo(*sorting, apply_virtual_row, *child)); if (infos.back()) { @@ -1088,13 +1206,13 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes) } } - sorting->convertToFinishSorting(*max_sort_descr, use_buffering); + sorting->convertToFinishSorting(*max_sort_descr, use_buffering, false); } - else if (auto order_info = buildInputOrderInfo(*sorting, *node.children.front())) + else if (auto order_info = buildInputOrderInfo(*sorting, apply_virtual_row, *node.children.front())) { /// Use buffering only if have filter or don't have limit. bool use_buffering = order_info->limit == 0; - sorting->convertToFinishSorting(order_info->sort_description_for_merging, use_buffering); + sorting->convertToFinishSorting(order_info->sort_description_for_merging, use_buffering, apply_virtual_row); } } @@ -1233,10 +1351,10 @@ size_t tryReuseStorageOrderingForWindowFunctions(QueryPlan::Node * parent_node, if (order_info) { - bool can_read = read_from_merge_tree->requestReadingInOrder(order_info->used_prefix_of_sorting_key_size, order_info->direction, order_info->limit); + bool can_read = read_from_merge_tree->requestReadingInOrder(order_info->used_prefix_of_sorting_key_size, order_info->direction, order_info->limit, {}); if (!can_read) return 0; - sorting->convertToFinishSorting(order_info->sort_description_for_merging, false); + sorting->convertToFinishSorting(order_info->sort_description_for_merging, false, false); } return 0; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 3186df6a6b3..626e43898e4 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -175,7 +176,9 @@ namespace Setting extern const SettingsBool use_skip_indexes; extern const SettingsBool use_skip_indexes_if_final; extern const SettingsBool use_uncompressed_cache; + extern const SettingsBool query_plan_merge_filters; extern const SettingsUInt64 merge_tree_min_read_task_size; + extern const SettingsBool read_in_order_use_virtual_row; } namespace MergeTreeSetting @@ -206,6 +209,7 @@ static MergeTreeReaderSettings getMergeTreeReaderSettings( .use_asynchronous_read_from_pool = settings[Setting::allow_asynchronous_read_from_io_pool_for_merge_tree] && (settings[Setting::max_streams_to_max_threads_ratio] > 1 || settings[Setting::max_streams_for_merge_tree_reading] > 1), .enable_multiple_prewhere_read_steps = settings[Setting::enable_multiple_prewhere_read_steps], + .force_short_circuit_execution = settings[Setting::query_plan_merge_filters] }; } @@ -680,7 +684,34 @@ Pipe ReadFromMergeTree::readInOrder( if (set_total_rows_approx) source->addTotalRowsApprox(total_rows); - pipes.emplace_back(std::move(source)); + Pipe pipe(source); + + if (virtual_row_conversion && (read_type == ReadType::InOrder)) + { + const auto & index = part_with_ranges.data_part->getIndex(); + const auto & primary_key = storage_snapshot->metadata->primary_key; + size_t mark_range_begin = part_with_ranges.ranges.front().begin; + + ColumnsWithTypeAndName pk_columns; + size_t num_columns = virtual_row_conversion->getRequiredColumnsWithTypes().size(); + pk_columns.reserve(num_columns); + + for (size_t j = 0; j < num_columns; ++j) + { + auto column = primary_key.data_types[j]->createColumn()->cloneEmpty(); + column->insert((*(*index)[j])[mark_range_begin]); + pk_columns.push_back({std::move(column), primary_key.data_types[j], primary_key.column_names[j]}); + } + + Block pk_block(std::move(pk_columns)); + + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, pk_block, virtual_row_conversion); + }); + } + + pipes.emplace_back(std::move(pipe)); } auto pipe = Pipe::unitePipes(std::move(pipes)); @@ -1140,7 +1171,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( if (pipe.numOutputPorts() > 1) { auto transform = std::make_shared( - pipe.getHeader(), pipe.numOutputPorts(), sort_description, block_size.max_block_size_rows, /*max_block_size_bytes=*/0, SortingQueueStrategy::Batch); + pipe.getHeader(), pipe.numOutputPorts(), sort_description, block_size.max_block_size_rows, /*max_block_size_bytes=*/0, SortingQueueStrategy::Batch, + 0, false, nullptr, false, /*apply_virtual_row_conversions*/ false); pipe.addTransform(std::move(transform)); } @@ -1799,7 +1831,7 @@ void ReadFromMergeTree::updateSortDescription() enable_vertical_final); } -bool ReadFromMergeTree::requestReadingInOrder(size_t prefix_size, int direction, size_t read_limit) +bool ReadFromMergeTree::requestReadingInOrder(size_t prefix_size, int direction, size_t read_limit, std::optional virtual_row_conversion_) { /// if dirction is not set, use current one if (!direction) @@ -1822,6 +1854,10 @@ bool ReadFromMergeTree::requestReadingInOrder(size_t prefix_size, int direction, /// Let prefer in-order optimization over vertical FINAL for now enable_vertical_final = false; + /// Disable virtual row for FINAL. + if (virtual_row_conversion_ && !isQueryWithFinal() && context->getSettingsRef()[Setting::read_in_order_use_virtual_row]) + virtual_row_conversion = std::make_shared(std::move(*virtual_row_conversion_)); + updateSortDescription(); return true; @@ -2238,6 +2274,12 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const expression->describeActions(format_settings.out, prefix); } } + + if (virtual_row_conversion) + { + format_settings.out << prefix << "Virtual row conversions" << '\n'; + virtual_row_conversion->describeActions(format_settings.out, prefix); + } } void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const @@ -2277,6 +2319,9 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const map.add("Prewhere info", std::move(prewhere_info_map)); } + + if (virtual_row_conversion) + map.add("Virtual row conversions", virtual_row_conversion->toTree()); } void ReadFromMergeTree::describeIndexes(FormatSettings & format_settings) const diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 46a02f5643b..2b02fa82761 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -187,7 +187,7 @@ public: StorageMetadataPtr getStorageMetadata() const { return storage_snapshot->metadata; } /// Returns `false` if requested reading cannot be performed. - bool requestReadingInOrder(size_t prefix_size, int direction, size_t limit); + bool requestReadingInOrder(size_t prefix_size, int direction, size_t limit, std::optional virtual_row_conversion_); bool readsInOrder() const; const InputOrderInfoPtr & getInputOrder() const { return query_info.input_order_info; } const SortDescription & getSortDescription() const override { return result_sort_description; } @@ -281,6 +281,9 @@ private: std::optional read_task_callback; bool enable_vertical_final = false; bool enable_remove_parts_from_snapshot_optimization = true; + + ExpressionActionsPtr virtual_row_conversion; + std::optional number_of_current_replica; }; diff --git a/src/Processors/QueryPlan/SortingStep.cpp b/src/Processors/QueryPlan/SortingStep.cpp index 4fde246f764..e6cd9b77f4d 100644 --- a/src/Processors/QueryPlan/SortingStep.cpp +++ b/src/Processors/QueryPlan/SortingStep.cpp @@ -77,13 +77,11 @@ static ITransformingStep::Traits getTraits(size_t limit) } SortingStep::SortingStep( - const Header & input_header, - SortDescription description_, - UInt64 limit_, - const Settings & settings_) + const Header & input_header, SortDescription description_, UInt64 limit_, const Settings & settings_, bool is_sorting_for_merge_join_) : ITransformingStep(input_header, input_header, getTraits(limit_)) , type(Type::Full) , result_description(std::move(description_)) + , is_sorting_for_merge_join(is_sorting_for_merge_join_) , limit(limit_) , sort_settings(settings_) { @@ -147,11 +145,12 @@ void SortingStep::updateLimit(size_t limit_) } } -void SortingStep::convertToFinishSorting(SortDescription prefix_description_, bool use_buffering_) +void SortingStep::convertToFinishSorting(SortDescription prefix_description_, bool use_buffering_, bool apply_virtual_row_conversions_) { type = Type::FinishSorting; prefix_description = std::move(prefix_description_); use_buffering = use_buffering_; + apply_virtual_row_conversions = apply_virtual_row_conversions_; } void SortingStep::scatterByPartitionIfNeeded(QueryPipelineBuilder& pipeline) @@ -255,7 +254,10 @@ void SortingStep::mergingSorted(QueryPipelineBuilder & pipeline, const SortDescr /*max_block_size_bytes=*/0, SortingQueueStrategy::Batch, limit_, - always_read_till_end); + always_read_till_end, + nullptr, + false, + apply_virtual_row_conversions); pipeline.addTransform(std::move(transform)); } diff --git a/src/Processors/QueryPlan/SortingStep.h b/src/Processors/QueryPlan/SortingStep.h index 6cdf626d4c8..d831fe0234f 100644 --- a/src/Processors/QueryPlan/SortingStep.h +++ b/src/Processors/QueryPlan/SortingStep.h @@ -39,7 +39,8 @@ public: const Header & input_header, SortDescription description_, UInt64 limit_, - const Settings & settings_); + const Settings & settings_, + bool is_sorting_for_merge_join_ = false); /// Full with partitioning SortingStep( @@ -81,7 +82,9 @@ public: bool hasPartitions() const { return !partition_by_description.empty(); } - void convertToFinishSorting(SortDescription prefix_description, bool use_buffering_); + bool isSortingForMergeJoin() const { return is_sorting_for_merge_join; } + + void convertToFinishSorting(SortDescription prefix_description, bool use_buffering_, bool apply_virtual_row_conversions_); Type getType() const { return type; } const Settings & getSettings() const { return sort_settings; } @@ -125,9 +128,13 @@ private: SortDescription partition_by_description; + /// See `findQueryForParallelReplicas` + bool is_sorting_for_merge_join = false; + UInt64 limit; bool always_read_till_end = false; bool use_buffering = false; + bool apply_virtual_row_conversions = false; Settings sort_settings; }; diff --git a/src/Processors/Sinks/SinkToStorage.h b/src/Processors/Sinks/SinkToStorage.h index c728fa87b1e..4bdcb2fe855 100644 --- a/src/Processors/Sinks/SinkToStorage.h +++ b/src/Processors/Sinks/SinkToStorage.h @@ -5,6 +5,8 @@ namespace DB { +class Context; + /// Sink which is returned from Storage::write. class SinkToStorage : public ExceptionKeepingTransform { @@ -16,12 +18,14 @@ public: const Block & getHeader() const { return inputs.front().getHeader(); } void addTableLock(const TableLockHolder & lock) { table_locks.push_back(lock); } + void addInterpreterContext(std::shared_ptr context) { interpreter_context.emplace_back(std::move(context)); } protected: virtual void consume(Chunk & chunk) = 0; private: std::vector table_locks; + std::vector> interpreter_context; void onConsume(Chunk chunk) override; GenerateResult onGenerate() override; diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index a7d18664786..d1f4290a5b2 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -107,7 +107,7 @@ struct ManyAggregatedData if (variant->aggregator) { // variant is moved here and will be destroyed in the destructor of the lambda function. - pool->trySchedule( + pool->scheduleOrThrowOnError( [my_variant = std::move(variant), thread_group = CurrentThread::getGroup()]() { SCOPE_EXIT_SAFE( diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp index 95f4a674ebb..ab782f3e521 100644 --- a/src/Processors/Transforms/FillingTransform.cpp +++ b/src/Processors/Transforms/FillingTransform.cpp @@ -7,18 +7,20 @@ #include #include #include +#include #include #include #include +#include namespace DB { -constexpr bool debug_logging_enabled = false; +constexpr static bool debug_logging_enabled = false; template -void logDebug(String key, const T & value, const char * separator = " : ") +inline static void logDebug(const char * key, const T & value, const char * separator = " : ") { if constexpr (debug_logging_enabled) { @@ -60,15 +62,74 @@ static FillColumnDescription::StepFunction getStepFunction( { #define DECLARE_CASE(NAME) \ case IntervalKind::Kind::NAME: \ - return [step, scale, &date_lut](Field & field) { \ + return [step, scale, &date_lut](Field & field, Int32 jumps_count) { \ field = Add##NAME##sImpl::execute(static_cast(\ - field.safeGet()), static_cast(step), date_lut, utc_time_zone, scale); }; + field.safeGet()), static_cast(step) * jumps_count, date_lut, utc_time_zone, scale); }; FOR_EACH_INTERVAL_KIND(DECLARE_CASE) #undef DECLARE_CASE } } +static FillColumnDescription::StepFunction getStepFunction(const Field & step, const std::optional & step_kind, const DataTypePtr & type) +{ + WhichDataType which(type); + + if (step_kind) + { + if (which.isDate() || which.isDate32()) + { + Int64 avg_seconds = step.safeGet() * step_kind->toAvgSeconds(); + if (std::abs(avg_seconds) < 86400) + throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, + "Value of step is to low ({} seconds). Must be >= 1 day", std::abs(avg_seconds)); + } + + if (which.isDate()) + return getStepFunction(step_kind.value(), step.safeGet(), DateLUT::instance()); + else if (which.isDate32()) + return getStepFunction(step_kind.value(), step.safeGet(), DateLUT::instance()); + else if (const auto * date_time = checkAndGetDataType(type.get())) + return getStepFunction(step_kind.value(), step.safeGet(), date_time->getTimeZone()); + else if (const auto * date_time64 = checkAndGetDataType(type.get())) + { + const auto & step_dec = step.safeGet &>(); + Int64 converted_step = DecimalUtils::convertTo(step_dec.getValue(), step_dec.getScale()); + static const DateLUTImpl & utc_time_zone = DateLUT::instance("UTC"); + + switch (step_kind.value()) // NOLINT(bugprone-switch-missing-default-case) + { +#define DECLARE_CASE(NAME) \ + case IntervalKind::Kind::NAME: \ + return [converted_step, &time_zone = date_time64->getTimeZone()](Field & field, Int32 jumps_count) \ + { \ + auto field_decimal = field.safeGet>(); \ + auto res = Add##NAME##sImpl::execute(field_decimal.getValue(), converted_step * jumps_count, time_zone, utc_time_zone, field_decimal.getScale()); \ + field = DecimalField(res, field_decimal.getScale()); \ + }; \ + break; + + FOR_EACH_INTERVAL_KIND(DECLARE_CASE) +#undef DECLARE_CASE + } + } + else + throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, + "STEP of Interval type can be used only with Date/DateTime types, but got {}", type->getName()); + } + else + { + return [step](Field & field, Int32 jumps_count) + { + auto shifted_step = step; + if (jumps_count != 1) + applyVisitor(FieldVisitorScale(jumps_count), shifted_step); + + applyVisitor(FieldVisitorSum(shifted_step), field); + }; + } +} + static bool tryConvertFields(FillColumnDescription & descr, const DataTypePtr & type) { auto max_type = Field::Types::Null; @@ -125,7 +186,8 @@ static bool tryConvertFields(FillColumnDescription & descr, const DataTypePtr & if (descr.fill_from.getType() > max_type || descr.fill_to.getType() > max_type - || descr.fill_step.getType() > max_type) + || descr.fill_step.getType() > max_type + || descr.fill_staleness.getType() > max_type) return false; if (!descr.fill_from.isNull()) @@ -134,56 +196,11 @@ static bool tryConvertFields(FillColumnDescription & descr, const DataTypePtr & descr.fill_to = convertFieldToTypeOrThrow(descr.fill_to, *to_type); if (!descr.fill_step.isNull()) descr.fill_step = convertFieldToTypeOrThrow(descr.fill_step, *to_type); + if (!descr.fill_staleness.isNull()) + descr.fill_staleness = convertFieldToTypeOrThrow(descr.fill_staleness, *to_type); - if (descr.step_kind) - { - if (which.isDate() || which.isDate32()) - { - Int64 avg_seconds = descr.fill_step.safeGet() * descr.step_kind->toAvgSeconds(); - if (std::abs(avg_seconds) < 86400) - throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, - "Value of step is to low ({} seconds). Must be >= 1 day", std::abs(avg_seconds)); - } - - if (which.isDate()) - descr.step_func = getStepFunction(*descr.step_kind, descr.fill_step.safeGet(), DateLUT::instance()); - else if (which.isDate32()) - descr.step_func = getStepFunction(*descr.step_kind, descr.fill_step.safeGet(), DateLUT::instance()); - else if (const auto * date_time = checkAndGetDataType(type.get())) - descr.step_func = getStepFunction(*descr.step_kind, descr.fill_step.safeGet(), date_time->getTimeZone()); - else if (const auto * date_time64 = checkAndGetDataType(type.get())) - { - const auto & step_dec = descr.fill_step.safeGet &>(); - Int64 step = DecimalUtils::convertTo(step_dec.getValue(), step_dec.getScale()); - static const DateLUTImpl & utc_time_zone = DateLUT::instance("UTC"); - - switch (*descr.step_kind) // NOLINT(bugprone-switch-missing-default-case) - { -#define DECLARE_CASE(NAME) \ - case IntervalKind::Kind::NAME: \ - descr.step_func = [step, &time_zone = date_time64->getTimeZone()](Field & field) \ - { \ - auto field_decimal = field.safeGet>(); \ - auto res = Add##NAME##sImpl::execute(field_decimal.getValue(), step, time_zone, utc_time_zone, field_decimal.getScale()); \ - field = DecimalField(res, field_decimal.getScale()); \ - }; \ - break; - - FOR_EACH_INTERVAL_KIND(DECLARE_CASE) -#undef DECLARE_CASE - } - } - else - throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, - "STEP of Interval type can be used only with Date/DateTime types, but got {}", type->getName()); - } - else - { - descr.step_func = [step = descr.fill_step](Field & field) - { - applyVisitor(FieldVisitorSum(step), field); - }; - } + descr.step_func = getStepFunction(descr.fill_step, descr.step_kind, type); + descr.staleness_step_func = getStepFunction(descr.fill_staleness, descr.staleness_kind, type); return true; } @@ -218,6 +235,7 @@ FillingTransform::FillingTransform( fill_column_positions.push_back(block_position); auto & descr = filling_row.getFillDescription(i); + running_with_staleness |= !descr.fill_staleness.isNull(); const Block & output_header = getOutputPort().getHeader(); const DataTypePtr & type = removeNullable(output_header.getByPosition(block_position).type); @@ -437,7 +455,7 @@ void FillingTransform::initColumns( non_const_columns.reserve(input_columns.size()); for (const auto & column : input_columns) - non_const_columns.push_back(column->convertToFullColumnIfConst()); + non_const_columns.push_back(column->convertToFullColumnIfConst()->convertToFullColumnIfSparse()); for (const auto & column : non_const_columns) output_columns.push_back(column->cloneEmpty()->assumeMutable()); @@ -482,26 +500,26 @@ bool FillingTransform::generateSuffixIfNeeded( MutableColumnRawPtrs res_sort_prefix_columns, MutableColumnRawPtrs res_other_columns) { - logDebug("generateSuffixIfNeeded() filling_row", filling_row); - logDebug("generateSuffixIfNeeded() next_row", next_row); + logDebug("generateSuffixIfNeeded filling_row", filling_row); + logDebug("generateSuffixIfNeeded next_row", next_row); /// Determines if we should insert filling row before start generating next rows - bool should_insert_first = (next_row < filling_row && !filling_row_inserted) || next_row.isNull(); + bool should_insert_first = (next_row < filling_row && !filling_row_inserted) || (next_row.isNull() && !filling_row.isNull()); logDebug("should_insert_first", should_insert_first); for (size_t i = 0, size = filling_row.size(); i < size; ++i) - next_row[i] = filling_row.getFillDescription(i).fill_to; + next_row[i] = Field{}; - logDebug("generateSuffixIfNeeded() next_row updated", next_row); + logDebug("generateSuffixIfNeeded next_row updated", next_row); - if (filling_row >= next_row) + if (!filling_row.hasSomeConstraints() || !filling_row.isConstraintsSatisfied()) { - logDebug("generateSuffixIfNeeded()", "no need to generate suffix"); + logDebug("generateSuffixIfNeeded", "will not generate suffix"); return false; } Block interpolate_block; - if (should_insert_first && filling_row < next_row) + if (should_insert_first) { interpolate(result_columns, interpolate_block); insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, interpolate_block); @@ -516,9 +534,7 @@ bool FillingTransform::generateSuffixIfNeeded( bool filling_row_changed = false; while (true) { - const auto [apply, changed] = filling_row.next(next_row); - filling_row_changed = changed; - if (!apply) + if (!filling_row.next(next_row, filling_row_changed)) break; interpolate(result_columns, interpolate_block); @@ -595,7 +611,7 @@ void FillingTransform::transformRange( if (!fill_from.isNull() && !equals(current_value, fill_from)) { - filling_row.initFromDefaults(i); + filling_row.initUsingFrom(i); filling_row_inserted = false; if (less(fill_from, current_value, filling_row.getDirection(i))) { @@ -609,6 +625,9 @@ void FillingTransform::transformRange( } } + /// Init staleness first interval + filling_row.updateConstraintsWithStalenessRow(input_fill_columns, range_begin); + for (size_t row_ind = range_begin; row_ind < range_end; ++row_ind) { logDebug("row", row_ind); @@ -619,21 +638,14 @@ void FillingTransform::transformRange( logDebug("should_insert_first", should_insert_first); for (size_t i = 0, size = filling_row.size(); i < size; ++i) - { - const auto current_value = (*input_fill_columns[i])[row_ind]; - const auto & fill_to = filling_row.getFillDescription(i).fill_to; + next_row[i] = (*input_fill_columns[i])[row_ind]; - if (fill_to.isNull() || less(current_value, fill_to, filling_row.getDirection(i))) - next_row[i] = current_value; - else - next_row[i] = fill_to; - } logDebug("next_row updated", next_row); /// The condition is true when filling row is initialized by value(s) in FILL FROM, /// and there are row(s) in current range with value(s) < then in the filling row. /// It can happen only once for a range. - if (should_insert_first && filling_row < next_row) + if (should_insert_first && filling_row < next_row && filling_row.isConstraintsSatisfied()) { interpolate(result_columns, interpolate_block); insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, interpolate_block); @@ -643,15 +655,37 @@ void FillingTransform::transformRange( bool filling_row_changed = false; while (true) { - const auto [apply, changed] = filling_row.next(next_row); - filling_row_changed = changed; - if (!apply) + if (!filling_row.next(next_row, filling_row_changed)) break; interpolate(result_columns, interpolate_block); insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, interpolate_block); copyRowFromColumns(res_sort_prefix_columns, input_sort_prefix_columns, row_ind); + filling_row_changed = false; } + + if (running_with_staleness) + { + /// Initialize staleness border for current row to generate it's prefix + filling_row.updateConstraintsWithStalenessRow(input_fill_columns, row_ind); + + while (filling_row.shift(next_row, filling_row_changed)) + { + logDebug("filling_row after shift", filling_row); + + do + { + logDebug("inserting prefix filling_row", filling_row); + + interpolate(result_columns, interpolate_block); + insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, interpolate_block); + copyRowFromColumns(res_sort_prefix_columns, input_sort_prefix_columns, row_ind); + filling_row_changed = false; + + } while (filling_row.next(next_row, filling_row_changed)); + } + } + /// new valid filling row was generated but not inserted, will use it during suffix generation if (filling_row_changed) filling_row_inserted = false; @@ -707,7 +741,7 @@ void FillingTransform::transform(Chunk & chunk) /// if no data was processed, then need to initialize filling_row if (last_row.empty()) { - filling_row.initFromDefaults(); + filling_row.initUsingFrom(); filling_row_inserted = false; } diff --git a/src/Processors/Transforms/FillingTransform.h b/src/Processors/Transforms/FillingTransform.h index a8866a97103..92ca4fe6c9e 100644 --- a/src/Processors/Transforms/FillingTransform.h +++ b/src/Processors/Transforms/FillingTransform.h @@ -84,6 +84,7 @@ private: SortDescription sort_prefix; const InterpolateDescriptionPtr interpolate_description; /// Contains INTERPOLATE columns + bool running_with_staleness = false; /// True if STALENESS clause was used. FillingRow filling_row; /// Current row, which is used to fill gaps. FillingRow next_row; /// Row to which we need to generate filling rows. bool filling_row_inserted = false; diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 1675e5d0386..77a437d4b97 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -394,7 +394,7 @@ void FullMergeJoinCursor::setChunk(Chunk && chunk) convertToFullIfSparse(chunk); current_chunk = std::move(chunk); - cursor = SortCursorImpl(sample_block, current_chunk.getColumns(), desc); + cursor = SortCursorImpl(sample_block, current_chunk.getColumns(), current_chunk.getNumRows(), desc); } bool FullMergeJoinCursor::fullyCompleted() const diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index d3299ea651f..5fb6a70585f 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -199,6 +199,7 @@ void MergeSortingTransform::consume(Chunk chunk) { bool have_all_inputs = false; bool use_average_block_sizes = false; + bool apply_virtual_row = false; external_merging_sorted = std::make_shared( header_without_constants, @@ -211,6 +212,7 @@ void MergeSortingTransform::consume(Chunk chunk) /*always_read_till_end_=*/ false, nullptr, use_average_block_sizes, + apply_virtual_row, have_all_inputs); processors.emplace_back(external_merging_sorted); diff --git a/src/Processors/Transforms/SortingTransform.cpp b/src/Processors/Transforms/SortingTransform.cpp index 6e65093e9e2..6a11354e2bf 100644 --- a/src/Processors/Transforms/SortingTransform.cpp +++ b/src/Processors/Transforms/SortingTransform.cpp @@ -42,7 +42,7 @@ MergeSorter::MergeSorter(const Block & header, Chunks chunks_, SortDescription & /// Convert to full column, because some cursors expect non-contant columns convertToFullIfConst(chunk); - cursors.emplace_back(header, chunk.getColumns(), description, chunk_index); + cursors.emplace_back(header, chunk.getColumns(), chunk.getNumRows(), description, chunk_index); has_collation |= cursors.back().has_collation; nonempty_chunks.emplace_back(std::move(chunk)); diff --git a/src/Processors/Transforms/VirtualRowTransform.cpp b/src/Processors/Transforms/VirtualRowTransform.cpp new file mode 100644 index 00000000000..5f2bf0b0788 --- /dev/null +++ b/src/Processors/Transforms/VirtualRowTransform.cpp @@ -0,0 +1,111 @@ +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +VirtualRowTransform::VirtualRowTransform(const Block & header_, const Block & pk_block_, ExpressionActionsPtr virtual_row_conversions_) + : IProcessor({header_}, {header_}) + , input(inputs.front()), output(outputs.front()) + , pk_block(pk_block_) + , virtual_row_conversions(std::move(virtual_row_conversions_)) +{ +} + +VirtualRowTransform::Status VirtualRowTransform::prepare() +{ + /// Check can output. + + if (output.isFinished()) + { + input.close(); + return Status::Finished; + } + + if (!output.canPush()) + { + input.setNotNeeded(); + return Status::PortFull; + } + + /// Output if has data. + if (generated) + { + output.push(std::move(current_chunk)); + generated = false; + return Status::PortFull; + } + + if (can_generate) + return Status::Ready; + + /// Check can input. + if (!has_input) + { + if (input.isFinished()) + { + output.finish(); + return Status::Finished; + } + + input.setNeeded(); + + if (!input.hasData()) + return Status::NeedData; + + /// Set input port NotNeeded after chunk was pulled. + current_chunk = input.pull(true); + has_input = true; + } + + /// Now transform. + return Status::Ready; +} + +void VirtualRowTransform::work() +{ + if (can_generate) + { + if (generated) + throw Exception(ErrorCodes::LOGICAL_ERROR, "VirtualRowTransform cannot consume chunk because it already was generated"); + + generated = true; + can_generate = false; + + if (!is_first) + { + if (current_chunk.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't generate chunk in VirtualRowTransform"); + return; + } + + is_first = false; + + Columns empty_columns; + const auto & header = getOutputs().front().getHeader(); + empty_columns.reserve(header.columns()); + for (size_t i = 0; i < header.columns(); ++i) + { + const ColumnWithTypeAndName & type_and_name = header.getByPosition(i); + empty_columns.push_back(type_and_name.type->createColumn()->cloneEmpty()); + } + + current_chunk.setColumns(empty_columns, 0); + current_chunk.getChunkInfos().add(std::make_shared(0, pk_block, virtual_row_conversions)); + } + else + { + if (!has_input) + throw Exception(ErrorCodes::LOGICAL_ERROR, "VirtualRowTransform cannot consume chunk because it wasn't read"); + + has_input = false; + can_generate = true; + } +} + +} diff --git a/src/Processors/Transforms/VirtualRowTransform.h b/src/Processors/Transforms/VirtualRowTransform.h new file mode 100644 index 00000000000..efc54419a6e --- /dev/null +++ b/src/Processors/Transforms/VirtualRowTransform.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +/// Virtual row is useful for read-in-order optimization when multiple parts exist. +class VirtualRowTransform : public IProcessor +{ +public: + explicit VirtualRowTransform(const Block & header_, const Block & pk_block_, ExpressionActionsPtr virtual_row_conversions_); + + String getName() const override { return "VirtualRowTransform"; } + + Status prepare() override; + void work() override; + +private: + InputPort & input; + OutputPort & output; + + Chunk current_chunk; + bool has_input = false; + bool generated = false; + bool can_generate = true; + bool is_first = true; + + Block pk_block; + ExpressionActionsPtr virtual_row_conversions; +}; + +} diff --git a/src/QueryPipeline/ExecutionSpeedLimits.cpp b/src/QueryPipeline/ExecutionSpeedLimits.cpp index 05fd394db77..fc0e86781f0 100644 --- a/src/QueryPipeline/ExecutionSpeedLimits.cpp +++ b/src/QueryPipeline/ExecutionSpeedLimits.cpp @@ -86,10 +86,12 @@ void ExecutionSpeedLimits::throttle( if (timeout_overflow_mode == OverflowMode::THROW && estimated_execution_time_seconds > max_estimated_execution_time.totalSeconds()) throw Exception( ErrorCodes::TOO_SLOW, - "Estimated query execution time ({} seconds) is too long. Maximum: {}. Estimated rows to process: {}", + "Estimated query execution time ({:.5f} seconds) is too long. Maximum: {}. Estimated rows to process: {} ({} read in {:.5f} seconds).", estimated_execution_time_seconds, max_estimated_execution_time.totalSeconds(), - total_rows_to_read); + total_rows_to_read, + read_rows, + elapsed_seconds); } if (max_execution_rps && rows_per_second >= max_execution_rps) diff --git a/src/Server/HTTP/authenticateUserByHTTP.cpp b/src/Server/HTTP/authenticateUserByHTTP.cpp index cbad91cc292..61029ed9560 100644 --- a/src/Server/HTTP/authenticateUserByHTTP.cpp +++ b/src/Server/HTTP/authenticateUserByHTTP.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -54,11 +55,13 @@ bool authenticateUserByHTTP( HTTPServerResponse & response, Session & session, std::unique_ptr & request_credentials, + const HTTPHandlerConnectionConfig & connection_config, ContextPtr global_context, LoggerPtr log) { /// Get the credentials created by the previous call of authenticateUserByHTTP() while handling the previous HTTP request. auto current_credentials = std::move(request_credentials); + const auto & config_credentials = connection_config.credentials; /// The user and password can be passed by headers (similar to X-Auth-*), /// which is used by load balancers to pass authentication information. @@ -70,6 +73,7 @@ bool authenticateUserByHTTP( /// The header 'X-ClickHouse-SSL-Certificate-Auth: on' enables checking the common name /// extracted from the SSL certificate used for this connection instead of checking password. bool has_ssl_certificate_auth = (request.get("X-ClickHouse-SSL-Certificate-Auth", "") == "on"); + bool has_config_credentials = config_credentials.has_value(); /// User name and password can be passed using HTTP Basic auth or query parameters /// (both methods are insecure). @@ -79,6 +83,10 @@ bool authenticateUserByHTTP( std::string spnego_challenge; SSLCertificateSubjects certificate_subjects; + if (config_credentials) + { + checkUserNameNotEmpty(config_credentials->getUserName(), "config authentication"); + } if (has_ssl_certificate_auth) { #if USE_SSL @@ -86,6 +94,8 @@ bool authenticateUserByHTTP( checkUserNameNotEmpty(user, "X-ClickHouse HTTP headers"); /// It is prohibited to mix different authorization schemes. + if (has_config_credentials) + throwMultipleAuthenticationMethods("SSL certificate authentication", "authentication set in config"); if (!password.empty()) throwMultipleAuthenticationMethods("SSL certificate authentication", "authentication via password"); if (has_http_credentials) @@ -109,6 +119,8 @@ bool authenticateUserByHTTP( checkUserNameNotEmpty(user, "X-ClickHouse HTTP headers"); /// It is prohibited to mix different authorization schemes. + if (has_config_credentials) + throwMultipleAuthenticationMethods("X-ClickHouse HTTP headers", "authentication set in config"); if (has_http_credentials) throwMultipleAuthenticationMethods("X-ClickHouse HTTP headers", "Authorization HTTP header"); if (has_credentials_in_query_params) @@ -117,6 +129,8 @@ bool authenticateUserByHTTP( else if (has_http_credentials) { /// It is prohibited to mix different authorization schemes. + if (has_config_credentials) + throwMultipleAuthenticationMethods("Authorization HTTP header", "authentication set in config"); if (has_credentials_in_query_params) throwMultipleAuthenticationMethods("Authorization HTTP header", "authentication via parameters"); @@ -190,6 +204,10 @@ bool authenticateUserByHTTP( return false; } } + else if (has_config_credentials) + { + current_credentials = std::make_unique(*config_credentials); + } else // I.e., now using user name and password strings ("Basic"). { if (!current_credentials) diff --git a/src/Server/HTTP/authenticateUserByHTTP.h b/src/Server/HTTP/authenticateUserByHTTP.h index 3b5a04cae68..02dcf828faa 100644 --- a/src/Server/HTTP/authenticateUserByHTTP.h +++ b/src/Server/HTTP/authenticateUserByHTTP.h @@ -11,13 +11,22 @@ class HTMLForm; class HTTPServerResponse; class Session; class Credentials; +class BasicCredentials; +struct HTTPHandlerConnectionConfig; /// Authenticates a user via HTTP protocol and initializes a session. +/// /// Usually retrieves the name and the password for that user from either the request's headers or from the query parameters. -/// Returns true when the user successfully authenticated, -/// the session instance will be configured accordingly, and the request_credentials instance will be dropped. -/// Returns false when the user is not authenticated yet, and the HTTP_UNAUTHORIZED response is sent with the "WWW-Authenticate" header, -/// in this case the `request_credentials` instance must be preserved until the next request or until any exception. +/// You can also pass user/password explicitly via `config_credentials`. +/// +/// Returns true when the user successfully authenticated: +/// - the session instance will be configured accordingly +/// - and the request_credentials instance will be dropped. +/// +/// Returns false when the user is not authenticated yet: +/// - the HTTP_UNAUTHORIZED response is sent with the "WWW-Authenticate" header +/// - the `request_credentials` instance must be preserved until the next request or until any exception. +/// /// Throws an exception if authentication failed. bool authenticateUserByHTTP( const HTTPServerRequest & request, @@ -25,6 +34,7 @@ bool authenticateUserByHTTP( HTTPServerResponse & response, Session & session, std::unique_ptr & request_credentials, + const HTTPHandlerConnectionConfig & connection_config, ContextPtr global_context, LoggerPtr log); diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 52b56860543..a9a981258a1 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include @@ -145,6 +144,15 @@ static std::chrono::steady_clock::duration parseSessionTimeout( return std::chrono::seconds(session_timeout); } +HTTPHandlerConnectionConfig::HTTPHandlerConnectionConfig(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix) +{ + if (config.has(config_prefix + ".handler.user") || config.has(config_prefix + ".handler.password")) + { + credentials.emplace( + config.getString(config_prefix + ".handler.user", "default"), + config.getString(config_prefix + ".handler.password", "")); + } +} void HTTPHandler::pushDelayedResults(Output & used_output) { @@ -183,11 +191,12 @@ void HTTPHandler::pushDelayedResults(Output & used_output) } -HTTPHandler::HTTPHandler(IServer & server_, const std::string & name, const HTTPResponseHeaderSetup & http_response_headers_override_) +HTTPHandler::HTTPHandler(IServer & server_, const HTTPHandlerConnectionConfig & connection_config_, const std::string & name, const HTTPResponseHeaderSetup & http_response_headers_override_) : server(server_) , log(getLogger(name)) , default_settings(server.context()->getSettingsRef()) , http_response_headers_override(http_response_headers_override_) + , connection_config(connection_config_) { server_display_name = server.config().getString("display_name", getFQDNOrHostName()); } @@ -200,7 +209,7 @@ HTTPHandler::~HTTPHandler() = default; bool HTTPHandler::authenticateUser(HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response) { - return authenticateUserByHTTP(request, params, response, *session, request_credentials, server.context(), log); + return authenticateUserByHTTP(request, params, response, *session, request_credentials, connection_config, server.context(), log); } @@ -767,8 +776,12 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse } DynamicQueryHandler::DynamicQueryHandler( - IServer & server_, const std::string & param_name_, const HTTPResponseHeaderSetup & http_response_headers_override_) - : HTTPHandler(server_, "DynamicQueryHandler", http_response_headers_override_), param_name(param_name_) + IServer & server_, + const HTTPHandlerConnectionConfig & connection_config, + const std::string & param_name_, + const HTTPResponseHeaderSetup & http_response_headers_override_) + : HTTPHandler(server_, connection_config, "DynamicQueryHandler", http_response_headers_override_) + , param_name(param_name_) { } @@ -825,12 +838,13 @@ std::string DynamicQueryHandler::getQuery(HTTPServerRequest & request, HTMLForm PredefinedQueryHandler::PredefinedQueryHandler( IServer & server_, + const HTTPHandlerConnectionConfig & connection_config, const NameSet & receive_params_, const std::string & predefined_query_, const CompiledRegexPtr & url_regex_, const std::unordered_map & header_name_with_regex_, const HTTPResponseHeaderSetup & http_response_headers_override_) - : HTTPHandler(server_, "PredefinedQueryHandler", http_response_headers_override_) + : HTTPHandler(server_, connection_config, "PredefinedQueryHandler", http_response_headers_override_) , receive_params(receive_params_) , predefined_query(predefined_query_) , url_regex(url_regex_) @@ -922,10 +936,11 @@ HTTPRequestHandlerFactoryPtr createDynamicHandlerFactory(IServer & server, { auto query_param_name = config.getString(config_prefix + ".handler.query_param_name", "query"); + HTTPHandlerConnectionConfig connection_config(config, config_prefix); HTTPResponseHeaderSetup http_response_headers_override = parseHTTPResponseHeaders(config, config_prefix); - auto creator = [&server, query_param_name, http_response_headers_override]() -> std::unique_ptr - { return std::make_unique(server, query_param_name, http_response_headers_override); }; + auto creator = [&server, query_param_name, http_response_headers_override, connection_config]() -> std::unique_ptr + { return std::make_unique(server, connection_config, query_param_name, http_response_headers_override); }; auto factory = std::make_shared>(std::move(creator)); factory->addFiltersFromConfig(config, config_prefix); @@ -967,6 +982,8 @@ HTTPRequestHandlerFactoryPtr createPredefinedHandlerFactory(IServer & server, Poco::Util::AbstractConfiguration::Keys headers_name; config.keys(config_prefix + ".headers", headers_name); + HTTPHandlerConnectionConfig connection_config(config, config_prefix); + for (const auto & header_name : headers_name) { auto expression = config.getString(config_prefix + ".headers." + header_name); @@ -1000,12 +1017,18 @@ HTTPRequestHandlerFactoryPtr createPredefinedHandlerFactory(IServer & server, predefined_query, regex, headers_name_with_regex, - http_response_headers_override] + http_response_headers_override, + connection_config] -> std::unique_ptr { return std::make_unique( - server, analyze_receive_params, predefined_query, regex, - headers_name_with_regex, http_response_headers_override); + server, + connection_config, + analyze_receive_params, + predefined_query, + regex, + headers_name_with_regex, + http_response_headers_override); }; factory = std::make_shared>(std::move(creator)); factory->addFiltersFromConfig(config, config_prefix); @@ -1018,18 +1041,21 @@ HTTPRequestHandlerFactoryPtr createPredefinedHandlerFactory(IServer & server, analyze_receive_params, predefined_query, headers_name_with_regex, - http_response_headers_override] + http_response_headers_override, + connection_config] -> std::unique_ptr { return std::make_unique( - server, analyze_receive_params, predefined_query, CompiledRegexPtr{}, - headers_name_with_regex, http_response_headers_override); + server, + connection_config, + analyze_receive_params, + predefined_query, + CompiledRegexPtr{}, + headers_name_with_regex, + http_response_headers_override); }; - factory = std::make_shared>(std::move(creator)); - factory->addFiltersFromConfig(config, config_prefix); - return factory; } diff --git a/src/Server/HTTPHandler.h b/src/Server/HTTPHandler.h index 6580b317f6e..2296fa70aeb 100644 --- a/src/Server/HTTPHandler.h +++ b/src/Server/HTTPHandler.h @@ -12,6 +12,7 @@ #include #include #include +#include #include "HTTPResponseHeaderWriter.h" @@ -26,17 +27,28 @@ namespace DB { class Session; -class Credentials; class IServer; struct Settings; class WriteBufferFromHTTPServerResponse; using CompiledRegexPtr = std::shared_ptr; +struct HTTPHandlerConnectionConfig +{ + std::optional credentials; + + /// TODO: + /// String quota; + /// String default_database; + + HTTPHandlerConnectionConfig() = default; + HTTPHandlerConnectionConfig(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); +}; + class HTTPHandler : public HTTPRequestHandler { public: - HTTPHandler(IServer & server_, const std::string & name, const HTTPResponseHeaderSetup & http_response_headers_override_); + HTTPHandler(IServer & server_, const HTTPHandlerConnectionConfig & connection_config_, const std::string & name, const HTTPResponseHeaderSetup & http_response_headers_override_); ~HTTPHandler() override; void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & write_event) override; @@ -146,16 +158,7 @@ private: // The request_credential instance may outlive a single request/response loop. // This happens only when the authentication mechanism requires more than a single request/response exchange (e.g., SPNEGO). std::unique_ptr request_credentials; - - // Returns true when the user successfully authenticated, - // the session instance will be configured accordingly, and the request_credentials instance will be dropped. - // Returns false when the user is not authenticated yet, and the 'Negotiate' response is sent, - // the session and request_credentials instances are preserved. - // Throws an exception if authentication failed. - bool authenticateUser( - HTTPServerRequest & request, - HTMLForm & params, - HTTPServerResponse & response); + HTTPHandlerConnectionConfig connection_config; /// Also initializes 'used_output'. void processQuery( @@ -174,6 +177,13 @@ private: Output & used_output); static void pushDelayedResults(Output & used_output); + +protected: + // @see authenticateUserByHTTP() + virtual bool authenticateUser( + HTTPServerRequest & request, + HTMLForm & params, + HTTPServerResponse & response); }; class DynamicQueryHandler : public HTTPHandler @@ -184,6 +194,7 @@ private: public: explicit DynamicQueryHandler( IServer & server_, + const HTTPHandlerConnectionConfig & connection_config, const std::string & param_name_ = "query", const HTTPResponseHeaderSetup & http_response_headers_override_ = std::nullopt); @@ -203,6 +214,7 @@ private: public: PredefinedQueryHandler( IServer & server_, + const HTTPHandlerConnectionConfig & connection_config, const NameSet & receive_params_, const std::string & predefined_query_, const CompiledRegexPtr & url_regex_, diff --git a/src/Server/HTTPHandlerFactory.cpp b/src/Server/HTTPHandlerFactory.cpp index 2d5ddd859fe..950cad4038a 100644 --- a/src/Server/HTTPHandlerFactory.cpp +++ b/src/Server/HTTPHandlerFactory.cpp @@ -275,7 +275,7 @@ void addDefaultHandlersFactory( auto dynamic_creator = [&server] () -> std::unique_ptr { - return std::make_unique(server, "query"); + return std::make_unique(server, HTTPHandlerConnectionConfig{}, "query"); }; auto query_handler = std::make_shared>(std::move(dynamic_creator)); query_handler->addFilter([](const auto & request) diff --git a/src/Server/PrometheusRequestHandler.cpp b/src/Server/PrometheusRequestHandler.cpp index cd18eac50a7..9c521e06667 100644 --- a/src/Server/PrometheusRequestHandler.cpp +++ b/src/Server/PrometheusRequestHandler.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include "config.h" #include @@ -137,7 +138,7 @@ protected: bool authenticateUser(HTTPServerRequest & request, HTTPServerResponse & response) { - return authenticateUserByHTTP(request, *params, response, *session, request_credentials, server().context(), log()); + return authenticateUserByHTTP(request, *params, response, *session, request_credentials, HTTPHandlerConnectionConfig{}, server().context(), log()); } void makeContext(HTTPServerRequest & request) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index e7e4ae25a68..4f54918445f 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1614,7 +1614,8 @@ void TCPHandler::receiveHello() if (e.code() != DB::ErrorCodes::AUTHENTICATION_FAILED) throw; - tryLogCurrentException(log, "SSL authentication failed, falling back to password authentication"); + tryLogCurrentException(log, "SSL authentication failed, falling back to password authentication", LogsLevel::information); + /// ^^ Log at debug level instead of default error level as authentication failures are not an unusual event. } } } diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index ab4403b3a94..fbca222b1e7 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -1457,14 +1457,6 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const ErrorCodes::BAD_ARGUMENTS, "The change of data type {} of column {} to {} is not allowed. It has known bugs", old_data_type->getName(), backQuote(column_name), command.data_type->getName()); - - bool has_object_type = isObject(command.data_type); - command.data_type->forEachChild([&](const IDataType & type){ has_object_type |= isObject(type); }); - if (has_object_type) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "The change of data type {} of column {} to {} is not supported.", - old_data_type->getName(), backQuote(column_name), command.data_type->getName()); } if (command.isRemovingProperty()) @@ -1496,7 +1488,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const if (command.to_remove == AlterCommand::RemoveProperty::CODEC && column_from_table.codec == nullptr) throw Exception( ErrorCodes::BAD_ARGUMENTS, - "Column {} doesn't have TTL, cannot remove it", + "Column {} doesn't have CODEC, cannot remove it", backQuote(column_name)); if (command.to_remove == AlterCommand::RemoveProperty::COMMENT && column_from_table.comment.empty()) throw Exception( diff --git a/src/Storages/IStorageCluster.h b/src/Storages/IStorageCluster.h index 893cf222556..d000e24562f 100644 --- a/src/Storages/IStorageCluster.h +++ b/src/Storages/IStorageCluster.h @@ -10,8 +10,8 @@ namespace DB /** - * Base cluster for Storages used in table functions like s3Cluster and hdfsCluster - * Needed for code simplification around parallel_distributed_insert_select + * Base cluster for Storages used in table functions like s3Cluster and hdfsCluster. + * Necessary for code simplification around parallel_distributed_insert_select. */ class IStorageCluster : public IStorage { diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp index 7e43966556e..5c0449612e7 100644 --- a/src/Storages/KeyDescription.cpp +++ b/src/Storages/KeyDescription.cpp @@ -151,6 +151,18 @@ KeyDescription KeyDescription::getSortingKeyFromAST( throw Exception(ErrorCodes::DATA_TYPE_CANNOT_BE_USED_IN_KEY, "Column {} with type {} is not allowed in key expression, it's not comparable", backQuote(result.sample_block.getByPosition(i).name), result.data_types.back()->getName()); + + auto check = [&](const IDataType & type) + { + if (isDynamic(type) || isVariant(type)) + throw Exception( + ErrorCodes::DATA_TYPE_CANNOT_BE_USED_IN_KEY, + "Column with type Variant/Dynamic is not allowed in key expression. Consider using a subcolumn with a specific data " + "type instead (for example 'column.Int64' or 'json.some.path.:Int64' if its a JSON path subcolumn) or casting this column to a specific data type"); + }; + + check(*result.data_types.back()); + result.data_types.back()->forEachChild(check); } return result; diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index e13ec5a7515..1d79ae5aacb 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -908,7 +908,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( { part_storage_for_loading->commitTransaction(); - MergeTreeDataPartBuilder builder(data, part_name, volume, part_relative_path, part_dir); + MergeTreeDataPartBuilder builder(data, part_name, volume, part_relative_path, part_dir, getReadSettings()); new_data_part = builder.withPartFormatFromDisk().build(); new_data_part->version.setCreationTID(Tx::PrehistoricTID, nullptr); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 20d7528d38a..51c445945e6 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -624,6 +624,15 @@ UInt64 IMergeTreeDataPart::getIndexSizeInAllocatedBytes() const return res; } +UInt64 IMergeTreeDataPart::getIndexGranularityBytes() const +{ + return index_granularity.getBytesSize(); +} +UInt64 IMergeTreeDataPart::getIndexGranularityAllocatedBytes() const +{ + return index_granularity.getBytesAllocated(); +} + void IMergeTreeDataPart::assertState(const std::initializer_list & affordable_states) const { if (!checkState(affordable_states)) @@ -735,7 +744,9 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks loadUUID(); loadColumns(require_columns_checksums); loadChecksums(require_columns_checksums); + loadIndexGranularity(); + index_granularity.shrinkToFitInMemory(); if (!(*storage.getSettings())[MergeTreeSetting::primary_key_lazy_load]) getIndex(); @@ -833,7 +844,7 @@ MergeTreeDataPartBuilder IMergeTreeDataPart::getProjectionPartBuilder(const Stri { const char * projection_extension = is_temp_projection ? ".tmp_proj" : ".proj"; auto projection_storage = getDataPartStorage().getProjection(projection_name + projection_extension, !is_temp_projection); - MergeTreeDataPartBuilder builder(storage, projection_name, projection_storage); + MergeTreeDataPartBuilder builder(storage, projection_name, projection_storage, getReadSettings()); return builder.withPartInfo(MergeListElement::FAKE_RESULT_PART_FOR_PROJECTION).withParentPart(this); } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index b41a1d840e1..55f1265318c 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -380,6 +380,8 @@ public: /// For data in RAM ('index') UInt64 getIndexSizeInBytes() const; UInt64 getIndexSizeInAllocatedBytes() const; + UInt64 getIndexGranularityBytes() const; + UInt64 getIndexGranularityAllocatedBytes() const; UInt64 getMarksCount() const; UInt64 getIndexSizeFromFile() const; diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index b8ac14b1750..d1c76505d7c 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -46,6 +46,8 @@ public: virtual void finish(bool sync) = 0; + virtual size_t getNumberOfOpenStreams() const = 0; + Columns releaseIndexColumns(); PlainMarksByName releaseCachedMarks(); diff --git a/src/Storages/MergeTree/IMergedBlockOutputStream.h b/src/Storages/MergeTree/IMergedBlockOutputStream.h index a901b03c115..7dd6d720170 100644 --- a/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -39,6 +39,11 @@ public: return writer->releaseCachedMarks(); } + size_t getNumberOfOpenStreams() const + { + return writer->getNumberOfOpenStreams(); + } + protected: /// Remove all columns marked expired in data_part. Also, clears checksums diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 17723d341fb..9e19ffe1b04 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -597,12 +597,15 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown( case (ActionsDAG::ActionType::COLUMN): { String name; - if (const auto * column_const = typeid_cast(node.column.get())) + if (const auto * column_const = typeid_cast(node.column.get()); + column_const && column_const->getDataType() != TypeIndex::Function) + { /// Re-generate column name for constant. - /// DAG form query (with enabled analyzer) uses suffixes for constants, like 1_UInt8. - /// DAG from PK does not use it. This breaks matching by column name sometimes. + /// DAG from the query (with enabled analyzer) uses suffixes for constants, like 1_UInt8. + /// DAG from the PK does not use it. This breaks matching by column name sometimes. /// Ideally, we should not compare names, but DAG subtrees instead. - name = ASTLiteral(column_const->getDataColumn()[0]).getColumnName(); + name = ASTLiteral(column_const->getField()).getColumnName(); + } else name = node.result_name; @@ -1158,6 +1161,7 @@ bool KeyCondition::tryPrepareSetIndex( const RPNBuilderFunctionTreeNode & func, RPNElement & out, size_t & out_key_column_num, + bool & allow_constant_transformation, bool & is_constant_transformed) { const auto & left_arg = func.getArgumentAt(0); @@ -1184,7 +1188,9 @@ bool KeyCondition::tryPrepareSetIndex( set_transforming_chains.push_back(set_transforming_chain); } // For partition index, checking if set can be transformed to prune any partitions - else if (single_point && canSetValuesBeWrappedByFunctions(node, index_mapping.key_index, data_type, set_transforming_chain)) + else if ( + single_point && allow_constant_transformation + && canSetValuesBeWrappedByFunctions(node, index_mapping.key_index, data_type, set_transforming_chain)) { indexes_mapping.push_back(index_mapping); data_types.push_back(data_type); @@ -1954,7 +1960,7 @@ bool KeyCondition::extractAtomFromTree(const RPNBuilderTreeNode & node, RPNEleme if (functionIsInOrGlobalInOperator(func_name)) { - if (tryPrepareSetIndex(func, out, key_column_num, is_constant_transformed)) + if (tryPrepareSetIndex(func, out, key_column_num, allow_constant_transformation, is_constant_transformed)) { key_arg_pos = 0; is_set_const = true; diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h index 8c946bd3bbd..20b40271dc2 100644 --- a/src/Storages/MergeTree/KeyCondition.h +++ b/src/Storages/MergeTree/KeyCondition.h @@ -312,6 +312,7 @@ private: const RPNBuilderFunctionTreeNode & func, RPNElement & out, size_t & out_key_column_num, + bool & allow_constant_transformation, bool & is_constant_transformed); /// Checks that the index can not be used. diff --git a/src/Storages/MergeTree/MergeSelectors/SimpleMergeSelector.cpp b/src/Storages/MergeTree/MergeSelectors/SimpleMergeSelector.cpp index c393349ef32..4f786215cbe 100644 --- a/src/Storages/MergeTree/MergeSelectors/SimpleMergeSelector.cpp +++ b/src/Storages/MergeTree/MergeSelectors/SimpleMergeSelector.cpp @@ -116,7 +116,7 @@ bool allow( double sum_size, double max_size, double min_age, - double range_size, + size_t range_size, double partition_size, double min_size_to_lower_base_log, double max_size_to_lower_base_log, @@ -125,6 +125,9 @@ bool allow( if (settings.min_age_to_force_merge && min_age >= settings.min_age_to_force_merge) return true; + if (settings.min_parts_to_merge_at_once && range_size < settings.min_parts_to_merge_at_once) + return false; + /// Map size to 0..1 using logarithmic scale /// Use log(1 + x) instead of log1p(x) because our sum_size is always integer. /// Also log1p seems to be slow and significantly affect performance of merges assignment. diff --git a/src/Storages/MergeTree/MergeSelectors/SimpleMergeSelector.h b/src/Storages/MergeTree/MergeSelectors/SimpleMergeSelector.h index 2d4129b8bf8..1e7676c6aed 100644 --- a/src/Storages/MergeTree/MergeSelectors/SimpleMergeSelector.h +++ b/src/Storages/MergeTree/MergeSelectors/SimpleMergeSelector.h @@ -90,6 +90,8 @@ public: { /// Zero means unlimited. Can be overridden by the same merge tree setting. size_t max_parts_to_merge_at_once = 100; + /// Zero means no minimum. Can be overridden by the same merge tree setting. + size_t min_parts_to_merge_at_once = 0; /// Some sort of a maximum number of parts in partition. Can be overridden by the same merge tree setting. size_t parts_to_throw_insert = 3000; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index e73bc18557c..ba218dc1b7d 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -352,13 +352,13 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const if (global_ctx->parent_part) { auto data_part_storage = global_ctx->parent_part->getDataPartStorage().getProjection(local_tmp_part_basename, /* use parent transaction */ false); - builder.emplace(*global_ctx->data, global_ctx->future_part->name, data_part_storage); + builder.emplace(*global_ctx->data, global_ctx->future_part->name, data_part_storage, getReadSettings()); builder->withParentPart(global_ctx->parent_part); } else { auto local_single_disk_volume = std::make_shared("volume_" + global_ctx->future_part->name, global_ctx->disk, 0); - builder.emplace(global_ctx->data->getDataPartBuilder(global_ctx->future_part->name, local_single_disk_volume, local_tmp_part_basename)); + builder.emplace(global_ctx->data->getDataPartBuilder(global_ctx->future_part->name, local_single_disk_volume, local_tmp_part_basename, getReadSettings())); builder->withPartStorageType(global_ctx->future_part->part_format.storage_type); } diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 7ba358d2d35..4a7e02a7a51 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -330,7 +330,7 @@ MergeTreeReadTaskColumns getReadTaskColumns( auto prewhere_actions = MergeTreeSelectProcessor::getPrewhereActions( prewhere_info, actions_settings, - reader_settings.enable_multiple_prewhere_read_steps); + reader_settings.enable_multiple_prewhere_read_steps, reader_settings.force_short_circuit_execution); for (const auto & step : prewhere_actions.steps) add_step(*step); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 4ed8c67469d..b2f35d0a309 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1442,7 +1442,7 @@ void MergeTreeData::loadUnexpectedDataPart(UnexpectedPartLoadState & state) try { - state.part = getDataPartBuilder(part_name, single_disk_volume, part_name) + state.part = getDataPartBuilder(part_name, single_disk_volume, part_name, getReadSettings()) .withPartInfo(part_info) .withPartFormatFromDisk() .build(); @@ -1457,7 +1457,7 @@ void MergeTreeData::loadUnexpectedDataPart(UnexpectedPartLoadState & state) /// Build a fake part and mark it as broken in case of filesystem error. /// If the error impacts part directory instead of single files, /// an exception will be thrown during detach and silently ignored. - state.part = getDataPartBuilder(part_name, single_disk_volume, part_name) + state.part = getDataPartBuilder(part_name, single_disk_volume, part_name, getReadSettings()) .withPartStorageType(MergeTreeDataPartStorageType::Full) .withPartType(MergeTreeDataPartType::Wide) .build(); @@ -1491,7 +1491,7 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPart( /// Build a fake part and mark it as broken in case of filesystem error. /// If the error impacts part directory instead of single files, /// an exception will be thrown during detach and silently ignored. - res.part = getDataPartBuilder(part_name, single_disk_volume, part_name) + res.part = getDataPartBuilder(part_name, single_disk_volume, part_name, getReadSettings()) .withPartStorageType(MergeTreeDataPartStorageType::Full) .withPartType(MergeTreeDataPartType::Wide) .build(); @@ -1512,7 +1512,7 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPart( try { - res.part = getDataPartBuilder(part_name, single_disk_volume, part_name) + res.part = getDataPartBuilder(part_name, single_disk_volume, part_name, getReadSettings()) .withPartInfo(part_info) .withPartFormatFromDisk() .build(); @@ -2343,11 +2343,16 @@ void MergeTreeData::stopOutdatedAndUnexpectedDataPartsLoadingTask() } } -void MergeTreeData::prewarmMarkCache(ThreadPool & pool) +void MergeTreeData::prewarmMarkCacheIfNeeded(ThreadPool & pool) { if (!(*getSettings())[MergeTreeSetting::prewarm_mark_cache]) return; + prewarmMarkCache(pool); +} + +void MergeTreeData::prewarmMarkCache(ThreadPool & pool) +{ auto * mark_cache = getContext()->getMarkCache().get(); if (!mark_cache) return; @@ -3830,9 +3835,9 @@ MergeTreeDataPartFormat MergeTreeData::choosePartFormatOnDisk(size_t bytes_uncom } MergeTreeDataPartBuilder MergeTreeData::getDataPartBuilder( - const String & name, const VolumePtr & volume, const String & part_dir) const + const String & name, const VolumePtr & volume, const String & part_dir, const ReadSettings & read_settings_) const { - return MergeTreeDataPartBuilder(*this, name, volume, relative_data_path, part_dir); + return MergeTreeDataPartBuilder(*this, name, volume, relative_data_path, part_dir, read_settings_); } void MergeTreeData::changeSettings( @@ -5914,7 +5919,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartRestoredFromBackup(cons /// Load this part from the directory `temp_part_dir`. auto load_part = [&] { - MergeTreeDataPartBuilder builder(*this, part_name, single_disk_volume, parent_part_dir, part_dir_name); + MergeTreeDataPartBuilder builder(*this, part_name, single_disk_volume, parent_part_dir, part_dir_name, getReadSettings()); builder.withPartFormatFromDisk(); part = std::move(builder).build(); part->version.setCreationTID(Tx::PrehistoricTID, nullptr); @@ -5929,7 +5934,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartRestoredFromBackup(cons if (!part) { /// Make a fake data part only to copy its files to /detached/. - part = MergeTreeDataPartBuilder{*this, part_name, single_disk_volume, parent_part_dir, part_dir_name} + part = MergeTreeDataPartBuilder{*this, part_name, single_disk_volume, parent_part_dir, part_dir_name, getReadSettings()} .withPartStorageType(MergeTreeDataPartStorageType::Full) .withPartType(MergeTreeDataPartType::Wide) .build(); @@ -6581,7 +6586,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const LOG_DEBUG(log, "Checking part {}", new_name); auto single_disk_volume = std::make_shared("volume_" + old_name, disk); - auto part = getDataPartBuilder(old_name, single_disk_volume, source_dir / new_name) + auto part = getDataPartBuilder(old_name, single_disk_volume, source_dir / new_name, getReadSettings()) .withPartFormatFromDisk() .build(); @@ -7636,7 +7641,7 @@ std::pair MergeTreeData::cloneAn std::string(fs::path(dst_part_storage->getFullRootPath()) / tmp_dst_part_name), with_copy); - auto dst_data_part = MergeTreeDataPartBuilder(*this, dst_part_name, dst_part_storage) + auto dst_data_part = MergeTreeDataPartBuilder(*this, dst_part_name, dst_part_storage, getReadSettings()) .withPartFormatFromDisk() .build(); @@ -8905,7 +8910,7 @@ std::pair MergeTreeData::createE VolumePtr data_part_volume = createVolumeFromReservation(reservation, volume); auto tmp_dir_holder = getTemporaryPartDirectoryHolder(EMPTY_PART_TMP_PREFIX + new_part_name); - auto new_data_part = getDataPartBuilder(new_part_name, data_part_volume, EMPTY_PART_TMP_PREFIX + new_part_name) + auto new_data_part = getDataPartBuilder(new_part_name, data_part_volume, EMPTY_PART_TMP_PREFIX + new_part_name, getReadSettings()) .withBytesAndRowsOnDisk(0, 0) .withPartInfo(new_part_info) .build(); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index a32106f76bb..fe360907875 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -241,7 +241,7 @@ public: MergeTreeDataPartFormat choosePartFormat(size_t bytes_uncompressed, size_t rows_count) const; MergeTreeDataPartFormat choosePartFormatOnDisk(size_t bytes_uncompressed, size_t rows_count) const; - MergeTreeDataPartBuilder getDataPartBuilder(const String & name, const VolumePtr & volume, const String & part_dir) const; + MergeTreeDataPartBuilder getDataPartBuilder(const String & name, const VolumePtr & volume, const String & part_dir, const ReadSettings & read_settings_) const; /// Auxiliary object to add a set of parts into the working set in two steps: /// * First, as PreActive parts (the parts are ready, but not yet in the active set). @@ -508,6 +508,7 @@ public: /// Prewarm mark cache for the most recent data parts. void prewarmMarkCache(ThreadPool & pool); + void prewarmMarkCacheIfNeeded(ThreadPool & pool); String getLogName() const { return log.loadName(); } diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 62ad9d4a52a..488f4b2390d 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -48,6 +48,16 @@ namespace CurrentMetrics { extern const Metric BackgroundMergesAndMutationsPoolTask; } +namespace ProfileEvents +{ + + extern const Event MergerMutatorsGetPartsForMergeElapsedMicroseconds; + extern const Event MergerMutatorPrepareRangesForMergeElapsedMicroseconds; + extern const Event MergerMutatorSelectPartsForMergeElapsedMicroseconds; + extern const Event MergerMutatorRangesForMergeCount; + extern const Event MergerMutatorPartsInRangesForMergeCount; + extern const Event MergerMutatorSelectRangePartsCount; +} namespace DB { @@ -71,6 +81,8 @@ namespace MergeTreeSetting extern const MergeTreeSettingsUInt64 parts_to_throw_insert; extern const MergeTreeSettingsMergeSelectorAlgorithm merge_selector_algorithm; extern const MergeTreeSettingsBool merge_selector_enable_heuristic_to_remove_small_parts_at_right; + extern const MergeTreeSettingsFloat merge_selector_base; + extern const MergeTreeSettingsUInt64 min_parts_to_merge_at_once; } namespace ErrorCodes @@ -214,6 +226,7 @@ MergeTreeDataMergerMutator::PartitionIdsHint MergeTreeDataMergerMutator::getPart { PartitionIdsHint res; MergeTreeData::DataPartsVector data_parts = getDataPartsToSelectMergeFrom(txn); + if (data_parts.empty()) return res; @@ -255,7 +268,8 @@ MergeTreeDataMergerMutator::PartitionIdsHint MergeTreeDataMergerMutator::getPart if (status == SelectPartsDecision::SELECTED) res.insert(all_partition_ids[i]); else - LOG_TEST(log, "Nothing to merge in partition {}: {}", all_partition_ids[i], out_disable_reason.text); + LOG_TEST(log, "Nothing to merge in partition {} with max_total_size_to_merge = {} (looked up {} ranges): {}", + all_partition_ids[i], ReadableSize(max_total_size_to_merge), ranges_per_partition[i].size(), out_disable_reason.text); } String best_partition_id_to_optimize = getBestPartitionToOptimizeEntire(info.partitions_info); @@ -271,6 +285,8 @@ MergeTreeDataMergerMutator::PartitionIdsHint MergeTreeDataMergerMutator::getPart MergeTreeData::DataPartsVector MergeTreeDataMergerMutator::getDataPartsToSelectMergeFrom( const MergeTreeTransactionPtr & txn, const PartitionIdsHint * partitions_hint) const { + + Stopwatch get_data_parts_for_merge_timer; auto res = getDataPartsToSelectMergeFrom(txn); if (!partitions_hint) return res; @@ -279,6 +295,8 @@ MergeTreeData::DataPartsVector MergeTreeDataMergerMutator::getDataPartsToSelectM { return !partitions_hint->contains(part->info.partition_id); }); + + ProfileEvents::increment(ProfileEvents::MergerMutatorsGetPartsForMergeElapsedMicroseconds, get_data_parts_for_merge_timer.elapsedMicroseconds()); return res; } @@ -356,6 +374,7 @@ MergeTreeDataMergerMutator::MergeSelectingInfo MergeTreeDataMergerMutator::getPo const MergeTreeTransactionPtr & txn, PreformattedMessage & out_disable_reason) const { + Stopwatch ranges_for_merge_timer; MergeSelectingInfo res; res.current_time = std::time(nullptr); @@ -456,6 +475,10 @@ MergeTreeDataMergerMutator::MergeSelectingInfo MergeTreeDataMergerMutator::getPo prev_part = ∂ } + ProfileEvents::increment(ProfileEvents::MergerMutatorPartsInRangesForMergeCount, res.parts_selected_precondition); + ProfileEvents::increment(ProfileEvents::MergerMutatorRangesForMergeCount, res.parts_ranges.size()); + ProfileEvents::increment(ProfileEvents::MergerMutatorPrepareRangesForMergeElapsedMicroseconds, ranges_for_merge_timer.elapsedMicroseconds()); + return res; } @@ -470,6 +493,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges( PreformattedMessage & out_disable_reason, bool dry_run) { + Stopwatch select_parts_from_ranges_timer; const auto data_settings = data.getSettings(); IMergeSelector::PartsRange parts_to_merge; @@ -542,6 +566,8 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges( simple_merge_settings.window_size = (*data_settings)[MergeTreeSetting::merge_selector_window_size]; simple_merge_settings.max_parts_to_merge_at_once = (*data_settings)[MergeTreeSetting::max_parts_to_merge_at_once]; simple_merge_settings.enable_heuristic_to_remove_small_parts_at_right = (*data_settings)[MergeTreeSetting::merge_selector_enable_heuristic_to_remove_small_parts_at_right]; + simple_merge_settings.base = (*data_settings)[MergeTreeSetting::merge_selector_base]; + simple_merge_settings.min_parts_to_merge_at_once = (*data_settings)[MergeTreeSetting::min_parts_to_merge_at_once]; if (!(*data_settings)[MergeTreeSetting::min_age_to_force_merge_on_partition_only]) simple_merge_settings.min_age_to_force_merge = (*data_settings)[MergeTreeSetting::min_age_to_force_merge_seconds]; @@ -568,7 +594,8 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges( if (parts_to_merge.empty()) { - out_disable_reason = PreformattedMessage::create("Did not find any parts to merge (with usual merge selectors)"); + ProfileEvents::increment(ProfileEvents::MergerMutatorSelectPartsForMergeElapsedMicroseconds, select_parts_from_ranges_timer.elapsedMicroseconds()); + out_disable_reason = PreformattedMessage::create("Did not find any parts to merge (with usual merge selectors) in {}ms", select_parts_from_ranges_timer.elapsedMicroseconds() / 1000); return SelectPartsDecision::CANNOT_SELECT; } } @@ -581,8 +608,11 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges( parts.push_back(part); } - LOG_DEBUG(log, "Selected {} parts from {} to {}", parts.size(), parts.front()->name, parts.back()->name); + LOG_DEBUG(log, "Selected {} parts from {} to {} in {}ms", parts.size(), parts.front()->name, parts.back()->name, select_parts_from_ranges_timer.elapsedMicroseconds() / 1000); + ProfileEvents::increment(ProfileEvents::MergerMutatorSelectRangePartsCount, parts.size()); + future_part->assign(std::move(parts)); + ProfileEvents::increment(ProfileEvents::MergerMutatorSelectPartsForMergeElapsedMicroseconds, select_parts_from_ranges_timer.elapsedMicroseconds()); return SelectPartsDecision::SELECTED; } diff --git a/src/Storages/MergeTree/MergeTreeDataPartBuilder.cpp b/src/Storages/MergeTree/MergeTreeDataPartBuilder.cpp index 37f578b0c25..6ec4bc31d90 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartBuilder.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartBuilder.cpp @@ -14,20 +14,22 @@ namespace ErrorCodes } MergeTreeDataPartBuilder::MergeTreeDataPartBuilder( - const MergeTreeData & data_, String name_, VolumePtr volume_, String root_path_, String part_dir_) + const MergeTreeData & data_, String name_, VolumePtr volume_, String root_path_, String part_dir_, const ReadSettings & read_settings_) : data(data_) , name(std::move(name_)) , volume(std::move(volume_)) , root_path(std::move(root_path_)) , part_dir(std::move(part_dir_)) + , read_settings(read_settings_) { } MergeTreeDataPartBuilder::MergeTreeDataPartBuilder( - const MergeTreeData & data_, String name_, MutableDataPartStoragePtr part_storage_) + const MergeTreeData & data_, String name_, MutableDataPartStoragePtr part_storage_, const ReadSettings & read_settings_) : data(data_) , name(std::move(name_)) , part_storage(std::move(part_storage_)) + , read_settings(read_settings_) { } @@ -73,7 +75,8 @@ MutableDataPartStoragePtr MergeTreeDataPartBuilder::getPartStorageByType( MergeTreeDataPartStorageType storage_type_, const VolumePtr & volume_, const String & root_path_, - const String & part_dir_) + const String & part_dir_, + const ReadSettings &) /// Unused here, but used in private repo. { if (!volume_) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create part storage, because volume is not specified"); @@ -112,7 +115,7 @@ MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartType(MergeTreeDataP MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartStorageType(MergeTreeDataPartStorageType storage_type_) { - part_storage = getPartStorageByType(storage_type_, volume, root_path, part_dir); + part_storage = getPartStorageByType(storage_type_, volume, root_path, part_dir, read_settings); return *this; } @@ -126,7 +129,8 @@ MergeTreeDataPartBuilder::PartStorageAndMarkType MergeTreeDataPartBuilder::getPartStorageAndMarkType( const VolumePtr & volume_, const String & root_path_, - const String & part_dir_) + const String & part_dir_, + const ReadSettings & read_settings_) { auto disk = volume_->getDisk(); auto part_relative_path = fs::path(root_path_) / part_dir_; @@ -138,7 +142,7 @@ MergeTreeDataPartBuilder::getPartStorageAndMarkType( if (MarkType::isMarkFileExtension(ext)) { - auto storage = getPartStorageByType(MergeTreeDataPartStorageType::Full, volume_, root_path_, part_dir_); + auto storage = getPartStorageByType(MergeTreeDataPartStorageType::Full, volume_, root_path_, part_dir_, read_settings_); return {std::move(storage), MarkType(ext)}; } } @@ -156,7 +160,7 @@ MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartFormatFromDisk() MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartFormatFromVolume() { assert(volume); - auto [storage, mark_type] = getPartStorageAndMarkType(volume, root_path, part_dir); + auto [storage, mark_type] = getPartStorageAndMarkType(volume, root_path, part_dir, read_settings); if (!storage || !mark_type) { diff --git a/src/Storages/MergeTree/MergeTreeDataPartBuilder.h b/src/Storages/MergeTree/MergeTreeDataPartBuilder.h index 0f54ff0a631..bce881a1970 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartBuilder.h +++ b/src/Storages/MergeTree/MergeTreeDataPartBuilder.h @@ -21,8 +21,8 @@ using VolumePtr = std::shared_ptr; class MergeTreeDataPartBuilder { public: - MergeTreeDataPartBuilder(const MergeTreeData & data_, String name_, VolumePtr volume_, String root_path_, String part_dir_); - MergeTreeDataPartBuilder(const MergeTreeData & data_, String name_, MutableDataPartStoragePtr part_storage_); + MergeTreeDataPartBuilder(const MergeTreeData & data_, String name_, VolumePtr volume_, String root_path_, String part_dir_, const ReadSettings & read_settings_); + MergeTreeDataPartBuilder(const MergeTreeData & data_, String name_, MutableDataPartStoragePtr part_storage_, const ReadSettings & read_settings_); std::shared_ptr build(); @@ -42,7 +42,8 @@ public: static PartStorageAndMarkType getPartStorageAndMarkType( const VolumePtr & volume_, const String & root_path_, - const String & part_dir_); + const String & part_dir_, + const ReadSettings & read_settings); private: Self & withPartFormatFromVolume(); @@ -52,7 +53,8 @@ private: MergeTreeDataPartStorageType storage_type_, const VolumePtr & volume_, const String & root_path_, - const String & part_dir_); + const String & part_dir_, + const ReadSettings & read_settings); const MergeTreeData & data; const String name; @@ -64,6 +66,8 @@ private: std::optional part_type; MutableDataPartStoragePtr part_storage; const IMergeTreeDataPart * parent_part = nullptr; + + const ReadSettings read_settings; }; } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 67a2c1ee9f1..c8d11ced683 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -63,23 +63,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( for (const auto & column : columns_list) { auto compression = getCodecDescOrDefault(column.name, default_codec); - addStreams(column, nullptr, compression); - } -} - -void MergeTreeDataPartWriterCompact::initDynamicStreamsIfNeeded(const Block & block) -{ - if (is_dynamic_streams_initialized) - return; - - is_dynamic_streams_initialized = true; - for (const auto & column : columns_list) - { - if (column.type->hasDynamicSubcolumns()) - { - auto compression = getCodecDescOrDefault(column.name, default_codec); - addStreams(column, block.getByName(column.name).column, compression); - } + MergeTreeDataPartWriterCompact::addStreams(column, nullptr, compression); } } @@ -181,20 +165,25 @@ void writeColumnSingleGranule( void MergeTreeDataPartWriterCompact::write(const Block & block, const IColumn::Permutation * permutation) { - /// On first block of data initialize streams for dynamic subcolumns. - initDynamicStreamsIfNeeded(block); + Block result_block = block; + + /// During serialization columns with dynamic subcolumns (like JSON/Dynamic) must have the same dynamic structure. + /// But it may happen that they don't (for example during ALTER MODIFY COLUMN from some type to JSON/Dynamic). + /// In this case we use dynamic structure of the column from the first written block and adjust columns from + /// the next blocks so they match this dynamic structure. + initOrAdjustDynamicStructureIfNeeded(result_block); /// Fill index granularity for this block /// if it's unknown (in case of insert data or horizontal merge, /// but not in case of vertical merge) if (compute_granularity) { - size_t index_granularity_for_block = computeIndexGranularity(block); + size_t index_granularity_for_block = computeIndexGranularity(result_block); assert(index_granularity_for_block >= 1); - fillIndexGranularity(index_granularity_for_block, block.rows()); + fillIndexGranularity(index_granularity_for_block, result_block.rows()); } - Block result_block = permuteBlockIfNeeded(block, permutation); + result_block = permuteBlockIfNeeded(result_block, permutation); if (!header) header = result_block.cloneEmpty(); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index b440a37222d..b3e2e78491d 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -32,6 +32,8 @@ public: void fillChecksums(MergeTreeDataPartChecksums & checksums, NameSet & checksums_to_remove) override; void finish(bool sync) override; + size_t getNumberOfOpenStreams() const override { return 1; } + private: /// Finish serialization of the data. Flush rows in buffer to disk, compute checksums. void fillDataChecksums(MergeTreeDataPartChecksums & checksums); @@ -48,9 +50,7 @@ private: void addToChecksums(MergeTreeDataPartChecksums & checksums); - void addStreams(const NameAndTypePair & name_and_type, const ColumnPtr & column, const ASTPtr & effective_codec_desc); - - void initDynamicStreamsIfNeeded(const Block & block); + void addStreams(const NameAndTypePair & name_and_type, const ColumnPtr & column, const ASTPtr & effective_codec_desc) override; Block header; @@ -104,8 +104,6 @@ private: /// then finally to 'marks_file'. std::unique_ptr marks_compressor; std::unique_ptr marks_source_hashing; - - bool is_dynamic_streams_initialized = false; }; } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index 388737915ab..c483d47fed7 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -564,6 +564,45 @@ Names MergeTreeDataPartWriterOnDisk::getSkipIndicesColumns() const return Names(skip_indexes_column_names_set.begin(), skip_indexes_column_names_set.end()); } +void MergeTreeDataPartWriterOnDisk::initOrAdjustDynamicStructureIfNeeded(Block & block) +{ + if (!is_dynamic_streams_initialized) + { + for (const auto & column : columns_list) + { + if (column.type->hasDynamicSubcolumns()) + { + /// Create all streams for dynamic subcolumns using dynamic structure from block. + auto compression = getCodecDescOrDefault(column.name, default_codec); + addStreams(column, block.getByName(column.name).column, compression); + } + } + is_dynamic_streams_initialized = true; + block_sample = block.cloneEmpty(); + } + else + { + size_t size = block.columns(); + for (size_t i = 0; i != size; ++i) + { + auto & column = block.getByPosition(i); + const auto & sample_column = block_sample.getByPosition(i); + /// Check if the dynamic structure of this column is different from the sample column. + if (column.type->hasDynamicSubcolumns() && !column.column->dynamicStructureEquals(*sample_column.column)) + { + /// We need to change the dynamic structure of the column so it matches the sample column. + /// To do it, we create empty column of this type, take dynamic structure from sample column + /// and insert data into it. Resulting column will have required dynamic structure and the content + /// of the column in current block. + auto new_column = sample_column.type->createColumn(); + new_column->takeDynamicStructureFromSourceColumns({sample_column.column}); + new_column->insertRangeFrom(*column.column, 0, column.column->size()); + column.column = std::move(new_column); + } + } + } +} + template struct MergeTreeDataPartWriterOnDisk::Stream; template struct MergeTreeDataPartWriterOnDisk::Stream; diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h index 4a760c20b58..49d654c15e1 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h @@ -154,6 +154,14 @@ protected: /// Get unique non ordered skip indices column. Names getSkipIndicesColumns() const; + virtual void addStreams(const NameAndTypePair & name_and_type, const ColumnPtr & column, const ASTPtr & effective_codec_desc) = 0; + + /// On first block create all required streams for columns with dynamic subcolumns and remember the block sample. + /// On each next block check if dynamic structure of the columns equals to the dynamic structure of the same + /// columns in the sample block. If for some column dynamic structure is different, adjust it so it matches + /// the structure from the sample. + void initOrAdjustDynamicStructureIfNeeded(Block & block); + const MergeTreeIndices skip_indices; const ColumnsStatistics stats; @@ -188,6 +196,10 @@ protected: size_t current_mark = 0; GinIndexStoreFactory::GinIndexStores gin_index_stores; + + bool is_dynamic_streams_initialized = false; + Block block_sample; + private: void initSkipIndices(); void initPrimaryIndex(); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 433c7c21613..7c9724b1b75 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -116,24 +116,7 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( for (const auto & column : columns_list) { auto compression = getCodecDescOrDefault(column.name, default_codec); - addStreams(column, nullptr, compression); - } -} - -void MergeTreeDataPartWriterWide::initDynamicStreamsIfNeeded(const DB::Block & block) -{ - if (is_dynamic_streams_initialized) - return; - - is_dynamic_streams_initialized = true; - block_sample = block.cloneEmpty(); - for (const auto & column : columns_list) - { - if (column.type->hasDynamicSubcolumns()) - { - auto compression = getCodecDescOrDefault(column.name, default_codec); - addStreams(column, block_sample.getByName(column.name).column, compression); - } + MergeTreeDataPartWriterWide::addStreams(column, nullptr, compression); } } @@ -277,15 +260,20 @@ void MergeTreeDataPartWriterWide::shiftCurrentMark(const Granules & granules_wri void MergeTreeDataPartWriterWide::write(const Block & block, const IColumn::Permutation * permutation) { - /// On first block of data initialize streams for dynamic subcolumns. - initDynamicStreamsIfNeeded(block); + Block block_to_write = block; + + /// During serialization columns with dynamic subcolumns (like JSON/Dynamic) must have the same dynamic structure. + /// But it may happen that they don't (for example during ALTER MODIFY COLUMN from some type to JSON/Dynamic). + /// In this case we use dynamic structure of the column from the first written block and adjust columns from + /// the next blocks so they match this dynamic structure. + initOrAdjustDynamicStructureIfNeeded(block_to_write); /// Fill index granularity for this block /// if it's unknown (in case of insert data or horizontal merge, /// but not in case of vertical part of vertical merge) if (compute_granularity) { - size_t index_granularity_for_block = computeIndexGranularity(block); + size_t index_granularity_for_block = computeIndexGranularity(block_to_write); if (rows_written_in_last_mark > 0) { size_t rows_left_in_last_mark = index_granularity.getMarkRows(getCurrentMark()) - rows_written_in_last_mark; @@ -303,11 +291,9 @@ void MergeTreeDataPartWriterWide::write(const Block & block, const IColumn::Perm } } - fillIndexGranularity(index_granularity_for_block, block.rows()); + fillIndexGranularity(index_granularity_for_block, block_to_write.rows()); } - Block block_to_write = block; - auto granules_to_write = getGranulesToWrite(index_granularity, block_to_write.rows(), getCurrentMark(), rows_written_in_last_mark); auto offset_columns = written_offset_columns ? *written_offset_columns : WrittenOffsetColumns{}; diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 68f016a7421..19304b28c6c 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -43,6 +43,8 @@ public: void finish(bool sync) final; + size_t getNumberOfOpenStreams() const override { return column_streams.size(); } + private: /// Finish serialization of data: write final mark if required and compute checksums /// Also validate written data in debug mode @@ -91,9 +93,7 @@ private: void addStreams( const NameAndTypePair & name_and_type, const ColumnPtr & column, - const ASTPtr & effective_codec_desc); - - void initDynamicStreamsIfNeeded(const Block & block); + const ASTPtr & effective_codec_desc) override; /// Method for self check (used in debug-build only). Checks that written /// data and corresponding marks are consistent. Otherwise throws logical @@ -142,10 +142,6 @@ private: /// How many rows we have already written in the current mark. /// More than zero when incoming blocks are smaller then their granularity. size_t rows_written_in_last_mark = 0; - - Block block_sample; - - bool is_dynamic_streams_initialized = false; }; } diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index ac29a9244b0..6d19f45e2c4 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -610,7 +610,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl( } } - auto new_data_part = data.getDataPartBuilder(part_name, data_part_volume, part_dir) + auto new_data_part = data.getDataPartBuilder(part_name, data_part_volume, part_dir, getReadSettings()) .withPartFormat(data.choosePartFormat(expected_size, block.rows())) .withPartInfo(new_part_info) .build(); diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.h b/src/Storages/MergeTree/MergeTreeIOSettings.h index 4d1d2533729..7506c726bc4 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -45,6 +45,8 @@ struct MergeTreeReaderSettings bool use_asynchronous_read_from_pool = false; /// If PREWHERE has multiple conditions combined with AND, execute them in separate read/filtering steps. bool enable_multiple_prewhere_read_steps = false; + /// In case of multiple prewhere steps, execute filtering earlier to support short-circuit properly. + bool force_short_circuit_execution = false; /// If true, try to lower size of read buffer according to granule size and compressed block size. bool adjust_read_buffer_size = true; /// If true, it's allowed to read the whole part without reading marks. diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp b/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp index d69a00643f0..bf0ba17d473 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp @@ -122,4 +122,20 @@ std::string MergeTreeIndexGranularity::describe() const { return fmt::format("initialized: {}, marks_rows_partial_sums: [{}]", initialized, fmt::join(marks_rows_partial_sums, ", ")); } + +void MergeTreeIndexGranularity::shrinkToFitInMemory() +{ + marks_rows_partial_sums.shrink_to_fit(); +} + +uint64_t MergeTreeIndexGranularity::getBytesSize() const +{ + return marks_rows_partial_sums.size() * sizeof(size_t); +} +uint64_t MergeTreeIndexGranularity::getBytesAllocated() const +{ + return marks_rows_partial_sums.capacity() * sizeof(size_t); +} + + } diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularity.h b/src/Storages/MergeTree/MergeTreeIndexGranularity.h index f66e721ec1e..c616d2ac49a 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularity.h +++ b/src/Storages/MergeTree/MergeTreeIndexGranularity.h @@ -100,6 +100,11 @@ public: void resizeWithFixedGranularity(size_t size, size_t fixed_granularity); std::string describe() const; + + void shrinkToFitInMemory(); + + uint64_t getBytesSize() const; + uint64_t getBytesAllocated() const; }; } diff --git a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp index 5a725922e14..0b17fa05072 100644 --- a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp @@ -178,23 +178,20 @@ String USearchIndexWithSerialization::Statistics::toString() const } MergeTreeIndexGranuleVectorSimilarity::MergeTreeIndexGranuleVectorSimilarity( const String & index_name_, - const Block & index_sample_block_, unum::usearch::metric_kind_t metric_kind_, unum::usearch::scalar_kind_t scalar_kind_, UsearchHnswParams usearch_hnsw_params_) - : MergeTreeIndexGranuleVectorSimilarity(index_name_, index_sample_block_, metric_kind_, scalar_kind_, usearch_hnsw_params_, nullptr) + : MergeTreeIndexGranuleVectorSimilarity(index_name_, metric_kind_, scalar_kind_, usearch_hnsw_params_, nullptr) { } MergeTreeIndexGranuleVectorSimilarity::MergeTreeIndexGranuleVectorSimilarity( const String & index_name_, - const Block & index_sample_block_, unum::usearch::metric_kind_t metric_kind_, unum::usearch::scalar_kind_t scalar_kind_, UsearchHnswParams usearch_hnsw_params_, USearchIndexWithSerializationPtr index_) : index_name(index_name_) - , index_sample_block(index_sample_block_) , metric_kind(metric_kind_) , scalar_kind(scalar_kind_) , usearch_hnsw_params(usearch_hnsw_params_) @@ -261,7 +258,7 @@ MergeTreeIndexAggregatorVectorSimilarity::MergeTreeIndexAggregatorVectorSimilari MergeTreeIndexGranulePtr MergeTreeIndexAggregatorVectorSimilarity::getGranuleAndReset() { - auto granule = std::make_shared(index_name, index_sample_block, metric_kind, scalar_kind, usearch_hnsw_params, index); + auto granule = std::make_shared(index_name, metric_kind, scalar_kind, usearch_hnsw_params, index); index = nullptr; return granule; } @@ -345,10 +342,11 @@ void MergeTreeIndexAggregatorVectorSimilarity::update(const Block & block, size_ throw Exception(ErrorCodes::INCORRECT_DATA, "Index granularity is too big: more than {} rows per index granule.", std::numeric_limits::max()); if (index_sample_block.columns() > 1) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected block with single column"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected that index is build over a single column"); - const String & index_column_name = index_sample_block.getByPosition(0).name; - const ColumnPtr & index_column = block.getByName(index_column_name).column; + const auto & index_column_name = index_sample_block.getByPosition(0).name; + + const auto & index_column = block.getByName(index_column_name).column; ColumnPtr column_cut = index_column->cut(*pos, rows_read); const auto * column_array = typeid_cast(column_cut.get()); @@ -382,8 +380,7 @@ void MergeTreeIndexAggregatorVectorSimilarity::update(const Block & block, size_ if (index->size() + rows > std::numeric_limits::max()) throw Exception(ErrorCodes::INCORRECT_DATA, "Size of vector similarity index would exceed 4 billion entries"); - DataTypePtr data_type = block.getDataTypes()[0]; - const auto * data_type_array = typeid_cast(data_type.get()); + const auto * data_type_array = typeid_cast(block.getByName(index_column_name).type.get()); if (!data_type_array) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected data type Array(Float*)"); const TypeIndex nested_type_index = data_type_array->getNestedType()->getTypeId(); @@ -490,7 +487,7 @@ MergeTreeIndexVectorSimilarity::MergeTreeIndexVectorSimilarity( MergeTreeIndexGranulePtr MergeTreeIndexVectorSimilarity::createIndexGranule() const { - return std::make_shared(index.name, index.sample_block, metric_kind, scalar_kind, usearch_hnsw_params); + return std::make_shared(index.name, metric_kind, scalar_kind, usearch_hnsw_params); } MergeTreeIndexAggregatorPtr MergeTreeIndexVectorSimilarity::createIndexAggregator(const MergeTreeWriterSettings & /*settings*/) const @@ -531,15 +528,17 @@ void vectorSimilarityIndexValidator(const IndexDescription & index, bool /* atta { const bool has_two_args = (index.arguments.size() == 2); const bool has_five_args = (index.arguments.size() == 5); + const bool has_six_args = (index.arguments.size() == 6); /// Legacy index creation syntax before #70616. Supported only to be able to load old tables, can be removed mid-2025. + /// The 6th argument (ef_search) is ignored. /// Check number and type of arguments - if (!has_two_args && !has_five_args) + if (!has_two_args && !has_five_args && !has_six_args) throw Exception(ErrorCodes::INCORRECT_QUERY, "Vector similarity index must have two or five arguments"); if (index.arguments[0].getType() != Field::Types::String) throw Exception(ErrorCodes::INCORRECT_QUERY, "First argument of vector similarity index (method) must be of type String"); if (index.arguments[1].getType() != Field::Types::String) throw Exception(ErrorCodes::INCORRECT_QUERY, "Second argument of vector similarity index (metric) must be of type String"); - if (has_five_args) + if (has_five_args || has_six_args) { if (index.arguments[2].getType() != Field::Types::String) throw Exception(ErrorCodes::INCORRECT_QUERY, "Third argument of vector similarity index (quantization) must be of type String"); diff --git a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.h b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.h index 9a81e168393..fe5049daf77 100644 --- a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.h +++ b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.h @@ -69,14 +69,12 @@ struct MergeTreeIndexGranuleVectorSimilarity final : public IMergeTreeIndexGranu { MergeTreeIndexGranuleVectorSimilarity( const String & index_name_, - const Block & index_sample_block_, unum::usearch::metric_kind_t metric_kind_, unum::usearch::scalar_kind_t scalar_kind_, UsearchHnswParams usearch_hnsw_params_); MergeTreeIndexGranuleVectorSimilarity( const String & index_name_, - const Block & index_sample_block_, unum::usearch::metric_kind_t metric_kind_, unum::usearch::scalar_kind_t scalar_kind_, UsearchHnswParams usearch_hnsw_params_, @@ -90,7 +88,6 @@ struct MergeTreeIndexGranuleVectorSimilarity final : public IMergeTreeIndexGranu bool empty() const override { return !index || index->size() == 0; } const String index_name; - const Block index_sample_block; const unum::usearch::metric_kind_t metric_kind; const unum::usearch::scalar_kind_t scalar_kind; const UsearchHnswParams usearch_hnsw_params; diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 48a4a37f444..e9c9f2b4b06 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -280,7 +280,7 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me cloned_part_storage = part->makeCloneOnDisk(disk, MergeTreeData::MOVING_DIR_NAME, read_settings, write_settings, cancellation_hook); } - MergeTreeDataPartBuilder builder(*data, part->name, cloned_part_storage); + MergeTreeDataPartBuilder builder(*data, part->name, cloned_part_storage, getReadSettings()); cloned_part.part = std::move(builder).withPartFormatFromDisk().build(); LOG_TRACE(log, "Part {} was cloned to {}", part->name, cloned_part.part->getDataPartStorage().getFullPath()); diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 5efd33ce09a..426e95fd95d 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include @@ -91,7 +91,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( , algorithm(std::move(algorithm_)) , prewhere_info(prewhere_info_) , actions_settings(actions_settings_) - , prewhere_actions(getPrewhereActions(prewhere_info, actions_settings, reader_settings_.enable_multiple_prewhere_read_steps)) + , prewhere_actions(getPrewhereActions(prewhere_info, actions_settings, reader_settings_.enable_multiple_prewhere_read_steps, reader_settings_.force_short_circuit_execution)) , reader_settings(reader_settings_) , result_header(transformHeader(pool->getHeader(), prewhere_info)) { @@ -124,9 +124,9 @@ String MergeTreeSelectProcessor::getName() const return fmt::format("MergeTreeSelect(pool: {}, algorithm: {})", pool->getName(), algorithm->getName()); } -bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere); +bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere, bool force_short_circuit_execution); -PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps) +PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps, bool force_short_circuit_execution) { PrewhereExprInfo prewhere_actions; if (prewhere_info) @@ -147,7 +147,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr pr } if (!enable_multiple_prewhere_read_steps || - !tryBuildPrewhereSteps(prewhere_info, actions_settings, prewhere_actions)) + !tryBuildPrewhereSteps(prewhere_info, actions_settings, prewhere_actions, force_short_circuit_execution)) { PrewhereExprStep prewhere_step { @@ -203,7 +203,7 @@ ChunkAndProgress MergeTreeSelectProcessor::read() auto chunk = Chunk(ordered_columns, res.row_count); if (add_part_level) - chunk.getChunkInfos().add(std::make_shared(task->getInfo().data_part->info.level)); + chunk.getChunkInfos().add(std::make_shared(task->getInfo().data_part->info.level)); return ChunkAndProgress{ .chunk = std::move(chunk), diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 33069a78e33..32a761cefb7 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -73,7 +73,8 @@ public: static PrewhereExprInfo getPrewhereActions( PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, - bool enable_multiple_prewhere_read_steps); + bool enable_multiple_prewhere_read_steps, + bool force_short_circuit_execution); void addPartLevelToChunk(bool add_part_level_) { add_part_level = add_part_level_; } diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 653973e9db7..83714d4e197 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -14,9 +14,10 @@ #include #include #include -#include +#include #include + namespace DB { @@ -271,7 +272,7 @@ try auto result = Chunk(std::move(res_columns), rows_read); if (add_part_level) - result.getChunkInfos().add(std::make_shared(data_part->info.level)); + result.getChunkInfos().add(std::make_shared(data_part->info.level)); return result; } } diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index 883191d59ab..fcd4e05cf00 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -101,6 +101,8 @@ namespace ErrorCodes DECLARE(Milliseconds, background_task_preferred_step_execution_time_ms, 50, "Target time to execution of one step of merge or mutation. Can be exceeded if one step takes longer time", 0) \ DECLARE(MergeSelectorAlgorithm, merge_selector_algorithm, MergeSelectorAlgorithm::SIMPLE, "The algorithm to select parts for merges assignment", EXPERIMENTAL) \ DECLARE(Bool, merge_selector_enable_heuristic_to_remove_small_parts_at_right, true, "Enable heuristic for selecting parts for merge which removes parts from right side of range, if their size is less than specified ratio (0.01) of sum_size. Works for Simple and StochasticSimple merge selectors", 0) \ + DECLARE(Float, merge_selector_base, 5.0, "Affects write amplification of assigned merges (expert level setting, don't change if you don't understand what it is doing). Works for Simple and StochasticSimple merge selectors", 0) \ + DECLARE(UInt64, min_parts_to_merge_at_once, 0, "Minimal amount of data parts which merge selector can pick to merge at once (expert level setting, don't change if you don't understand what it is doing). 0 - disabled. Works for Simple and StochasticSimple merge selectors.", 0) \ \ /** Inserts settings. */ \ DECLARE(UInt64, parts_to_delay_insert, 1000, "If table contains at least that many active parts in single partition, artificially slow down insert into table. Disabled if set to 0", 0) \ diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 604112c26ea..99852309c77 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -94,7 +94,7 @@ void MergeTreeSink::consume(Chunk & chunk) DelayedPartitions partitions; const Settings & settings = context->getSettingsRef(); - size_t streams = 0; + size_t total_streams = 0; bool support_parallel_write = false; auto token_info = chunk.getChunkInfos().get(); @@ -153,16 +153,18 @@ void MergeTreeSink::consume(Chunk & chunk) max_insert_delayed_streams_for_parallel_write = 0; /// In case of too much columns/parts in block, flush explicitly. - streams += temp_part.streams.size(); + size_t current_streams = 0; + for (const auto & stream : temp_part.streams) + current_streams += stream.stream->getNumberOfOpenStreams(); - if (streams > max_insert_delayed_streams_for_parallel_write) + if (total_streams + current_streams > max_insert_delayed_streams_for_parallel_write) { finishDelayedChunk(); delayed_chunk = std::make_unique(); delayed_chunk->partitions = std::move(partitions); finishDelayedChunk(); - streams = 0; + total_streams = 0; support_parallel_write = false; partitions = DelayedPartitions{}; } @@ -174,6 +176,8 @@ void MergeTreeSink::consume(Chunk & chunk) .block_dedup_token = block_dedup_token, .part_counters = std::move(part_counters), }); + + total_streams += current_streams; } if (need_to_define_dedup_token) diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp index 9c82817e8cb..1cc4006a285 100644 --- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp +++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -57,9 +58,9 @@ struct DAGNodeRef const ActionsDAG::Node * node; }; -/// Result name -> DAGNodeRef -using OriginalToNewNodeMap = std::unordered_map; -using NodeNameToLastUsedStepMap = std::unordered_map; +/// ResultNode -> DAGNodeRef +using OriginalToNewNodeMap = std::unordered_map; +using NodeNameToLastUsedStepMap = std::unordered_map; /// Clones the part of original DAG responsible for computing the original_dag_node and adds it to the new DAG. const ActionsDAG::Node & addClonedDAGToDAG( @@ -69,25 +70,28 @@ const ActionsDAG::Node & addClonedDAGToDAG( OriginalToNewNodeMap & node_remap, NodeNameToLastUsedStepMap & node_to_step_map) { - const String & node_name = original_dag_node->result_name; /// Look for the node in the map of already known nodes - if (node_remap.contains(node_name)) + if (node_remap.contains(original_dag_node)) { /// If the node is already in the new DAG, return it - const auto & node_ref = node_remap.at(node_name); + const auto & node_ref = node_remap.at(original_dag_node); if (node_ref.dag == new_dag.get()) return *node_ref.node; /// If the node is known from the previous steps, add it as an input, except for constants if (original_dag_node->type != ActionsDAG::ActionType::COLUMN) { - node_ref.dag->addOrReplaceInOutputs(*node_ref.node); + /// If the node was found in node_remap, it was not added to outputs yet. + /// The only exception is the filter node, which is always the first one. + if (node_ref.dag->getOutputs().at(0) != node_ref.node) + node_ref.dag->getOutputs().push_back(node_ref.node); + const auto & new_node = new_dag->addInput(node_ref.node->result_name, node_ref.node->result_type); - node_remap[node_name] = {new_dag.get(), &new_node}; /// TODO: here we update the node reference. Is it always correct? + node_remap[original_dag_node] = {new_dag.get(), &new_node}; /// Remember the index of the last step which reuses this node. /// We cannot remove this node from the outputs before that step. - node_to_step_map[node_name] = step; + node_to_step_map[original_dag_node] = step; return new_node; } } @@ -96,7 +100,7 @@ const ActionsDAG::Node & addClonedDAGToDAG( if (original_dag_node->type == ActionsDAG::ActionType::INPUT) { const auto & new_node = new_dag->addInput(original_dag_node->result_name, original_dag_node->result_type); - node_remap[node_name] = {new_dag.get(), &new_node}; + node_remap[original_dag_node] = {new_dag.get(), &new_node}; return new_node; } @@ -105,7 +109,7 @@ const ActionsDAG::Node & addClonedDAGToDAG( { const auto & new_node = new_dag->addColumn( ColumnWithTypeAndName(original_dag_node->column, original_dag_node->result_type, original_dag_node->result_name)); - node_remap[node_name] = {new_dag.get(), &new_node}; + node_remap[original_dag_node] = {new_dag.get(), &new_node}; return new_node; } @@ -113,7 +117,7 @@ const ActionsDAG::Node & addClonedDAGToDAG( { const auto & alias_child = addClonedDAGToDAG(step, original_dag_node->children[0], new_dag, node_remap, node_to_step_map); const auto & new_node = new_dag->addAlias(alias_child, original_dag_node->result_name); - node_remap[node_name] = {new_dag.get(), &new_node}; + node_remap[original_dag_node] = {new_dag.get(), &new_node}; return new_node; } @@ -128,7 +132,7 @@ const ActionsDAG::Node & addClonedDAGToDAG( } const auto & new_node = new_dag->addFunction(original_dag_node->function_base, new_children, original_dag_node->result_name); - node_remap[node_name] = {new_dag.get(), &new_node}; + node_remap[original_dag_node] = {new_dag.get(), &new_node}; return new_node; } @@ -138,11 +142,9 @@ const ActionsDAG::Node & addClonedDAGToDAG( const ActionsDAG::Node & addFunction( const ActionsDAGPtr & new_dag, const FunctionOverloadResolverPtr & function, - ActionsDAG::NodeRawConstPtrs children, - OriginalToNewNodeMap & node_remap) + ActionsDAG::NodeRawConstPtrs children) { const auto & new_node = new_dag->addFunction(function, children, ""); - node_remap[new_node.result_name] = {new_dag.get(), &new_node}; return new_node; } @@ -152,14 +154,12 @@ const ActionsDAG::Node & addFunction( const ActionsDAG::Node & addCast( const ActionsDAGPtr & dag, const ActionsDAG::Node & node_to_cast, - const DataTypePtr & to_type, - OriginalToNewNodeMap & node_remap) + const DataTypePtr & to_type) { if (!node_to_cast.result_type->equals(*to_type)) return node_to_cast; const auto & new_node = dag->addCast(node_to_cast, to_type, {}); - node_remap[new_node.result_name] = {dag.get(), &new_node}; return new_node; } @@ -169,8 +169,7 @@ const ActionsDAG::Node & addCast( /// 2. makes sure that the result contains only 0 or 1 values even if the source column contains non-boolean values. const ActionsDAG::Node & addAndTrue( const ActionsDAGPtr & dag, - const ActionsDAG::Node & filter_node_to_normalize, - OriginalToNewNodeMap & node_remap) + const ActionsDAG::Node & filter_node_to_normalize) { Field const_true_value(true); @@ -181,7 +180,7 @@ const ActionsDAG::Node & addAndTrue( const auto * const_true_node = &dag->addColumn(std::move(const_true_column)); ActionsDAG::NodeRawConstPtrs children = {&filter_node_to_normalize, const_true_node}; FunctionOverloadResolverPtr func_builder_and = std::make_unique(std::make_shared()); - return addFunction(dag, func_builder_and, children, node_remap); + return addFunction(dag, func_builder_and, children); } } @@ -206,7 +205,11 @@ const ActionsDAG::Node & addAndTrue( /// 6. Find all outputs of the original DAG /// 7. Find all outputs that were computed in the already built DAGs, mark these nodes as outputs in the steps where they were computed /// 8. Add computation of the remaining outputs to the last step with the procedure similar to 4 -bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere) +bool tryBuildPrewhereSteps( + PrewhereInfoPtr prewhere_info, + const ExpressionActionsSettings & actions_settings, + PrewhereExprInfo & prewhere, + bool force_short_circuit_execution) { if (!prewhere_info) return true; @@ -243,7 +246,10 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction struct Step { ActionsDAGPtr actions; - String column_name; + /// Original condition, in case if we have only one condition, and it was not casted + const ActionsDAG::Node * original_node; + /// Result condition node + const ActionsDAG::Node * result_node; }; std::vector steps; @@ -254,7 +260,8 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction { const auto & condition_group = condition_groups[step_index]; ActionsDAGPtr step_dag = std::make_unique(); - String result_name; + const ActionsDAG::Node * original_node = nullptr; + const ActionsDAG::Node * result_node; std::vector new_condition_nodes; for (const auto * node : condition_group) @@ -267,48 +274,37 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction { /// Add AND function to combine the conditions FunctionOverloadResolverPtr func_builder_and = std::make_unique(std::make_shared()); - const auto & and_function_node = addFunction(step_dag, func_builder_and, new_condition_nodes, node_remap); - step_dag->addOrReplaceInOutputs(and_function_node); - result_name = and_function_node.result_name; + const auto & and_function_node = addFunction(step_dag, func_builder_and, new_condition_nodes); + result_node = &and_function_node; } else { - const auto & result_node = *new_condition_nodes.front(); + result_node = new_condition_nodes.front(); /// Check if explicit cast is needed for the condition to serve as a filter. - const auto result_type_name = result_node.result_type->getName(); - if (result_type_name == "UInt8" || - result_type_name == "Nullable(UInt8)" || - result_type_name == "LowCardinality(UInt8)" || - result_type_name == "LowCardinality(Nullable(UInt8))") - { - /// No need to cast - step_dag->addOrReplaceInOutputs(result_node); - result_name = result_node.result_name; - } - else + if (!isUInt8(removeNullable(removeLowCardinality(result_node->result_type)))) { /// Build "condition AND True" expression to "cast" the condition to UInt8 or Nullable(UInt8) depending on its type. - const auto & cast_node = addAndTrue(step_dag, result_node, node_remap); - step_dag->addOrReplaceInOutputs(cast_node); - result_name = cast_node.result_name; + result_node = &addAndTrue(step_dag, *result_node); } } - steps.push_back({std::move(step_dag), result_name}); + step_dag->getOutputs().insert(step_dag->getOutputs().begin(), result_node); + steps.push_back({std::move(step_dag), original_node, result_node}); } /// 6. Find all outputs of the original DAG auto original_outputs = prewhere_info->prewhere_actions.getOutputs(); + steps.back().actions->getOutputs().clear(); /// 7. Find all outputs that were computed in the already built DAGs, mark these nodes as outputs in the steps where they were computed /// 8. Add computation of the remaining outputs to the last step with the procedure similar to 4 - NameSet all_output_names; + std::unordered_set all_outputs; for (const auto * output : original_outputs) { - all_output_names.insert(output->result_name); - if (node_remap.contains(output->result_name)) + all_outputs.insert(output); + if (node_remap.contains(output)) { - const auto & new_node_info = node_remap[output->result_name]; - new_node_info.dag->addOrReplaceInOutputs(*new_node_info.node); + const auto & new_node_info = node_remap[output]; + new_node_info.dag->getOutputs().push_back(new_node_info.node); } else if (output->result_name == prewhere_info->prewhere_column_name) { @@ -319,20 +315,21 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction /// 1. AND the last condition with constant True. This is needed to make sure that in the last step filter has UInt8 type /// but contains values other than 0 and 1 (e.g. if it is (number%5) it contains 2,3,4) /// 2. CAST the result to the exact type of the PREWHERE column from the original DAG - const auto & last_step_result_node_info = node_remap[steps.back().column_name]; auto & last_step_dag = steps.back().actions; + auto & last_step_result_node = steps.back().result_node; /// Build AND(last_step_result_node, true) - const auto & and_node = addAndTrue(last_step_dag, *last_step_result_node_info.node, node_remap); + const auto & and_node = addAndTrue(last_step_dag, *last_step_result_node); /// Build CAST(and_node, type of PREWHERE column) - const auto & cast_node = addCast(last_step_dag, and_node, output->result_type, node_remap); + const auto & cast_node = addCast(last_step_dag, and_node, output->result_type); /// Add alias for the result with the name of the PREWHERE column const auto & prewhere_result_node = last_step_dag->addAlias(cast_node, output->result_name); - last_step_dag->addOrReplaceInOutputs(prewhere_result_node); + last_step_dag->getOutputs().push_back(&prewhere_result_node); + steps.back().result_node = &prewhere_result_node; } else { const auto & node_in_new_dag = addClonedDAGToDAG(steps.size() - 1, output, steps.back().actions, node_remap, node_to_step); - steps.back().actions->addOrReplaceInOutputs(node_in_new_dag); + steps.back().actions->getOutputs().push_back(&node_in_new_dag); } } @@ -345,17 +342,18 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction { .type = PrewhereExprStep::Filter, .actions = std::make_shared(std::move(*step.actions), actions_settings), - .filter_column_name = step.column_name, + .filter_column_name = step.result_node->result_name, /// Don't remove if it's in the list of original outputs .remove_filter_column = - !all_output_names.contains(step.column_name) && node_to_step[step.column_name] <= step_index, - .need_filter = false, + step.original_node && !all_outputs.contains(step.original_node) && node_to_step[step.original_node] <= step_index, + .need_filter = force_short_circuit_execution, .perform_alter_conversions = true, }; prewhere.steps.push_back(std::make_shared(std::move(new_step))); } + prewhere.steps.back()->remove_filter_column = prewhere_info->remove_prewhere_column; prewhere.steps.back()->need_filter = prewhere_info->need_filter; } diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 77c34aae30a..39096718b5c 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -207,6 +207,8 @@ MergedBlockOutputStream::Finalizer MergedBlockOutputStream::finalizePartAsync( new_part->setBytesOnDisk(checksums.getTotalSizeOnDisk()); new_part->setBytesUncompressedOnDisk(checksums.getTotalSizeUncompressedOnDisk()); new_part->index_granularity = writer->getIndexGranularity(); + /// Just in case + new_part->index_granularity.shrinkToFitInMemory(); new_part->calculateColumnsAndSecondaryIndicesSizesOnDisk(); /// In mutation, existing_rows_count is already calculated in PartMergerWriter diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 753b0c5d2fe..7f6588fc632 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -984,6 +984,8 @@ void finalizeMutatedPart( new_data_part->rows_count = source_part->rows_count; new_data_part->index_granularity = source_part->index_granularity; + /// Just in case + new_data_part->index_granularity.shrinkToFitInMemory(); new_data_part->setIndex(*source_part->getIndex()); new_data_part->minmax_idx = source_part->minmax_idx; new_data_part->modification_time = time(nullptr); @@ -2289,7 +2291,7 @@ bool MutateTask::prepare() String tmp_part_dir_name = prefix + ctx->future_part->name; ctx->temporary_directory_lock = ctx->data->getTemporaryPartDirectoryHolder(tmp_part_dir_name); - auto builder = ctx->data->getDataPartBuilder(ctx->future_part->name, single_disk_volume, tmp_part_dir_name); + auto builder = ctx->data->getDataPartBuilder(ctx->future_part->name, single_disk_volume, tmp_part_dir_name, getReadSettings()); builder.withPartFormat(ctx->future_part->part_format); builder.withPartInfo(ctx->future_part->part_info); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 93124e634bd..addaeb65350 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -29,8 +29,8 @@ namespace MergeTreeSetting namespace ErrorCodes { extern const int REPLICA_IS_ALREADY_ACTIVE; - extern const int REPLICA_STATUS_CHANGED; extern const int LOGICAL_ERROR; + extern const int SUPPORT_IS_DISABLED; } namespace FailPoints @@ -217,26 +217,10 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() } else { - /// Table was created before 20.4 and was never altered, - /// let's initialize replica metadata version from global metadata version. - - const String & zookeeper_path = storage.zookeeper_path, & replica_path = storage.replica_path; - - Coordination::Stat table_metadata_version_stat; - zookeeper->get(zookeeper_path + "/metadata", &table_metadata_version_stat); - - Coordination::Requests ops; - ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/metadata", table_metadata_version_stat.version)); - ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/metadata_version", toString(table_metadata_version_stat.version), zkutil::CreateMode::Persistent)); - - Coordination::Responses res; - auto code = zookeeper->tryMulti(ops, res); - - if (code == Coordination::Error::ZBADVERSION) - throw Exception(ErrorCodes::REPLICA_STATUS_CHANGED, "Failed to initialize metadata_version " - "because table was concurrently altered, will retry"); - - zkutil::KeeperMultiException::check(code, ops, res); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "It seems you have upgraded from a version earlier than 20.4 straight to one later than 24.10. " + "ClickHouse does not support upgrades that span more than a year. " + "Please update gradually (through intermediate versions)."); } storage.queue.removeCurrentPartsFromMutations(); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index f1b0e5ec385..c0e25a54bf3 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -341,7 +341,7 @@ void ReplicatedMergeTreeSinkImpl::consume(Chunk & chunk) using DelayedPartitions = std::vector; DelayedPartitions partitions; - size_t streams = 0; + size_t total_streams = 0; bool support_parallel_write = false; for (auto & current_block : part_blocks) @@ -418,15 +418,18 @@ void ReplicatedMergeTreeSinkImpl::consume(Chunk & chunk) max_insert_delayed_streams_for_parallel_write = 0; /// In case of too much columns/parts in block, flush explicitly. - streams += temp_part.streams.size(); - if (streams > max_insert_delayed_streams_for_parallel_write) + size_t current_streams = 0; + for (const auto & stream : temp_part.streams) + current_streams += stream.stream->getNumberOfOpenStreams(); + + if (total_streams + current_streams > max_insert_delayed_streams_for_parallel_write) { finishDelayedChunk(zookeeper); delayed_chunk = std::make_unique::DelayedChunk>(replicas_num); delayed_chunk->partitions = std::move(partitions); finishDelayedChunk(zookeeper); - streams = 0; + total_streams = 0; support_parallel_write = false; partitions = DelayedPartitions{}; } @@ -447,6 +450,8 @@ void ReplicatedMergeTreeSinkImpl::consume(Chunk & chunk) std::move(unmerged_block), std::move(part_counters) /// profile_events_scope must be reset here. )); + + total_streams += current_streams; } if (need_to_define_dedup_token) @@ -614,8 +619,17 @@ bool ReplicatedMergeTreeSinkImpl::writeExistingPart(MergeTreeData::Mutabl String block_id = deduplicate ? fmt::format("{}_{}", part->info.partition_id, part->checksums.getTotalChecksumHex()) : ""; bool deduplicated = commitPart(zookeeper, part, block_id, replicas_num).second; + int error = 0; /// Set a special error code if the block is duplicate - int error = (deduplicate && deduplicated) ? ErrorCodes::INSERT_WAS_DEDUPLICATED : 0; + /// And remove attaching_ prefix + if (deduplicate && deduplicated) + { + error = ErrorCodes::INSERT_WAS_DEDUPLICATED; + if (!endsWith(part->getDataPartStorage().getRelativePath(), "detached/attaching_" + part->name + "/")) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected relative path for a deduplicated part: {}", part->getDataPartStorage().getRelativePath()); + fs::path new_relative_path = fs::path("detached") / part->getNewName(part->info); + part->renameTo(new_relative_path, false); + } PartLog::addNewPart(storage.getContext(), PartLog::PartLogEntry(part, watch.elapsed(), profile_events_scope.getSnapshot()), ExecutionStatus(error)); return deduplicated; } @@ -875,8 +889,9 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: } } - /// Save the current temporary path in case we need to revert the change to retry (ZK connection loss) + /// Save the current temporary path and name in case we need to revert the change to retry (ZK connection loss) or in case part is deduplicated. const String temporary_part_relative_path = part->getDataPartStorage().getPartDirectory(); + const String initial_part_name = part->name; /// Obtain incremental block number and lock it. The lock holds our intention to add the block to the filesystem. /// We remove the lock just after renaming the part. In case of exception, block number will be marked as abandoned. @@ -1045,16 +1060,6 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: zkutil::KeeperMultiException::check(multi_code, ops, responses); } - transaction.rollback(); - - if (!Coordination::isUserError(multi_code)) - throw Exception( - ErrorCodes::UNEXPECTED_ZOOKEEPER_ERROR, - "Unexpected ZooKeeper error while adding block {} with ID '{}': {}", - block_number, - toString(block_id), - multi_code); - auto failed_op_idx = zkutil::getFailedOpIndex(multi_code, responses); String failed_op_path = ops[failed_op_idx]->getPath(); @@ -1064,6 +1069,11 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: LOG_INFO(log, "Block with ID {} already exists (it was just appeared) for part {}. Ignore it.", toString(block_id), part->name); + transaction.rollbackPartsToTemporaryState(); + part->is_temp = true; + part->setName(initial_part_name); + part->renameTo(temporary_part_relative_path, false); + if constexpr (async_insert) { retry_context.conflict_block_ids = std::vector({failed_op_path}); @@ -1075,6 +1085,16 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: return CommitRetryContext::DUPLICATED_PART; } + transaction.rollback(); + + if (!Coordination::isUserError(multi_code)) + throw Exception( + ErrorCodes::UNEXPECTED_ZOOKEEPER_ERROR, + "Unexpected ZooKeeper error while adding block {} with ID '{}': {}", + block_number, + toString(block_id), + multi_code); + if (multi_code == Coordination::Error::ZNONODE && failed_op_idx == block_unlock_op_idx) throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Insert query (for block {}) was canceled by concurrent ALTER PARTITION or TRUNCATE", diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h new file mode 100644 index 00000000000..1a694a25dff --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h @@ -0,0 +1,112 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ + +template +concept StorageConfiguration = std::derived_from; + +template +class DataLakeConfiguration : public BaseStorageConfiguration, public std::enable_shared_from_this +{ +public: + using Configuration = StorageObjectStorage::Configuration; + + bool isDataLakeConfiguration() const override { return true; } + + std::string getEngineName() const override { return DataLakeMetadata::name; } + + void update(ObjectStoragePtr object_storage, ContextPtr local_context) override + { + BaseStorageConfiguration::update(object_storage, local_context); + auto new_metadata = DataLakeMetadata::create(object_storage, weak_from_this(), local_context); + if (current_metadata && *current_metadata == *new_metadata) + return; + + current_metadata = std::move(new_metadata); + BaseStorageConfiguration::setPaths(current_metadata->getDataFiles()); + BaseStorageConfiguration::setPartitionColumns(current_metadata->getPartitionColumns()); + } + + std::optional tryGetTableStructureFromMetadata() const override + { + if (!current_metadata) + return std::nullopt; + auto schema_from_metadata = current_metadata->getTableSchema(); + if (!schema_from_metadata.empty()) + { + return ColumnsDescription(std::move(schema_from_metadata)); + } + return std::nullopt; + } + +private: + DataLakeMetadataPtr current_metadata; + + ReadFromFormatInfo prepareReadingFromFormat( + ObjectStoragePtr object_storage, + const Strings & requested_columns, + const StorageSnapshotPtr & storage_snapshot, + bool supports_subset_of_columns, + ContextPtr local_context) override + { + auto info = DB::prepareReadingFromFormat(requested_columns, storage_snapshot, local_context, supports_subset_of_columns); + if (!current_metadata) + { + current_metadata = DataLakeMetadata::create(object_storage, weak_from_this(), local_context); + } + auto column_mapping = current_metadata->getColumnNameToPhysicalNameMapping(); + if (!column_mapping.empty()) + { + for (const auto & [column_name, physical_name] : column_mapping) + { + auto & column = info.format_header.getByName(column_name); + column.name = physical_name; + } + } + return info; + } +}; + +#if USE_AVRO +#if USE_AWS_S3 +using StorageS3IcebergConfiguration = DataLakeConfiguration; +# endif + +#if USE_AZURE_BLOB_STORAGE +using StorageAzureIcebergConfiguration = DataLakeConfiguration; +# endif + +#if USE_HDFS +using StorageHDFSIcebergConfiguration = DataLakeConfiguration; +# endif + +using StorageLocalIcebergConfiguration = DataLakeConfiguration; +#endif + +#if USE_PARQUET +#if USE_AWS_S3 +using StorageS3DeltaLakeConfiguration = DataLakeConfiguration; +# endif +#endif + +#if USE_AWS_S3 +using StorageS3HudiConfiguration = DataLakeConfiguration; +#endif +} diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp index 8f9bd5b19b8..ef0adc15186 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp @@ -56,22 +56,18 @@ namespace ErrorCodes struct DeltaLakeMetadataImpl { - using ConfigurationPtr = DeltaLakeMetadata::ConfigurationPtr; + using ConfigurationObserverPtr = DeltaLakeMetadata::ConfigurationObserverPtr; ObjectStoragePtr object_storage; - ConfigurationPtr configuration; + ConfigurationObserverPtr configuration; ContextPtr context; /** * Useful links: * - https://github.com/delta-io/delta/blob/master/PROTOCOL.md#data-files */ - DeltaLakeMetadataImpl(ObjectStoragePtr object_storage_, - ConfigurationPtr configuration_, - ContextPtr context_) - : object_storage(object_storage_) - , configuration(configuration_) - , context(context_) + DeltaLakeMetadataImpl(ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, ContextPtr context_) + : object_storage(object_storage_), configuration(configuration_), context(context_) { } @@ -111,6 +107,7 @@ struct DeltaLakeMetadataImpl }; DeltaLakeMetadata processMetadataFiles() { + auto configuration_ptr = configuration.lock(); std::set result_files; NamesAndTypesList current_schema; DataLakePartitionColumns current_partition_columns; @@ -122,7 +119,7 @@ struct DeltaLakeMetadataImpl while (true) { const auto filename = withPadding(++current_version) + metadata_file_suffix; - const auto file_path = std::filesystem::path(configuration->getPath()) / deltalake_metadata_directory / filename; + const auto file_path = std::filesystem::path(configuration_ptr->getPath()) / deltalake_metadata_directory / filename; if (!object_storage->exists(StoredObject(file_path))) break; @@ -136,7 +133,7 @@ struct DeltaLakeMetadataImpl } else { - const auto keys = listFiles(*object_storage, *configuration, deltalake_metadata_directory, metadata_file_suffix); + const auto keys = listFiles(*object_storage, *configuration_ptr, deltalake_metadata_directory, metadata_file_suffix); for (const String & key : keys) processMetadataFile(key, current_schema, current_partition_columns, result_files); } @@ -246,6 +243,8 @@ struct DeltaLakeMetadataImpl } } + auto configuration_ptr = configuration.lock(); + if (object->has("add")) { auto add_object = object->get("add").extract(); @@ -253,7 +252,7 @@ struct DeltaLakeMetadataImpl throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to extract `add` field"); auto path = add_object->getValue("path"); - result.insert(fs::path(configuration->getPath()) / path); + result.insert(fs::path(configuration_ptr->getPath()) / path); auto filename = fs::path(path).filename().string(); auto it = file_partition_columns.find(filename); @@ -297,7 +296,7 @@ struct DeltaLakeMetadataImpl throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to extract `remove` field"); auto path = remove_object->getValue("path"); - result.erase(fs::path(configuration->getPath()) / path); + result.erase(fs::path(configuration_ptr->getPath()) / path); } } } @@ -488,7 +487,9 @@ struct DeltaLakeMetadataImpl */ size_t readLastCheckpointIfExists() const { - const auto last_checkpoint_file = std::filesystem::path(configuration->getPath()) / deltalake_metadata_directory / "_last_checkpoint"; + auto configuration_ptr = configuration.lock(); + const auto last_checkpoint_file + = std::filesystem::path(configuration_ptr->getPath()) / deltalake_metadata_directory / "_last_checkpoint"; if (!object_storage->exists(StoredObject(last_checkpoint_file))) return 0; @@ -555,7 +556,11 @@ struct DeltaLakeMetadataImpl return 0; const auto checkpoint_filename = withPadding(version) + ".checkpoint.parquet"; - const auto checkpoint_path = std::filesystem::path(configuration->getPath()) / deltalake_metadata_directory / checkpoint_filename; + + auto configuration_ptr = configuration.lock(); + + const auto checkpoint_path + = std::filesystem::path(configuration_ptr->getPath()) / deltalake_metadata_directory / checkpoint_filename; LOG_TRACE(log, "Using checkpoint file: {}", checkpoint_path.string()); @@ -671,7 +676,7 @@ struct DeltaLakeMetadataImpl } LOG_TEST(log, "Adding {}", path); - const auto [_, inserted] = result.insert(std::filesystem::path(configuration->getPath()) / path); + const auto [_, inserted] = result.insert(std::filesystem::path(configuration_ptr->getPath()) / path); if (!inserted) throw Exception(ErrorCodes::INCORRECT_DATA, "File already exists {}", path); } @@ -682,10 +687,7 @@ struct DeltaLakeMetadataImpl LoggerPtr log = getLogger("DeltaLakeMetadataParser"); }; -DeltaLakeMetadata::DeltaLakeMetadata( - ObjectStoragePtr object_storage_, - ConfigurationPtr configuration_, - ContextPtr context_) +DeltaLakeMetadata::DeltaLakeMetadata(ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, ContextPtr context_) { auto impl = DeltaLakeMetadataImpl(object_storage_, configuration_, context_); auto result = impl.processMetadataFiles(); diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h index a479a3dd293..031d1fb9e96 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h @@ -1,5 +1,9 @@ #pragma once +#include "config.h" + +#if USE_PARQUET + #include #include #include @@ -12,13 +16,10 @@ namespace DB class DeltaLakeMetadata final : public IDataLakeMetadata { public: - using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr; + using ConfigurationObserverPtr = StorageObjectStorage::ConfigurationObserverPtr; static constexpr auto name = "DeltaLake"; - DeltaLakeMetadata( - ObjectStoragePtr object_storage_, - ConfigurationPtr configuration_, - ContextPtr context_); + DeltaLakeMetadata(ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, ContextPtr context_); Strings getDataFiles() const override { return data_files; } @@ -36,10 +37,7 @@ public: && data_files == deltalake_metadata->data_files; } - static DataLakeMetadataPtr create( - ObjectStoragePtr object_storage, - ConfigurationPtr configuration, - ContextPtr local_context) + static DataLakeMetadataPtr create(ObjectStoragePtr object_storage, ConfigurationObserverPtr configuration, ContextPtr local_context) { return std::make_unique(object_storage, configuration, local_context); } @@ -52,3 +50,5 @@ private: }; } + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/HudiMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/HudiMetadata.cpp index 91a586ccbf9..77ef769ed0e 100644 --- a/src/Storages/ObjectStorage/DataLakes/HudiMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/HudiMetadata.cpp @@ -1,11 +1,10 @@ -#include -#include #include -#include +#include +#include +#include #include #include -#include "config.h" -#include +#include namespace DB { @@ -43,8 +42,9 @@ namespace ErrorCodes */ Strings HudiMetadata::getDataFilesImpl() const { + auto configuration_ptr = configuration.lock(); auto log = getLogger("HudiMetadata"); - const auto keys = listFiles(*object_storage, *configuration, "", Poco::toLower(configuration->format)); + const auto keys = listFiles(*object_storage, *configuration_ptr, "", Poco::toLower(configuration_ptr->format)); using Partition = std::string; using FileID = std::string; @@ -86,13 +86,8 @@ Strings HudiMetadata::getDataFilesImpl() const return result; } -HudiMetadata::HudiMetadata( - ObjectStoragePtr object_storage_, - ConfigurationPtr configuration_, - ContextPtr context_) - : WithContext(context_) - , object_storage(object_storage_) - , configuration(configuration_) +HudiMetadata::HudiMetadata(ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, ContextPtr context_) + : WithContext(context_), object_storage(object_storage_), configuration(configuration_) { } diff --git a/src/Storages/ObjectStorage/DataLakes/HudiMetadata.h b/src/Storages/ObjectStorage/DataLakes/HudiMetadata.h index b060b1b0d39..cdab11c4277 100644 --- a/src/Storages/ObjectStorage/DataLakes/HudiMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/HudiMetadata.h @@ -13,14 +13,11 @@ namespace DB class HudiMetadata final : public IDataLakeMetadata, private WithContext { public: - using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr; + using ConfigurationObserverPtr = StorageObjectStorage::ConfigurationObserverPtr; static constexpr auto name = "Hudi"; - HudiMetadata( - ObjectStoragePtr object_storage_, - ConfigurationPtr configuration_, - ContextPtr context_); + HudiMetadata(ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, ContextPtr context_); Strings getDataFiles() const override; @@ -38,17 +35,14 @@ public: && data_files == hudi_metadata->data_files; } - static DataLakeMetadataPtr create( - ObjectStoragePtr object_storage, - ConfigurationPtr configuration, - ContextPtr local_context) + static DataLakeMetadataPtr create(ObjectStoragePtr object_storage, ConfigurationObserverPtr configuration, ContextPtr local_context) { return std::make_unique(object_storage, configuration, local_context); } private: const ObjectStoragePtr object_storage; - const ConfigurationPtr configuration; + const ConfigurationObserverPtr configuration; mutable Strings data_files; std::unordered_map column_name_to_physical_name; DataLakePartitionColumns partition_columns; diff --git a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h deleted file mode 100644 index 91fd9a9f981..00000000000 --- a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h +++ /dev/null @@ -1,169 +0,0 @@ -#pragma once - -#include "config.h" - -#if USE_AVRO - -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -/// Storage for read-only integration with Apache Iceberg tables in Amazon S3 (see https://iceberg.apache.org/) -/// Right now it's implemented on top of StorageS3 and right now it doesn't support -/// many Iceberg features like schema evolution, partitioning, positional and equality deletes. -template -class IStorageDataLake final : public StorageObjectStorage -{ -public: - using Storage = StorageObjectStorage; - using ConfigurationPtr = Storage::ConfigurationPtr; - - static StoragePtr create( - ConfigurationPtr base_configuration, - ContextPtr context, - const StorageID & table_id_, - const ColumnsDescription & columns_, - const ConstraintsDescription & constraints_, - const String & comment_, - std::optional format_settings_, - LoadingStrictnessLevel mode) - { - auto object_storage = base_configuration->createObjectStorage(context, /* is_readonly */true); - DataLakeMetadataPtr metadata; - NamesAndTypesList schema_from_metadata; - const bool use_schema_from_metadata = columns_.empty(); - - if (base_configuration->format == "auto") - base_configuration->format = "Parquet"; - - ConfigurationPtr configuration = base_configuration->clone(); - - try - { - metadata = DataLakeMetadata::create(object_storage, base_configuration, context); - configuration->setPaths(metadata->getDataFiles()); - if (use_schema_from_metadata) - schema_from_metadata = metadata->getTableSchema(); - } - catch (...) - { - if (mode <= LoadingStrictnessLevel::CREATE) - throw; - - metadata.reset(); - configuration->setPaths({}); - tryLogCurrentException(__PRETTY_FUNCTION__); - } - - return std::make_shared>( - base_configuration, std::move(metadata), configuration, object_storage, - context, table_id_, - use_schema_from_metadata ? ColumnsDescription(schema_from_metadata) : columns_, - constraints_, comment_, format_settings_); - } - - String getName() const override { return DataLakeMetadata::name; } - - static ColumnsDescription getTableStructureFromData( - ObjectStoragePtr object_storage_, - ConfigurationPtr base_configuration, - const std::optional & format_settings_, - ContextPtr local_context) - { - auto metadata = DataLakeMetadata::create(object_storage_, base_configuration, local_context); - - auto schema_from_metadata = metadata->getTableSchema(); - if (!schema_from_metadata.empty()) - { - return ColumnsDescription(std::move(schema_from_metadata)); - } - - ConfigurationPtr configuration = base_configuration->clone(); - configuration->setPaths(metadata->getDataFiles()); - std::string sample_path; - return Storage::resolveSchemaFromData(object_storage_, configuration, format_settings_, sample_path, local_context); - } - - void updateConfiguration(ContextPtr local_context) override - { - Storage::updateConfiguration(local_context); - - auto new_metadata = DataLakeMetadata::create(Storage::object_storage, base_configuration, local_context); - if (current_metadata && *current_metadata == *new_metadata) - return; - - current_metadata = std::move(new_metadata); - auto updated_configuration = base_configuration->clone(); - updated_configuration->setPaths(current_metadata->getDataFiles()); - updated_configuration->setPartitionColumns(current_metadata->getPartitionColumns()); - - Storage::configuration = updated_configuration; - } - - template - IStorageDataLake( - ConfigurationPtr base_configuration_, - DataLakeMetadataPtr metadata_, - Args &&... args) - : Storage(std::forward(args)...) - , base_configuration(base_configuration_) - , current_metadata(std::move(metadata_)) - { - if (base_configuration->format == "auto") - { - base_configuration->format = Storage::configuration->format; - } - - if (current_metadata) - { - const auto & columns = current_metadata->getPartitionColumns(); - base_configuration->setPartitionColumns(columns); - Storage::configuration->setPartitionColumns(columns); - } - } - -private: - ConfigurationPtr base_configuration; - DataLakeMetadataPtr current_metadata; - - ReadFromFormatInfo prepareReadingFromFormat( - const Strings & requested_columns, - const StorageSnapshotPtr & storage_snapshot, - bool supports_subset_of_columns, - ContextPtr local_context) override - { - auto info = DB::prepareReadingFromFormat(requested_columns, storage_snapshot, local_context, supports_subset_of_columns); - if (!current_metadata) - { - Storage::updateConfiguration(local_context); - current_metadata = DataLakeMetadata::create(Storage::object_storage, base_configuration, local_context); - } - auto column_mapping = current_metadata->getColumnNameToPhysicalNameMapping(); - if (!column_mapping.empty()) - { - for (const auto & [column_name, physical_name] : column_mapping) - { - auto & column = info.format_header.getByName(column_name); - column.name = physical_name; - } - } - return info; - } -}; - -using StorageIceberg = IStorageDataLake; -using StorageDeltaLake = IStorageDataLake; -using StorageHudi = IStorageDataLake; - -} - -#endif diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index e27612ca4de..f0a80a41d4e 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -51,7 +51,7 @@ extern const int UNSUPPORTED_METHOD; IcebergMetadata::IcebergMetadata( ObjectStoragePtr object_storage_, - ConfigurationPtr configuration_, + ConfigurationObserverPtr configuration_, DB::ContextPtr context_, Int32 metadata_version_, Int32 format_version_, @@ -382,12 +382,12 @@ std::pair getMetadataFileAndVersion( } -DataLakeMetadataPtr IcebergMetadata::create( - ObjectStoragePtr object_storage, - ConfigurationPtr configuration, - ContextPtr local_context) +DataLakeMetadataPtr +IcebergMetadata::create(ObjectStoragePtr object_storage, ConfigurationObserverPtr configuration, ContextPtr local_context) { - const auto [metadata_version, metadata_file_path] = getMetadataFileAndVersion(object_storage, *configuration); + auto configuration_ptr = configuration.lock(); + + const auto [metadata_version, metadata_file_path] = getMetadataFileAndVersion(object_storage, *configuration_ptr); auto log = getLogger("IcebergMetadata"); LOG_DEBUG(log, "Parse metadata {}", metadata_file_path); @@ -416,12 +416,13 @@ DataLakeMetadataPtr IcebergMetadata::create( if (snapshot->getValue("snapshot-id") == current_snapshot_id) { const auto path = snapshot->getValue("manifest-list"); - manifest_list_file = std::filesystem::path(configuration->getPath()) / "metadata" / std::filesystem::path(path).filename(); + manifest_list_file = std::filesystem::path(configuration_ptr->getPath()) / "metadata" / std::filesystem::path(path).filename(); break; } } - return std::make_unique(object_storage, configuration, local_context, metadata_version, format_version, manifest_list_file, schema_id, schema); + return std::make_unique( + object_storage, configuration_ptr, local_context, metadata_version, format_version, manifest_list_file, schema_id, schema); } /** @@ -451,6 +452,7 @@ DataLakeMetadataPtr IcebergMetadata::create( */ Strings IcebergMetadata::getDataFiles() const { + auto configuration_ptr = configuration.lock(); if (!data_files.empty()) return data_files; @@ -483,7 +485,7 @@ Strings IcebergMetadata::getDataFiles() const { const auto file_path = col_str->getDataAt(i).toView(); const auto filename = std::filesystem::path(file_path).filename(); - manifest_files.emplace_back(std::filesystem::path(configuration->getPath()) / "metadata" / filename); + manifest_files.emplace_back(std::filesystem::path(configuration_ptr->getPath()) / "metadata" / filename); } NameSet files; @@ -618,9 +620,9 @@ Strings IcebergMetadata::getDataFiles() const const auto status = status_int_column->getInt(i); const auto data_path = std::string(file_path_string_column->getDataAt(i).toView()); - const auto pos = data_path.find(configuration->getPath()); + const auto pos = data_path.find(configuration_ptr->getPath()); if (pos == std::string::npos) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected to find {} in data path: {}", configuration->getPath(), data_path); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected to find {} in data path: {}", configuration_ptr->getPath(), data_path); const auto file_path = data_path.substr(pos); diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 7b0deab91c3..eb5cac591f2 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -1,5 +1,7 @@ #pragma once +#include "config.h" + #if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format. #include @@ -61,13 +63,13 @@ namespace DB class IcebergMetadata : public IDataLakeMetadata, private WithContext { public: - using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr; + using ConfigurationObserverPtr = StorageObjectStorage::ConfigurationObserverPtr; static constexpr auto name = "Iceberg"; IcebergMetadata( ObjectStoragePtr object_storage_, - ConfigurationPtr configuration_, + ConfigurationObserverPtr configuration_, ContextPtr context_, Int32 metadata_version_, Int32 format_version_, @@ -92,16 +94,13 @@ public: return iceberg_metadata && getVersion() == iceberg_metadata->getVersion(); } - static DataLakeMetadataPtr create( - ObjectStoragePtr object_storage, - ConfigurationPtr configuration, - ContextPtr local_context); + static DataLakeMetadataPtr create(ObjectStoragePtr object_storage, ConfigurationObserverPtr configuration, ContextPtr local_context); private: size_t getVersion() const { return metadata_version; } const ObjectStoragePtr object_storage; - const ConfigurationPtr configuration; + const ConfigurationObserverPtr configuration; Int32 metadata_version; Int32 format_version; String manifest_list_file; diff --git a/src/Storages/ObjectStorage/DataLakes/registerDataLakeStorages.cpp b/src/Storages/ObjectStorage/DataLakes/registerDataLakeStorages.cpp deleted file mode 100644 index 2c2d75d81a6..00000000000 --- a/src/Storages/ObjectStorage/DataLakes/registerDataLakeStorages.cpp +++ /dev/null @@ -1,154 +0,0 @@ -#include "config.h" - -#if USE_AWS_S3 - -# include -# include -# include -# include -# include -# include - -#if USE_HDFS -# include -#endif - -namespace DB -{ - -#if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format. - -void registerStorageIceberg(StorageFactory & factory) -{ - factory.registerStorage( - "Iceberg", - [&](const StorageFactory::Arguments & args) - { - auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); - - return StorageIceberg::create( - configuration, args.getContext(), args.table_id, args.columns, args.constraints, args.comment, std::nullopt, args.mode); - }, - { - .supports_settings = false, - .supports_schema_inference = true, - .source_access_type = AccessType::S3, - }); - - factory.registerStorage( - "IcebergS3", - [&](const StorageFactory::Arguments & args) - { - auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); - - return StorageIceberg::create( - configuration, args.getContext(), args.table_id, args.columns, args.constraints, args.comment, std::nullopt, args.mode); - }, - { - .supports_settings = false, - .supports_schema_inference = true, - .source_access_type = AccessType::S3, - }); - - factory.registerStorage( - "IcebergAzure", - [&](const StorageFactory::Arguments & args) - { - auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), true); - - return StorageIceberg::create( - configuration, args.getContext(), args.table_id, args.columns, args.constraints, args.comment, std::nullopt, args.mode); - }, - { - .supports_settings = false, - .supports_schema_inference = true, - .source_access_type = AccessType::AZURE, - }); - - factory.registerStorage( - "IcebergLocal", - [&](const StorageFactory::Arguments & args) - { - auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); - - return StorageIceberg::create( - configuration, args.getContext(), args.table_id, args.columns, - args.constraints, args.comment, std::nullopt, args.mode); - }, - { - .supports_settings = false, - .supports_schema_inference = true, - .source_access_type = AccessType::FILE, - }); - -#if USE_HDFS - factory.registerStorage( - "IcebergHDFS", - [&](const StorageFactory::Arguments & args) - { - auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); - - return StorageIceberg::create( - configuration, args.getContext(), args.table_id, args.columns, - args.constraints, args.comment, std::nullopt, args.mode); - }, - { - .supports_settings = false, - .supports_schema_inference = true, - .source_access_type = AccessType::HDFS, - }); -#endif -} - -#endif - -#if USE_PARQUET -void registerStorageDeltaLake(StorageFactory & factory) -{ - factory.registerStorage( - "DeltaLake", - [&](const StorageFactory::Arguments & args) - { - auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); - - return StorageDeltaLake::create( - configuration, args.getContext(), args.table_id, args.columns, - args.constraints, args.comment, std::nullopt, args.mode); - }, - { - .supports_settings = false, - .supports_schema_inference = true, - .source_access_type = AccessType::S3, - }); -} -#endif - -void registerStorageHudi(StorageFactory & factory) -{ - factory.registerStorage( - "Hudi", - [&](const StorageFactory::Arguments & args) - { - auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); - - return StorageHudi::create( - configuration, args.getContext(), args.table_id, args.columns, - args.constraints, args.comment, std::nullopt, args.mode); - }, - { - .supports_settings = false, - .supports_schema_inference = true, - .source_access_type = AccessType::S3, - }); -} - -} - -#endif diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 579d8e95059..fd2fe0400bb 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -14,14 +14,16 @@ #include #include -#include #include -#include -#include #include +#include #include #include -#include +#include +#include +#include +#include "Databases/LoadingStrictnessLevel.h" +#include "Storages/ColumnsDescription.h" namespace DB @@ -76,6 +78,7 @@ StorageObjectStorage::StorageObjectStorage( const ConstraintsDescription & constraints_, const String & comment, std::optional format_settings_, + LoadingStrictnessLevel mode, bool distributed_processing_, ASTPtr partition_by_) : IStorage(table_id_) @@ -86,9 +89,25 @@ StorageObjectStorage::StorageObjectStorage( , distributed_processing(distributed_processing_) , log(getLogger(fmt::format("Storage{}({})", configuration->getEngineName(), table_id_.getFullTableName()))) { - ColumnsDescription columns{columns_}; + try + { + configuration->update(object_storage, context); + } + catch (...) + { + // If we don't have format or schema yet, we can't ignore failed configuration update, because relevant configuration is crucial for format and schema inference + if (mode <= LoadingStrictnessLevel::CREATE || columns_.empty() || (configuration->format == "auto")) + { + throw; + } + else + { + tryLogCurrentException(log); + } + } std::string sample_path; + ColumnsDescription columns{columns_}; resolveSchemaAndFormat(columns, configuration->format, object_storage, configuration, format_settings, sample_path, context); configuration->check(context); @@ -124,12 +143,11 @@ bool StorageObjectStorage::supportsSubsetOfColumns(const ContextPtr & context) c return FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(configuration->format, context, format_settings); } -void StorageObjectStorage::updateConfiguration(ContextPtr context) +void StorageObjectStorage::Configuration::update(ObjectStoragePtr object_storage_ptr, ContextPtr context) { - IObjectStorage::ApplyNewSettingsOptions options{ .allow_client_change = !configuration->isStaticConfiguration() }; - object_storage->applyNewSettings(context->getConfigRef(), configuration->getTypeName() + ".", context, options); + IObjectStorage::ApplyNewSettingsOptions options{.allow_client_change = !isStaticConfiguration()}; + object_storage_ptr->applyNewSettings(context->getConfigRef(), getTypeName() + ".", context, options); } - namespace { class ReadFromObjectStorageStep : public SourceStepWithFilter @@ -243,7 +261,8 @@ private: }; } -ReadFromFormatInfo StorageObjectStorage::prepareReadingFromFormat( +ReadFromFormatInfo StorageObjectStorage::Configuration::prepareReadingFromFormat( + ObjectStoragePtr, const Strings & requested_columns, const StorageSnapshotPtr & storage_snapshot, bool supports_subset_of_columns, @@ -252,6 +271,11 @@ ReadFromFormatInfo StorageObjectStorage::prepareReadingFromFormat( return DB::prepareReadingFromFormat(requested_columns, storage_snapshot, local_context, supports_subset_of_columns); } +std::optional StorageObjectStorage::Configuration::tryGetTableStructureFromMetadata() const +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method tryGetTableStructureFromMetadata is not implemented for basic configuration"); +} + void StorageObjectStorage::read( QueryPlan & query_plan, const Names & column_names, @@ -262,7 +286,7 @@ void StorageObjectStorage::read( size_t max_block_size, size_t num_streams) { - updateConfiguration(local_context); + configuration->update(object_storage, local_context); if (partition_by && configuration->withPartitionWildcard()) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, @@ -270,8 +294,8 @@ void StorageObjectStorage::read( getName()); } - const auto read_from_format_info = prepareReadingFromFormat( - column_names, storage_snapshot, supportsSubsetOfColumns(local_context), local_context); + const auto read_from_format_info = configuration->prepareReadingFromFormat( + object_storage, column_names, storage_snapshot, supportsSubsetOfColumns(local_context), local_context); const bool need_only_count = (query_info.optimize_trivial_count || read_from_format_info.requested_columns.empty()) && local_context->getSettingsRef()[Setting::optimize_count_from_files]; @@ -300,7 +324,7 @@ SinkToStoragePtr StorageObjectStorage::write( ContextPtr local_context, bool /* async_insert */) { - updateConfiguration(local_context); + configuration->update(object_storage, local_context); const auto sample_block = metadata_snapshot->getSampleBlock(); const auto & settings = configuration->getQuerySettings(local_context); @@ -409,6 +433,16 @@ ColumnsDescription StorageObjectStorage::resolveSchemaFromData( std::string & sample_path, const ContextPtr & context) { + if (configuration->isDataLakeConfiguration()) + { + configuration->update(object_storage, context); + auto table_structure = configuration->tryGetTableStructureFromMetadata(); + if (table_structure) + { + return table_structure.value(); + } + } + ObjectInfos read_keys; auto iterator = createReadBufferIterator(object_storage, configuration, format_settings, read_keys, context); auto schema = readSchemaFromFormat(configuration->format, format_settings, *iterator, context); @@ -489,10 +523,17 @@ void StorageObjectStorage::Configuration::initialize( if (configuration.format == "auto") { - configuration.format = FormatFactory::instance().tryGetFormatFromFileName( - configuration.isArchive() - ? configuration.getPathInArchive() - : configuration.getPath()).value_or("auto"); + if (configuration.isDataLakeConfiguration()) + { + configuration.format = "Parquet"; + } + else + { + configuration.format + = FormatFactory::instance() + .tryGetFormatFromFileName(configuration.isArchive() ? configuration.getPathInArchive() : configuration.getPath()) + .value_or("auto"); + } } else FormatFactory::instance().checkFormatName(configuration.format); diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 3f90586c4f3..e2bb41a4935 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -1,12 +1,13 @@ #pragma once -#include -#include #include -#include +#include #include -#include #include +#include #include +#include +#include +#include "Storages/ColumnsDescription.h" namespace DB { @@ -25,6 +26,7 @@ class StorageObjectStorage : public IStorage public: class Configuration; using ConfigurationPtr = std::shared_ptr; + using ConfigurationObserverPtr = std::weak_ptr; using ObjectInfo = RelativePathWithMetadata; using ObjectInfoPtr = std::shared_ptr; using ObjectInfos = std::vector; @@ -55,6 +57,7 @@ public: const ConstraintsDescription & constraints_, const String & comment, std::optional format_settings_, + LoadingStrictnessLevel mode, bool distributed_processing_ = false, ASTPtr partition_by_ = nullptr); @@ -120,16 +123,8 @@ public: const ContextPtr & context); protected: - virtual void updateConfiguration(ContextPtr local_context); - String getPathSample(StorageInMemoryMetadata metadata, ContextPtr context); - virtual ReadFromFormatInfo prepareReadingFromFormat( - const Strings & requested_columns, - const StorageSnapshotPtr & storage_snapshot, - bool supports_subset_of_columns, - ContextPtr local_context); - static std::unique_ptr createReadBufferIterator( const ObjectStoragePtr & object_storage, const ConfigurationPtr & configuration, @@ -207,14 +202,29 @@ public: void setPartitionColumns(const DataLakePartitionColumns & columns) { partition_columns = columns; } const DataLakePartitionColumns & getPartitionColumns() const { return partition_columns; } + virtual bool isDataLakeConfiguration() const { return false; } + + virtual ReadFromFormatInfo prepareReadingFromFormat( + ObjectStoragePtr object_storage, + const Strings & requested_columns, + const StorageSnapshotPtr & storage_snapshot, + bool supports_subset_of_columns, + ContextPtr local_context); + + virtual std::optional tryGetTableStructureFromMetadata() const; + String format = "auto"; String compression_method = "auto"; String structure = "auto"; + virtual void update(ObjectStoragePtr object_storage, ContextPtr local_context); + + protected: virtual void fromNamedCollection(const NamedCollection & collection, ContextPtr context) = 0; virtual void fromAST(ASTs & args, ContextPtr context, bool with_structure) = 0; + void assertInitialized() const; bool initialized = false; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 52b0f00f71a..1ccf23ade90 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -517,9 +517,19 @@ std::unique_ptr StorageObjectStorageSource::createReadBu LOG_TRACE(log, "Downloading object of size {} with initial prefetch", object_size); + bool prefer_bigger_buffer_size = read_settings.filesystem_cache_prefer_bigger_buffer_size && impl->isCached(); + size_t buffer_size = prefer_bigger_buffer_size + ? std::max(read_settings.remote_fs_buffer_size, DBMS_DEFAULT_BUFFER_SIZE) + : read_settings.remote_fs_buffer_size; + if (object_size) + buffer_size = std::min(object_size, buffer_size); + auto & reader = context_->getThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER); impl = std::make_unique( - std::move(impl), reader, modified_read_settings, + std::move(impl), + reader, + modified_read_settings, + buffer_size, context_->getAsyncReadCounters(), context_->getFilesystemReadPrefetchesLog()); diff --git a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp index d0cacc29adf..e94f1860176 100644 --- a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -10,17 +11,19 @@ namespace DB { -#if USE_AWS_S3 || USE_AZURE_BLOB_STORAGE || USE_HDFS - namespace ErrorCodes { extern const int BAD_ARGUMENTS; } -static std::shared_ptr createStorageObjectStorage( - const StorageFactory::Arguments & args, - StorageObjectStorage::ConfigurationPtr configuration, - ContextPtr context) +namespace +{ + +// LocalObjectStorage is only supported for Iceberg Datalake operations where Avro format is required. For regular file access, use FileStorage instead. +#if USE_AWS_S3 || USE_AZURE_BLOB_STORAGE || USE_HDFS || USE_AVRO + +std::shared_ptr +createStorageObjectStorage(const StorageFactory::Arguments & args, StorageObjectStorage::ConfigurationPtr configuration, ContextPtr context) { auto & engine_args = args.engine_args; if (engine_args.empty()) @@ -52,18 +55,20 @@ static std::shared_ptr createStorageObjectStorage( return std::make_shared( configuration, - configuration->createObjectStorage(context, /* is_readonly */false), + configuration->createObjectStorage(context, /* is_readonly */ false), args.getContext(), args.table_id, args.columns, args.constraints, args.comment, format_settings, + args.mode, /* distributed_processing */ false, partition_by); } #endif +} #if USE_AZURE_BLOB_STORAGE void registerStorageAzure(StorageFactory & factory) @@ -148,4 +153,133 @@ void registerStorageObjectStorage(StorageFactory & factory) UNUSED(factory); } +#if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format. + +void registerStorageIceberg(StorageFactory & factory) +{ +#if USE_AWS_S3 + factory.registerStorage( + "Iceberg", + [&](const StorageFactory::Arguments & args) + { + auto configuration = std::make_shared(); + StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); + + return createStorageObjectStorage(args, configuration, args.getLocalContext()); + }, + { + .supports_settings = false, + .supports_schema_inference = true, + .source_access_type = AccessType::S3, + }); + + factory.registerStorage( + "IcebergS3", + [&](const StorageFactory::Arguments & args) + { + auto configuration = std::make_shared(); + StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); + + return createStorageObjectStorage(args, configuration, args.getLocalContext()); + }, + { + .supports_settings = false, + .supports_schema_inference = true, + .source_access_type = AccessType::S3, + }); +#endif +#if USE_AZURE_BLOB_STORAGE + factory.registerStorage( + "IcebergAzure", + [&](const StorageFactory::Arguments & args) + { + auto configuration = std::make_shared(); + StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), true); + + return createStorageObjectStorage(args, configuration, args.getLocalContext()); + }, + { + .supports_settings = false, + .supports_schema_inference = true, + .source_access_type = AccessType::AZURE, + }); +#endif +#if USE_HDFS + factory.registerStorage( + "IcebergHDFS", + [&](const StorageFactory::Arguments & args) + { + auto configuration = std::make_shared(); + StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); + + return createStorageObjectStorage(args, configuration, args.getLocalContext()); + }, + { + .supports_settings = false, + .supports_schema_inference = true, + .source_access_type = AccessType::HDFS, + }); +#endif + factory.registerStorage( + "IcebergLocal", + [&](const StorageFactory::Arguments & args) + { + auto configuration = std::make_shared(); + StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); + + return createStorageObjectStorage(args, configuration, args.getLocalContext()); + }, + { + .supports_settings = false, + .supports_schema_inference = true, + .source_access_type = AccessType::FILE, + }); +} + +#endif + + +#if USE_PARQUET +void registerStorageDeltaLake(StorageFactory & factory) +{ +#if USE_AWS_S3 + factory.registerStorage( + "DeltaLake", + [&](const StorageFactory::Arguments & args) + { + auto configuration = std::make_shared(); + StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); + + return createStorageObjectStorage(args, configuration, args.getLocalContext()); + }, + { + .supports_settings = false, + .supports_schema_inference = true, + .source_access_type = AccessType::S3, + }); +#endif + UNUSED(factory); +} +#endif + +void registerStorageHudi(StorageFactory & factory) +{ +#if USE_AWS_S3 + factory.registerStorage( + "Hudi", + [&](const StorageFactory::Arguments & args) + { + auto configuration = std::make_shared(); + StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); + + return createStorageObjectStorage(args, configuration, args.getLocalContext()); + }, + { + .supports_settings = false, + .supports_schema_inference = true, + .source_access_type = AccessType::S3, + }); +#endif + UNUSED(factory); +} } diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 200872a2f4c..6630d5a6c8f 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -747,6 +747,7 @@ ObjectStorageQueueSettings StorageObjectStorageQueue::getSettings() const /// so let's reconstruct. ObjectStorageQueueSettings settings; const auto & table_metadata = getTableMetadata(); + settings[ObjectStorageQueueSetting::mode] = table_metadata.mode; settings[ObjectStorageQueueSetting::after_processing] = table_metadata.after_processing; settings[ObjectStorageQueueSetting::keeper_path] = zk_path; settings[ObjectStorageQueueSetting::loading_retries] = table_metadata.loading_retries; @@ -764,6 +765,7 @@ ObjectStorageQueueSettings StorageObjectStorageQueue::getSettings() const settings[ObjectStorageQueueSetting::max_processed_files_before_commit] = commit_settings.max_processed_files_before_commit; settings[ObjectStorageQueueSetting::max_processed_rows_before_commit] = commit_settings.max_processed_rows_before_commit; settings[ObjectStorageQueueSetting::max_processed_bytes_before_commit] = commit_settings.max_processed_bytes_before_commit; + settings[ObjectStorageQueueSetting::max_processing_time_sec_before_commit] = commit_settings.max_processing_time_sec_before_commit; return settings; } diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp deleted file mode 100644 index ac560b58962..00000000000 --- a/src/Storages/StorageExternalDistributed.cpp +++ /dev/null @@ -1,233 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -namespace Setting -{ - extern const SettingsUInt64 glob_expansion_max_elements; - extern const SettingsUInt64 postgresql_connection_attempt_timeout; - extern const SettingsBool postgresql_connection_pool_auto_close_connection; - extern const SettingsUInt64 postgresql_connection_pool_retries; - extern const SettingsUInt64 postgresql_connection_pool_size; - extern const SettingsUInt64 postgresql_connection_pool_wait_timeout; -} - -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - -StorageExternalDistributed::StorageExternalDistributed( - const StorageID & table_id_, - std::unordered_set && shards_, - const ColumnsDescription & columns_, - const ConstraintsDescription & constraints_, - const String & comment) - : IStorage(table_id_) - , shards(shards_) -{ - StorageInMemoryMetadata storage_metadata; - storage_metadata.setColumns(columns_); - storage_metadata.setConstraints(constraints_); - storage_metadata.setComment(comment); - setInMemoryMetadata(storage_metadata); -} - -void StorageExternalDistributed::read( - QueryPlan & query_plan, - const Names & column_names, - const StorageSnapshotPtr & storage_snapshot, - SelectQueryInfo & query_info, - ContextPtr context, - QueryProcessingStage::Enum processed_stage, - size_t max_block_size, - size_t num_streams) -{ - std::vector> plans; - for (const auto & shard : shards) - { - plans.emplace_back(std::make_unique()); - shard->read( - *plans.back(), - column_names, - storage_snapshot, - query_info, - context, - processed_stage, - max_block_size, - num_streams - ); - } - - if (plans.empty()) - { - auto header = storage_snapshot->getSampleBlockForColumns(column_names); - InterpreterSelectQuery::addEmptySourceToQueryPlan(query_plan, header, query_info); - } - - if (plans.size() == 1) - { - query_plan = std::move(*plans.front()); - return; - } - - Headers input_headers; - input_headers.reserve(plans.size()); - for (auto & plan : plans) - input_headers.emplace_back(plan->getCurrentHeader()); - - auto union_step = std::make_unique(std::move(input_headers)); - query_plan.unitePlans(std::move(union_step), std::move(plans)); -} - -void registerStorageExternalDistributed(StorageFactory & factory) -{ - factory.registerStorage("ExternalDistributed", [](const StorageFactory::Arguments & args) - { - ASTs & engine_args = args.engine_args; - if (engine_args.size() < 2) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Engine ExternalDistributed must have at least 2 arguments: " - "engine_name, named_collection and/or description"); - - auto context = args.getLocalContext(); - const auto & settings = context->getSettingsRef(); - size_t max_addresses = settings[Setting::glob_expansion_max_elements]; - auto get_addresses = [&](const std::string addresses_expr) - { - return parseRemoteDescription(addresses_expr, 0, addresses_expr.size(), ',', max_addresses); - }; - - std::unordered_set shards; - ASTs inner_engine_args(engine_args.begin() + 1, engine_args.end()); - - ASTPtr * address_arg = nullptr; - - /// If there is a named collection argument, named `addresses_expr` - for (auto & node : inner_engine_args) - { - if (ASTFunction * func = node->as(); func && func->name == "equals" && func->arguments) - { - if (ASTExpressionList * func_args = func->arguments->as(); func_args && func_args->children.size() == 2) - { - if (ASTIdentifier * arg_name = func_args->children[0]->as(); arg_name && arg_name->name() == "addresses_expr") - { - address_arg = &func_args->children[1]; - break; - } - } - } - } - - /// Otherwise it is the first argument. - if (!address_arg) - address_arg = &inner_engine_args.at(0); - - String addresses_expr = checkAndGetLiteralArgument(*address_arg, "addresses"); - Strings shards_addresses = get_addresses(addresses_expr); - - auto engine_name = checkAndGetLiteralArgument(engine_args[0], "engine_name"); - if (engine_name == "URL") - { - auto format_settings = StorageURL::getFormatSettingsFromArgs(args); - for (const auto & shard_address : shards_addresses) - { - *address_arg = std::make_shared(shard_address); - auto configuration = StorageURL::getConfiguration(inner_engine_args, context); - auto uri_options = parseRemoteDescription(shard_address, 0, shard_address.size(), '|', max_addresses); - if (uri_options.size() > 1) - { - shards.insert( - std::make_shared( - uri_options, args.table_id, configuration.format, format_settings, - args.columns, args.constraints, context, configuration.compression_method)); - } - else - { - shards.insert(std::make_shared( - shard_address, args.table_id, configuration.format, format_settings, - args.columns, args.constraints, String{}, context, configuration.compression_method)); - } - } - } -#if USE_MYSQL - else if (engine_name == "MySQL") - { - MySQLSettings mysql_settings; - for (const auto & shard_address : shards_addresses) - { - *address_arg = std::make_shared(shard_address); - auto configuration = StorageMySQL::getConfiguration(inner_engine_args, context, mysql_settings); - configuration.addresses = parseRemoteDescriptionForExternalDatabase(shard_address, max_addresses, 3306); - auto pool = createMySQLPoolWithFailover(configuration, mysql_settings); - shards.insert(std::make_shared( - args.table_id, std::move(pool), configuration.database, configuration.table, - /* replace_query = */ false, /* on_duplicate_clause = */ "", - args.columns, args.constraints, String{}, context, mysql_settings)); - } - } -#endif -#if USE_LIBPQXX - else if (engine_name == "PostgreSQL") - { - for (const auto & shard_address : shards_addresses) - { - *address_arg = std::make_shared(shard_address); - auto configuration = StoragePostgreSQL::getConfiguration(inner_engine_args, context); - configuration.addresses = parseRemoteDescriptionForExternalDatabase(shard_address, max_addresses, 5432); - auto pool = std::make_shared( - configuration, - settings[Setting::postgresql_connection_pool_size], - settings[Setting::postgresql_connection_pool_wait_timeout], - settings[Setting::postgresql_connection_pool_retries], - settings[Setting::postgresql_connection_pool_auto_close_connection], - settings[Setting::postgresql_connection_attempt_timeout]); - shards.insert(std::make_shared( - args.table_id, std::move(pool), configuration.table, args.columns, args.constraints, String{}, context)); - } - } -#endif - else - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "External storage engine {} is not supported for StorageExternalDistributed. " - "Supported engines are: MySQL, PostgreSQL, URL", - engine_name); - } - - return std::make_shared( - args.table_id, - std::move(shards), - args.columns, - args.constraints, - args.comment); - }, - { - .source_access_type = AccessType::SOURCES, - }); -} - -} diff --git a/src/Storages/StorageExternalDistributed.h b/src/Storages/StorageExternalDistributed.h deleted file mode 100644 index 56c7fe86f34..00000000000 --- a/src/Storages/StorageExternalDistributed.h +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once - -#include "config.h" - -#include - - -namespace DB -{ - -/// Storages MySQL and PostgreSQL use ConnectionPoolWithFailover and support multiple replicas. -/// This class unites multiple storages with replicas into multiple shards with replicas. -/// A query to external database is passed to one replica on each shard, the result is united. -/// Replicas on each shard have the same priority, traversed replicas are moved to the end of the queue. -/// Similar approach is used for URL storage. -class StorageExternalDistributed final : public DB::IStorage -{ -public: - StorageExternalDistributed( - const StorageID & table_id_, - std::unordered_set && shards_, - const ColumnsDescription & columns_, - const ConstraintsDescription & constraints_, - const String & comment); - - std::string getName() const override { return "ExternalDistributed"; } - - void read( - QueryPlan & query_plan, - const Names & column_names, - const StorageSnapshotPtr & storage_snapshot, - SelectQueryInfo & query_info, - ContextPtr context, - QueryProcessingStage::Enum processed_stage, - size_t max_block_size, - size_t num_streams) override; - -private: - using Shards = std::unordered_set; - Shards shards; -}; - -} diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index d047b28e076..9491c40b65f 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -228,10 +228,20 @@ StorageMaterializedView::StorageMaterializedView( if (!fixed_uuid) { - if (to_inner_uuid != UUIDHelpers::Nil) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "TO INNER UUID is not allowed for materialized views with REFRESH without APPEND"); - if (to_table_id.hasUUID()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "explicit UUID is not allowed for target table of materialized view with REFRESH without APPEND"); + if (mode >= LoadingStrictnessLevel::ATTACH) + { + /// Old versions of ClickHouse (when refreshable MV was experimental) could add useless + /// UUIDs to attach queries. + to_table_id.uuid = UUIDHelpers::Nil; + to_inner_uuid = UUIDHelpers::Nil; + } + else + { + if (to_inner_uuid != UUIDHelpers::Nil) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "TO INNER UUID is not allowed for materialized views with REFRESH without APPEND"); + if (to_table_id.hasUUID()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "explicit UUID is not allowed for target table of materialized view with REFRESH without APPEND"); + } } if (!has_inner_table) @@ -382,6 +392,7 @@ void StorageMaterializedView::read( } query_plan.addStorageHolder(storage); + query_plan.addInterpreterContext(context); query_plan.addTableLock(std::move(lock)); } } @@ -405,6 +416,7 @@ SinkToStoragePtr StorageMaterializedView::write(const ASTPtr & query, const Stor auto sink = storage->write(query, metadata_snapshot, context, async_insert); + sink->addInterpreterContext(context); sink->addTableLock(lock); return sink; } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index f9f90562426..c99593eaa8f 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1563,7 +1563,7 @@ bool ReadFromMerge::requestReadingInOrder(InputOrderInfoPtr order_info_) auto request_read_in_order = [order_info_](ReadFromMergeTree & read_from_merge_tree) { return read_from_merge_tree.requestReadingInOrder( - order_info_->used_prefix_of_sorting_key_size, order_info_->direction, order_info_->limit); + order_info_->used_prefix_of_sorting_key_size, order_info_->direction, order_info_->limit, {}); }; bool ok = true; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 40cd6e01dba..1ba0617d8ae 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -155,7 +155,7 @@ StorageMergeTree::StorageMergeTree( loadMutations(); loadDeduplicationLog(); - prewarmMarkCache(getActivePartsLoadingThreadPool().get()); + prewarmMarkCacheIfNeeded(getActivePartsLoadingThreadPool().get()); } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index bbfedb2f355..793fd02c656 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -509,7 +509,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } loadDataParts(skip_sanity_checks, expected_parts_on_this_replica); - prewarmMarkCache(getActivePartsLoadingThreadPool().get()); + prewarmMarkCacheIfNeeded(getActivePartsLoadingThreadPool().get()); if (LoadingStrictnessLevel::ATTACH <= mode) { @@ -2095,7 +2095,7 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo const auto part_old_name = part_info->getPartNameV1(); const auto volume = std::make_shared("volume_" + part_old_name, disk); - auto part = getDataPartBuilder(entry.new_part_name, volume, fs::path(DETACHED_DIR_NAME) / part_old_name) + auto part = getDataPartBuilder(entry.new_part_name, volume, fs::path(DETACHED_DIR_NAME) / part_old_name, getReadSettings()) .withPartFormatFromDisk() .build(); diff --git a/src/Storages/System/StorageSystemDashboards.cpp b/src/Storages/System/StorageSystemDashboards.cpp index 96ba7e59cf2..27579da4bfe 100644 --- a/src/Storages/System/StorageSystemDashboards.cpp +++ b/src/Storages/System/StorageSystemDashboards.cpp @@ -227,6 +227,194 @@ FROM merge('system', '^metric_log') WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} GROUP BY t ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + /// Default per host dashboard for self-managed ClickHouse + { + { "dashboard", "Overview (host)" }, + { "title", "Queries/second" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(ProfileEvent_Query) +FROM merge('system', '^metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "CPU Usage (cores)" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(ProfileEvent_OSCPUVirtualTimeMicroseconds) / 1000000 +FROM merge('system', '^metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "Queries Running" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(CurrentMetric_Query) +FROM merge('system', '^metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "Merges Running" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(CurrentMetric_Merge) +FROM merge('system', '^metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "Selected Bytes/second" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(ProfileEvent_SelectedBytes) +FROM merge('system', '^metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "IO Wait" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(ProfileEvent_OSIOWaitMicroseconds) / 1000000 +FROM merge('system', '^metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "CPU Wait" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(ProfileEvent_OSCPUWaitMicroseconds) / 1000000 +FROM merge('system', '^metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "OS CPU Usage (Userspace)" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(value) +FROM merge('system', '^asynchronous_metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} AND metric = 'OSUserTimeNormalized' +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "OS CPU Usage (Kernel)" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(value) +FROM merge('system', '^asynchronous_metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} AND metric = 'OSSystemTimeNormalized' +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "Read From Disk" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(ProfileEvent_OSReadBytes) +FROM merge('system', '^metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "Read From Filesystem" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(ProfileEvent_OSReadChars) +FROM merge('system', '^metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "Memory (tracked)" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(CurrentMetric_MemoryTracking) +FROM merge('system', '^metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "Load Average (15 minutes)" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(value) +FROM merge('system', '^asynchronous_metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} AND metric = 'LoadAverage15' +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "Selected Rows/second" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(ProfileEvent_SelectedRows) +FROM merge('system', '^metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "Inserted Rows/second" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(ProfileEvent_InsertedRows) +FROM merge('system', '^metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "Total MergeTree Parts" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, avg(value) +FROM merge('system', '^asynchronous_metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} AND metric = 'TotalPartsOfMergeTreeTables' +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} +)EOQ") } + }, + { + { "dashboard", "Overview (host)" }, + { "title", "Max Parts For Partition" }, + { "query", trim(R"EOQ( +SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t, hostname, max(value) +FROM merge('system', '^asynchronous_metric_log') +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} AND metric = 'MaxPartCountForPartition' +GROUP BY t, hostname +ORDER BY t WITH FILL STEP {rounding:UInt32} )EOQ") } }, /// Default dashboard for ClickHouse Cloud @@ -369,7 +557,143 @@ ORDER BY t WITH FILL STEP {rounding:UInt32} { "dashboard", "Cloud overview" }, { "title", "Concurrent network connections" }, { "query", "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, max(TCP_Connections), max(MySQL_Connections), max(HTTP_Connections) FROM (SELECT event_time, sum(CurrentMetric_TCPConnection) AS TCP_Connections, sum(CurrentMetric_MySQLConnection) AS MySQL_Connections, sum(CurrentMetric_HTTPConnection) AS HTTP_Connections FROM clusterAllReplicas(default, merge('system', '^metric_log')) WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} GROUP BY event_time) GROUP BY t ORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } - } + }, + /// Default per host dashboard for ClickHouse Cloud + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Queries/second" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_Query) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "CPU Usage (cores)" }, + { "query", "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, hostname, avg(metric) / 1000000\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_OSCPUVirtualTimeMicroseconds) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32} GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Queries Running" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(CurrentMetric_Query) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Merges Running" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(CurrentMetric_Merge) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Selected Bytes/second" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_SelectedBytes) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "IO Wait (local fs)" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_OSIOWaitMicroseconds) / 1000000 AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "S3 read wait" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_ReadBufferFromS3Microseconds) / 1000000 AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "S3 read errors/sec" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_ReadBufferFromS3RequestsErrors) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "CPU Wait" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_OSCPUWaitMicroseconds) / 1000000 AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "OS CPU Usage (Userspace, normalized)" }, + { "query", "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, hostname, avg(value)\nFROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log'))\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nAND metric = 'OSUserTimeNormalized'\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "OS CPU Usage (Kernel, normalized)" }, + { "query", "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, hostname, avg(value)\nFROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log'))\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nAND metric = 'OSSystemTimeNormalized'\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Read From Disk (bytes/sec)" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_OSReadBytes) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Read From Filesystem (bytes/sec)" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_OSReadChars) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Memory (tracked, bytes)" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(CurrentMetric_MemoryTracking) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Load Average (15 minutes)" }, + { "query", "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, hostname, avg(value)\nFROM (\n SELECT event_time, hostname, sum(value) AS value\n FROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n AND metric = 'LoadAverage15'\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Selected Rows/sec" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_SelectedRows) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Inserted Rows/sec" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_InsertedRows) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Total MergeTree Parts" }, + { "query", "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, hostname, max(value)\nFROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log'))\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nAND metric = 'TotalPartsOfMergeTreeTables'\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Max Parts For Partition" }, + { "query", "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, hostname, max(value)\nFROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log'))\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nAND metric = 'MaxPartCountForPartition'\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Read From S3 (bytes/sec)" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_ReadBufferFromS3Bytes) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Filesystem Cache Size" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(CurrentMetric_FilesystemCacheSize) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Disk S3 write req/sec" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT as t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_DiskS3PutObject + ProfileEvent_DiskS3UploadPart + ProfileEvent_DiskS3CreateMultipartUpload + ProfileEvent_DiskS3CompleteMultipartUpload) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\n GROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Disk S3 read req/sec" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_DiskS3GetObject + ProfileEvent_DiskS3HeadObject + ProfileEvent_DiskS3ListObjects) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\nGROUP BY t, hostname\nORDER BY t\nWITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "FS cache hit rate" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, sum(ProfileEvent_CachedReadBufferReadFromCacheBytes) / (sum(ProfileEvent_CachedReadBufferReadFromCacheBytes) + sum(ProfileEvent_CachedReadBufferReadFromSourceBytes)) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\nGROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Page cache hit rate" }, + { "query", "SELECT \n toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t,\n hostname,\n avg(metric)\nFROM (\n SELECT event_time, hostname, greatest(0, (sum(ProfileEvent_OSReadChars) - sum(ProfileEvent_OSReadBytes)) / (sum(ProfileEvent_OSReadChars) + sum(ProfileEvent_ReadBufferFromS3Bytes))) AS metric \n FROM clusterAllReplicas(default, merge('system', '^metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n GROUP BY event_time, hostname)\nGROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Network receive bytes/sec" }, + { "query", "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, hostname, avg(value)\nFROM (\n SELECT event_time, hostname, sum(value) AS value\n FROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n AND metric LIKE 'NetworkReceiveBytes%'\n GROUP BY event_time, hostname)\nGROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, + { + { "dashboard", "Cloud overview (host)" }, + { "title", "Network send bytes/sec" }, + { "query", "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, hostname, avg(value)\nFROM (\n SELECT event_time, hostname, sum(value) AS value\n FROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log'))\n WHERE event_date >= toDate(now() - {seconds:UInt32})\n AND event_time >= now() - {seconds:UInt32}\n AND metric LIKE 'NetworkSendBytes%'\n GROUP BY event_time, hostname)\nGROUP BY t, hostname\nORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1" } + }, }; auto add_dashboards = [&](const auto & dashboards) diff --git a/src/Storages/System/StorageSystemGrants.cpp b/src/Storages/System/StorageSystemGrants.cpp index 5de1f8cef55..aa010e44388 100644 --- a/src/Storages/System/StorageSystemGrants.cpp +++ b/src/Storages/System/StorageSystemGrants.cpp @@ -30,8 +30,8 @@ ColumnsDescription StorageSystemGrants::getColumnsDescription() {"column", std::make_shared(std::make_shared()), "Name of a column to which access is granted."}, {"is_partial_revoke", std::make_shared(), "Logical value. It shows whether some privileges have been revoked. Possible values: " - "0 — The row describes a partial revoke, " - "1 — The row describes a grant." + "0 — The row describes a grant, " + "1 — The row describes a partial revoke." }, {"grant_option", std::make_shared(), "Permission is granted WITH GRANT OPTION."}, }; diff --git a/src/Storages/System/StorageSystemParts.cpp b/src/Storages/System/StorageSystemParts.cpp index 56a45d7b51d..d0e34842198 100644 --- a/src/Storages/System/StorageSystemParts.cpp +++ b/src/Storages/System/StorageSystemParts.cpp @@ -75,6 +75,8 @@ StorageSystemParts::StorageSystemParts(const StorageID & table_id_) {"data_version", std::make_shared(), "Number that is used to determine which mutations should be applied to the data part (mutations with a version higher than data_version)."}, {"primary_key_bytes_in_memory", std::make_shared(), "The amount of memory (in bytes) used by primary key values."}, {"primary_key_bytes_in_memory_allocated", std::make_shared(), "The amount of memory (in bytes) reserved for primary key values."}, + {"index_granularity_bytes_in_memory", std::make_shared(), "The amount of memory (in bytes) used by index granularity values."}, + {"index_granularity_bytes_in_memory_allocated", std::make_shared(), "The amount of memory (in bytes) reserved for index granularity values."}, {"is_frozen", std::make_shared(), "Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup does not exist. "}, {"database", std::make_shared(), "Name of the database."}, @@ -216,6 +218,10 @@ void StorageSystemParts::processNextStorage( columns[res_index++]->insert(part->getIndexSizeInBytes()); if (columns_mask[src_index++]) columns[res_index++]->insert(part->getIndexSizeInAllocatedBytes()); + if (columns_mask[src_index++]) + columns[res_index++]->insert(part->getIndexGranularityBytes()); + if (columns_mask[src_index++]) + columns[res_index++]->insert(part->getIndexGranularityAllocatedBytes()); if (columns_mask[src_index++]) columns[res_index++]->insert(part->is_frozen.load(std::memory_order_relaxed)); diff --git a/src/Storages/TimeSeries/PrometheusRemoteReadProtocol.cpp b/src/Storages/TimeSeries/PrometheusRemoteReadProtocol.cpp index df0f6b8bc5c..b8a3b2911b9 100644 --- a/src/Storages/TimeSeries/PrometheusRemoteReadProtocol.cpp +++ b/src/Storages/TimeSeries/PrometheusRemoteReadProtocol.cpp @@ -245,6 +245,7 @@ namespace table_join->strictness = JoinStrictness::Semi; table_join->on_expression = makeASTFunction("equals", makeASTColumn(data_table_id, TimeSeriesColumnNames::ID), makeASTColumn(tags_table_id, TimeSeriesColumnNames::ID)); + table_join->children.push_back(table_join->on_expression); table->table_join = table_join; auto table_exp = std::make_shared(); diff --git a/src/Storages/fuzzers/CMakeLists.txt b/src/Storages/fuzzers/CMakeLists.txt index 2c7c0c16fc2..719b9b77cd9 100644 --- a/src/Storages/fuzzers/CMakeLists.txt +++ b/src/Storages/fuzzers/CMakeLists.txt @@ -4,4 +4,4 @@ clickhouse_add_executable (mergetree_checksum_fuzzer mergetree_checksum_fuzzer.c target_link_libraries (mergetree_checksum_fuzzer PRIVATE dbms) clickhouse_add_executable (columns_description_fuzzer columns_description_fuzzer.cpp) -target_link_libraries (columns_description_fuzzer PRIVATE) +target_link_libraries (columns_description_fuzzer PRIVATE dbms) diff --git a/src/Storages/registerStorages.cpp b/src/Storages/registerStorages.cpp index cfd406ccbe2..458b151a400 100644 --- a/src/Storages/registerStorages.cpp +++ b/src/Storages/registerStorages.cpp @@ -41,10 +41,11 @@ void registerStorageS3Queue(StorageFactory & factory); #if USE_PARQUET void registerStorageDeltaLake(StorageFactory & factory); #endif +#endif + #if USE_AVRO void registerStorageIceberg(StorageFactory & factory); #endif -#endif #if USE_AZURE_BLOB_STORAGE void registerStorageAzureQueue(StorageFactory & factory); @@ -93,10 +94,6 @@ void registerStoragePostgreSQL(StorageFactory & factory); void registerStorageMaterializedPostgreSQL(StorageFactory & factory); #endif -#if USE_MYSQL || USE_LIBPQXX -void registerStorageExternalDistributed(StorageFactory & factory); -#endif - #if USE_FILELOG void registerStorageFileLog(StorageFactory & factory); #endif @@ -144,6 +141,10 @@ void registerStorages(bool use_legacy_mongodb_integration [[maybe_unused]]) registerStorageAzureQueue(factory); #endif +#if USE_AVRO + registerStorageIceberg(factory); +#endif + #if USE_AWS_S3 registerStorageHudi(factory); registerStorageS3Queue(factory); @@ -152,14 +153,10 @@ void registerStorages(bool use_legacy_mongodb_integration [[maybe_unused]]) registerStorageDeltaLake(factory); #endif - #if USE_AVRO - registerStorageIceberg(factory); - #endif +#endif - #endif - - #if USE_HDFS - #if USE_HIVE +#if USE_HDFS +# if USE_HIVE registerStorageHive(factory); #endif #endif @@ -205,10 +202,6 @@ void registerStorages(bool use_legacy_mongodb_integration [[maybe_unused]]) registerStorageMaterializedPostgreSQL(factory); #endif - #if USE_MYSQL || USE_LIBPQXX - registerStorageExternalDistributed(factory); - #endif - #if USE_SQLITE registerStorageSQLite(factory); #endif diff --git a/src/TableFunctions/ITableFunctionCluster.h b/src/TableFunctions/ITableFunctionCluster.h index 744d7139d16..6935ac39e79 100644 --- a/src/TableFunctions/ITableFunctionCluster.h +++ b/src/TableFunctions/ITableFunctionCluster.h @@ -51,7 +51,7 @@ protected: "corresponding table function", getName()); - /// Evaluate only first argument, everything else will be done Base class + /// Evaluate only first argument, everything else will be done by the Base class args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context); /// Cluster name is always the first diff --git a/src/TableFunctions/ITableFunctionDataLake.h b/src/TableFunctions/ITableFunctionDataLake.h deleted file mode 100644 index eff181d168f..00000000000 --- a/src/TableFunctions/ITableFunctionDataLake.h +++ /dev/null @@ -1,126 +0,0 @@ -#pragma once - -#include "config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -template -class ITableFunctionDataLake : public TableFunction -{ -public: - static constexpr auto name = Name::name; - std::string getName() const override { return name; } - -protected: - StoragePtr executeImpl( - const ASTPtr & /* ast_function */, - ContextPtr context, - const std::string & table_name, - ColumnsDescription cached_columns, - bool /*is_insert_query*/) const override - { - ColumnsDescription columns; - auto configuration = TableFunction::getConfiguration(); - if (configuration->structure != "auto") - columns = parseColumnsListFromString(configuration->structure, context); - else if (!cached_columns.empty()) - columns = cached_columns; - - StoragePtr storage = Storage::create( - configuration, context, StorageID(TableFunction::getDatabaseName(), table_name), - columns, ConstraintsDescription{}, String{}, std::nullopt, LoadingStrictnessLevel::CREATE); - - storage->startup(); - return storage; - } - - const char * getStorageTypeName() const override { return name; } - - ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override - { - auto configuration = TableFunction::getConfiguration(); - if (configuration->structure == "auto") - { - context->checkAccess(TableFunction::getSourceAccessType()); - auto object_storage = TableFunction::getObjectStorage(context, !is_insert_query); - return Storage::getTableStructureFromData(object_storage, configuration, std::nullopt, context); - } - - return parseColumnsListFromString(configuration->structure, context); - } - - void parseArguments(const ASTPtr & ast_function, ContextPtr context) override - { - auto configuration = TableFunction::getConfiguration(); - configuration->format = "Parquet"; - /// Set default format to Parquet if it's not specified in arguments. - TableFunction::parseArguments(ast_function, context); - } -}; - -struct TableFunctionIcebergName -{ - static constexpr auto name = "iceberg"; -}; - -struct TableFunctionIcebergS3Name -{ - static constexpr auto name = "icebergS3"; -}; - -struct TableFunctionIcebergAzureName -{ - static constexpr auto name = "icebergAzure"; -}; - -struct TableFunctionIcebergLocalName -{ - static constexpr auto name = "icebergLocal"; -}; - -struct TableFunctionIcebergHDFSName -{ - static constexpr auto name = "icebergHDFS"; -}; - -struct TableFunctionDeltaLakeName -{ - static constexpr auto name = "deltaLake"; -}; - -struct TableFunctionHudiName -{ - static constexpr auto name = "hudi"; -}; - -#if USE_AVRO -# if USE_AWS_S3 -using TableFunctionIceberg = ITableFunctionDataLake; -using TableFunctionIcebergS3 = ITableFunctionDataLake; -# endif -# if USE_AZURE_BLOB_STORAGE -using TableFunctionIcebergAzure = ITableFunctionDataLake; -# endif -using TableFunctionIcebergLocal = ITableFunctionDataLake; -#if USE_HDFS -using TableFunctionIcebergHDFS = ITableFunctionDataLake; -#endif -#endif -#if USE_AWS_S3 -# if USE_PARQUET -using TableFunctionDeltaLake = ITableFunctionDataLake; -#endif -using TableFunctionHudi = ITableFunctionDataLake; -#endif -} diff --git a/src/TableFunctions/TableFunctionMongoDB.cpp b/src/TableFunctions/TableFunctionMongoDB.cpp index e13427c1557..9f91839fb33 100644 --- a/src/TableFunctions/TableFunctionMongoDB.cpp +++ b/src/TableFunctions/TableFunctionMongoDB.cpp @@ -15,7 +15,7 @@ #include #include #include - +#include namespace DB { @@ -85,17 +85,11 @@ void TableFunctionMongoDB::parseArguments(const ASTPtr & ast_function, ContextPt { if (const auto * ast_func = typeid_cast(args[i].get())) { - const auto * args_expr = assert_cast(ast_func->arguments.get()); - auto function_args = args_expr->children; - if (function_args.size() != 2) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value defined argument"); - - auto arg_name = function_args[0]->as()->name(); - + const auto & [arg_name, arg_value] = getKeyValueMongoDBArgument(ast_func); if (arg_name == "structure") - structure = checkAndGetLiteralArgument(function_args[1], "structure"); + structure = checkAndGetLiteralArgument(arg_value, arg_name); else if (arg_name == "options") - main_arguments.push_back(function_args[1]); + main_arguments.push_back(arg_value); } else if (i == 5) { @@ -117,15 +111,11 @@ void TableFunctionMongoDB::parseArguments(const ASTPtr & ast_function, ContextPt { if (const auto * ast_func = typeid_cast(args[i].get())) { - const auto * args_expr = assert_cast(ast_func->arguments.get()); - auto function_args = args_expr->children; - if (function_args.size() != 2) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value defined argument"); - - auto arg_name = function_args[0]->as()->name(); - + const auto & [arg_name, arg_value] = getKeyValueMongoDBArgument(ast_func); if (arg_name == "structure") - structure = checkAndGetLiteralArgument(function_args[1], "structure"); + structure = checkAndGetLiteralArgument(arg_value, arg_name); + else if (arg_name == "options") + main_arguments.push_back(arg_value); } else if (i == 2) { @@ -145,6 +135,20 @@ void TableFunctionMongoDB::parseArguments(const ASTPtr & ast_function, ContextPt } +std::pair getKeyValueMongoDBArgument(const ASTFunction * ast_func) +{ + const auto * args_expr = assert_cast(ast_func->arguments.get()); + const auto & function_args = args_expr->children; + if (function_args.size() != 2 || ast_func->name != "equals" || !function_args[0]->as()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value defined argument, got {}", ast_func->formatForErrorMessage()); + + const auto & arg_name = function_args[0]->as()->name(); + if (arg_name == "structure" || arg_name == "options") + return std::make_pair(arg_name, function_args[1]); + + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value defined argument, got {}", ast_func->formatForErrorMessage()); +} + void registerTableFunctionMongoDB(TableFunctionFactory & factory) { factory.registerFunction( diff --git a/src/TableFunctions/TableFunctionMongoDB.h b/src/TableFunctions/TableFunctionMongoDB.h new file mode 100644 index 00000000000..2ab8ee9479f --- /dev/null +++ b/src/TableFunctions/TableFunctionMongoDB.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +#include +#include +#include + + +namespace DB +{ + +std::pair getKeyValueMongoDBArgument(const ASTFunction * ast_func); + +} + diff --git a/src/TableFunctions/TableFunctionMongoDBPocoLegacy.cpp b/src/TableFunctions/TableFunctionMongoDBPocoLegacy.cpp index dc1df7fcad8..4e27fd35e12 100644 --- a/src/TableFunctions/TableFunctionMongoDBPocoLegacy.cpp +++ b/src/TableFunctions/TableFunctionMongoDBPocoLegacy.cpp @@ -15,6 +15,7 @@ #include #include #include +#include namespace DB @@ -97,17 +98,11 @@ void TableFunctionMongoDBPocoLegacy::parseArguments(const ASTPtr & ast_function, { if (const auto * ast_func = typeid_cast(args[i].get())) { - const auto * args_expr = assert_cast(ast_func->arguments.get()); - auto function_args = args_expr->children; - if (function_args.size() != 2) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value defined argument"); - - auto arg_name = function_args[0]->as()->name(); - + const auto & [arg_name, arg_value] = getKeyValueMongoDBArgument(ast_func); if (arg_name == "structure") - structure = checkAndGetLiteralArgument(function_args[1], "structure"); + structure = checkAndGetLiteralArgument(arg_value, "structure"); else if (arg_name == "options") - main_arguments.push_back(function_args[1]); + main_arguments.push_back(arg_value); } else if (i == 5) { diff --git a/src/TableFunctions/TableFunctionObjectStorage.cpp b/src/TableFunctions/TableFunctionObjectStorage.cpp index 8559a4cd668..12de08afad0 100644 --- a/src/TableFunctions/TableFunctionObjectStorage.cpp +++ b/src/TableFunctions/TableFunctionObjectStorage.cpp @@ -117,8 +117,9 @@ StoragePtr TableFunctionObjectStorage::executeImpl( columns, ConstraintsDescription{}, String{}, - /* format_settings */std::nullopt, - /* distributed_processing */false, + /* format_settings */ std::nullopt, + /* mode */ LoadingStrictnessLevel::CREATE, + /* distributed_processing */ false, nullptr); storage->startup(); @@ -224,4 +225,87 @@ template class TableFunctionObjectStorage; #endif template class TableFunctionObjectStorage; + +#if USE_AVRO +void registerTableFunctionIceberg(TableFunctionFactory & factory) +{ +#if USE_AWS_S3 + factory.registerFunction( + {.documentation + = {.description = R"(The table function can be used to read the Iceberg table stored on S3 object store. Alias to icebergS3)", + .examples{{"iceberg", "SELECT * FROM iceberg(url, access_key_id, secret_access_key)", ""}}, + .categories{"DataLake"}}, + .allow_readonly = false}); + factory.registerFunction( + {.documentation + = {.description = R"(The table function can be used to read the Iceberg table stored on S3 object store.)", + .examples{{"icebergS3", "SELECT * FROM icebergS3(url, access_key_id, secret_access_key)", ""}}, + .categories{"DataLake"}}, + .allow_readonly = false}); + +#endif +#if USE_AZURE_BLOB_STORAGE + factory.registerFunction( + {.documentation + = {.description = R"(The table function can be used to read the Iceberg table stored on Azure object store.)", + .examples{{"icebergAzure", "SELECT * FROM icebergAzure(url, access_key_id, secret_access_key)", ""}}, + .categories{"DataLake"}}, + .allow_readonly = false}); +#endif +#if USE_HDFS + factory.registerFunction( + {.documentation + = {.description = R"(The table function can be used to read the Iceberg table stored on HDFS virtual filesystem.)", + .examples{{"icebergHDFS", "SELECT * FROM icebergHDFS(url)", ""}}, + .categories{"DataLake"}}, + .allow_readonly = false}); +#endif + factory.registerFunction( + {.documentation + = {.description = R"(The table function can be used to read the Iceberg table stored locally.)", + .examples{{"icebergLocal", "SELECT * FROM icebergLocal(filename)", ""}}, + .categories{"DataLake"}}, + .allow_readonly = false}); +} +#endif + + +#if USE_AWS_S3 +#if USE_PARQUET +void registerTableFunctionDeltaLake(TableFunctionFactory & factory) +{ + factory.registerFunction( + {.documentation + = {.description = R"(The table function can be used to read the DeltaLake table stored on object store.)", + .examples{{"deltaLake", "SELECT * FROM deltaLake(url, access_key_id, secret_access_key)", ""}}, + .categories{"DataLake"}}, + .allow_readonly = false}); +} +#endif + +void registerTableFunctionHudi(TableFunctionFactory & factory) +{ + factory.registerFunction( + {.documentation + = {.description = R"(The table function can be used to read the Hudi table stored on object store.)", + .examples{{"hudi", "SELECT * FROM hudi(url, access_key_id, secret_access_key)", ""}}, + .categories{"DataLake"}}, + .allow_readonly = false}); +} + +#endif + +void registerDataLakeTableFunctions(TableFunctionFactory & factory) +{ + UNUSED(factory); +#if USE_AVRO + registerTableFunctionIceberg(factory); +#endif +#if USE_AWS_S3 +#if USE_PARQUET + registerTableFunctionDeltaLake(factory); +#endif + registerTableFunctionHudi(factory); +#endif +} } diff --git a/src/TableFunctions/TableFunctionObjectStorage.h b/src/TableFunctions/TableFunctionObjectStorage.h index 6b923f93e75..19cd637bd80 100644 --- a/src/TableFunctions/TableFunctionObjectStorage.h +++ b/src/TableFunctions/TableFunctionObjectStorage.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -61,6 +62,48 @@ struct LocalDefinition static constexpr auto storage_type_name = "Local"; }; +struct IcebergDefinition +{ + static constexpr auto name = "iceberg"; + static constexpr auto storage_type_name = "S3"; +}; + +struct IcebergS3Definition +{ + static constexpr auto name = "icebergS3"; + static constexpr auto storage_type_name = "S3"; +}; + +struct IcebergAzureDefinition +{ + static constexpr auto name = "icebergAzure"; + static constexpr auto storage_type_name = "Azure"; +}; + +struct IcebergLocalDefinition +{ + static constexpr auto name = "icebergLocal"; + static constexpr auto storage_type_name = "Local"; +}; + +struct IcebergHDFSDefinition +{ + static constexpr auto name = "icebergHDFS"; + static constexpr auto storage_type_name = "HDFS"; +}; + +struct DeltaLakeDefinition +{ + static constexpr auto name = "deltaLake"; + static constexpr auto storage_type_name = "S3"; +}; + +struct HudiDefinition +{ + static constexpr auto name = "hudi"; + static constexpr auto storage_type_name = "S3"; +}; + template class TableFunctionObjectStorage : public ITableFunction { @@ -137,4 +180,25 @@ using TableFunctionHDFS = TableFunctionObjectStorage; + + +#if USE_AVRO +# if USE_AWS_S3 +using TableFunctionIceberg = TableFunctionObjectStorage; +using TableFunctionIcebergS3 = TableFunctionObjectStorage; +# endif +# if USE_AZURE_BLOB_STORAGE +using TableFunctionIcebergAzure = TableFunctionObjectStorage; +# endif +# if USE_HDFS +using TableFunctionIcebergHDFS = TableFunctionObjectStorage; +# endif +using TableFunctionIcebergLocal = TableFunctionObjectStorage; +#endif +#if USE_AWS_S3 +# if USE_PARQUET +using TableFunctionDeltaLake = TableFunctionObjectStorage; +# endif +using TableFunctionHudi = TableFunctionObjectStorage; +#endif } diff --git a/src/TableFunctions/TableFunctionObjectStorageCluster.cpp b/src/TableFunctions/TableFunctionObjectStorageCluster.cpp index 449bd2c8c49..5ca26aabe32 100644 --- a/src/TableFunctions/TableFunctionObjectStorageCluster.cpp +++ b/src/TableFunctions/TableFunctionObjectStorageCluster.cpp @@ -41,9 +41,10 @@ StoragePtr TableFunctionObjectStorageCluster::execute StorageID(Base::getDatabaseName(), table_name), columns, ConstraintsDescription{}, - /* comment */String{}, - /* format_settings */std::nullopt, /// No format_settings - /* distributed_processing */true, + /* comment */ String{}, + /* format_settings */ std::nullopt, /// No format_settings + /* mode */ LoadingStrictnessLevel::CREATE, + /* distributed_processing */ true, /*partition_by_=*/nullptr); } else diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index 2bdc0b449e0..8f4841a992b 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include diff --git a/src/TableFunctions/registerDataLakeTableFunctions.cpp b/src/TableFunctions/registerDataLakeTableFunctions.cpp deleted file mode 100644 index 63b09fcf9e7..00000000000 --- a/src/TableFunctions/registerDataLakeTableFunctions.cpp +++ /dev/null @@ -1,96 +0,0 @@ -#include -#include - -namespace DB -{ - -#if USE_AVRO -void registerTableFunctionIceberg(TableFunctionFactory & factory) -{ -# if USE_AWS_S3 - factory.registerFunction( - {.documentation - = {.description = R"(The table function can be used to read the Iceberg table stored on S3 object store. Alias to icebergS3)", - .examples{{"iceberg", "SELECT * FROM iceberg(url, access_key_id, secret_access_key)", ""}}, - .categories{"DataLake"}}, - .allow_readonly = false}); - factory.registerFunction( - {.documentation - = {.description = R"(The table function can be used to read the Iceberg table stored on S3 object store.)", - .examples{{"icebergS3", "SELECT * FROM icebergS3(url, access_key_id, secret_access_key)", ""}}, - .categories{"DataLake"}}, - .allow_readonly = false}); - -# endif -# if USE_AZURE_BLOB_STORAGE - factory.registerFunction( - {.documentation - = {.description = R"(The table function can be used to read the Iceberg table stored on Azure object store.)", - .examples{{"icebergAzure", "SELECT * FROM icebergAzure(url, access_key_id, secret_access_key)", ""}}, - .categories{"DataLake"}}, - .allow_readonly = false}); -# endif -# if USE_HDFS - factory.registerFunction( - {.documentation - = {.description = R"(The table function can be used to read the Iceberg table stored on HDFS object store.)", - .examples{{"icebergHDFS", "SELECT * FROM icebergHDFS(url)", ""}}, - .categories{"DataLake"}}, - .allow_readonly = false}); -# endif - factory.registerFunction( - {.documentation - = {.description = R"(The table function can be used to read the Iceberg table stored locally.)", - .examples{{"icebergLocal", "SELECT * FROM icebergLocal(filename)", ""}}, - .categories{"DataLake"}}, - .allow_readonly = false}); -} -#endif - -#if USE_AWS_S3 -# if USE_PARQUET -void registerTableFunctionDeltaLake(TableFunctionFactory & factory) -{ - factory.registerFunction( - { - .documentation = - { - .description=R"(The table function can be used to read the DeltaLake table stored on object store.)", - .examples{{"deltaLake", "SELECT * FROM deltaLake(url, access_key_id, secret_access_key)", ""}}, - .categories{"DataLake"} - }, - .allow_readonly = false - }); -} -#endif - -void registerTableFunctionHudi(TableFunctionFactory & factory) -{ - factory.registerFunction( - { - .documentation = - { - .description=R"(The table function can be used to read the Hudi table stored on object store.)", - .examples{{"hudi", "SELECT * FROM hudi(url, access_key_id, secret_access_key)", ""}}, - .categories{"DataLake"} - }, - .allow_readonly = false - }); -} -#endif - -void registerDataLakeTableFunctions(TableFunctionFactory & factory) -{ - UNUSED(factory); -#if USE_AVRO - registerTableFunctionIceberg(factory); -#endif -#if USE_AWS_S3 -# if USE_PARQUET - registerTableFunctionDeltaLake(factory); -#endif - registerTableFunctionHudi(factory); -#endif -} - -} diff --git a/tests/ci/cherry_pick.py b/tests/ci/cherry_pick.py index a796f63de6c..ca32d5bc24c 100644 --- a/tests/ci/cherry_pick.py +++ b/tests/ci/cherry_pick.py @@ -34,8 +34,9 @@ from typing import List, Optional import __main__ +from ci_buddy import CIBuddy from ci_config import Labels -from env_helper import TEMP_PATH +from env_helper import IS_CI, TEMP_PATH from get_robot_token import get_best_robot_token from git_helper import GIT_PREFIX, git_runner, is_shallow from github_helper import GitHub, PullRequest, PullRequests, Repository @@ -97,7 +98,7 @@ close it. self.pr = pr self.repo = repo - self.cherrypick_branch = f"cherrypick/{name}/{pr.merge_commit_sha}" + self.cherrypick_branch = f"cherrypick/{name}/{pr.number}" self.backport_branch = f"backport/{name}/{pr.number}" self.cherrypick_pr = None # type: Optional[PullRequest] self.backport_pr = None # type: Optional[PullRequest] @@ -653,6 +654,14 @@ def main(): bp.process_backports() if bp.error is not None: logging.error("Finished successfully, but errors occurred!") + if IS_CI: + ci_buddy = CIBuddy() + ci_buddy.post_job_error( + f"The cherry-pick finished with errors: {bp.error}", + with_instance_info=True, + with_wf_link=True, + critical=True, + ) raise bp.error diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 10431ce038f..eb0fe7c85c3 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -798,10 +798,6 @@ def _upload_build_profile_data( logging.info("Unknown CI logs host, skip uploading build profile data") return - if not pr_info.number == 0: - logging.info("Skipping uploading build profile data for PRs") - return - instance_type = get_instance_type() instance_id = get_instance_id() auth = { @@ -1268,6 +1264,7 @@ def main() -> int: s3, pr_info.number, pr_info.sha, + pr_info.head_ref, job_report.test_results, job_report.additional_files, job_report.check_name or _get_ext_check_name(args.job_name), @@ -1335,6 +1332,7 @@ def main() -> int: s3, pr_info.number, pr_info.sha, + pr_info.head_ref, job_report.test_results, job_report.additional_files, job_report.check_name or _get_ext_check_name(args.job_name), diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py index 164af72f4be..07b748180cd 100644 --- a/tests/ci/ci_buddy.py +++ b/tests/ci/ci_buddy.py @@ -3,14 +3,13 @@ import json import os from typing import Dict, List, Union -import boto3 import requests from botocore.exceptions import ClientError from ci_config import CI from ci_utils import WithIter from commit_status_helper import get_commit_filtered_statuses, get_repo -from get_robot_token import get_best_robot_token +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from github_helper import GitHub from pr_info import PRInfo @@ -89,15 +88,9 @@ class CIBuddy: def _get_webhooks(): name = "ci_buddy_web_hooks" - session = boto3.Session(region_name="us-east-1") # Replace with your region - ssm_client = session.client("ssm") json_string = None try: - response = ssm_client.get_parameter( - Name=name, - WithDecryption=True, # Set to True if the parameter is a SecureString - ) - json_string = response["Parameter"]["Value"] + json_string = get_parameter_from_ssm(name, decrypt=True) except ClientError as e: print(f"An error occurred: {e}") diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py index 6f2e3e70736..c271339db8b 100644 --- a/tests/ci/ci_cache.py +++ b/tests/ci/ci_cache.py @@ -795,11 +795,12 @@ class CiCache: # start waiting for the next TIMEOUT seconds if there are more than X(=4) jobs to wait # wait TIMEOUT seconds in rounds. Y(=5) is the max number of rounds expired_sec = 0 - start_at = int(time.time()) + start_at = time.time() while expired_sec < TIMEOUT and self.jobs_to_wait: await_finished: Set[str] = set() if not dry_run: - time.sleep(poll_interval_sec) + # Do not sleep longer than required + time.sleep(min(poll_interval_sec, TIMEOUT - expired_sec)) self.update() for job_name, job_config in self.jobs_to_wait.items(): num_batches = job_config.num_batches @@ -844,10 +845,12 @@ class CiCache: del self.jobs_to_wait[job] if not dry_run: - expired_sec = int(time.time()) - start_at - print( - f"...awaiting continues... seconds left [{TIMEOUT - expired_sec}]" - ) + expired_sec = int(time.time() - start_at) + msg = f"...awaiting continues... seconds left [{TIMEOUT - expired_sec}]" + if expired_sec >= TIMEOUT: + # Avoid `seconds left [-3]` + msg = f"awaiting for round {round_cnt} is finished" + print(msg) else: # make up for 2 iterations in dry_run expired_sec += int(TIMEOUT / 2) + 1 diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 9f5d5f1983d..67cdbbdcf6d 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -51,11 +51,11 @@ class CI: TAG_CONFIGS = { Tags.DO_NOT_TEST_LABEL: LabelConfig(run_jobs=[JobNames.STYLE_CHECK]), - Tags.CI_SET_ARM: LabelConfig( + Tags.CI_SET_AARCH64: LabelConfig( run_jobs=[ JobNames.STYLE_CHECK, BuildNames.PACKAGE_AARCH64, - JobNames.INTEGRATION_TEST_ARM, + JobNames.INTEGRATION_TEST_AARCH64, ] ), Tags.CI_SET_REQUIRED: LabelConfig( @@ -95,16 +95,16 @@ class CI: static_binary_name="aarch64", additional_pkgs=True, ), - runner_type=Runners.BUILDER_ARM, + runner_type=Runners.BUILDER_AARCH64, ), - BuildNames.PACKAGE_ARM_ASAN: CommonJobConfigs.BUILD.with_properties( + BuildNames.PACKAGE_AARCH64_ASAN: CommonJobConfigs.BUILD.with_properties( build_config=BuildConfig( - name=BuildNames.PACKAGE_ARM_ASAN, + name=BuildNames.PACKAGE_AARCH64_ASAN, compiler="clang-18-aarch64", sanitizer="address", package_type="deb", ), - runner_type=Runners.BUILDER_ARM, + runner_type=Runners.BUILDER_AARCH64, ), BuildNames.PACKAGE_ASAN: CommonJobConfigs.BUILD.with_properties( build_config=BuildConfig( @@ -276,16 +276,16 @@ class CI: JobNames.INSTALL_TEST_AMD: CommonJobConfigs.INSTALL_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE] ), - JobNames.INSTALL_TEST_ARM: CommonJobConfigs.INSTALL_TEST.with_properties( + JobNames.INSTALL_TEST_AARCH64: CommonJobConfigs.INSTALL_TEST.with_properties( required_builds=[BuildNames.PACKAGE_AARCH64], - runner_type=Runners.STYLE_CHECKER_ARM, + runner_type=Runners.STYLE_CHECKER_AARCH64, ), JobNames.STATEFUL_TEST_ASAN: CommonJobConfigs.STATEFUL_TEST.with_properties( required_builds=[BuildNames.PACKAGE_ASAN] ), - JobNames.STATEFUL_TEST_ARM_ASAN: CommonJobConfigs.STATEFUL_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_ARM_ASAN], - runner_type=Runners.FUNC_TESTER_ARM, + JobNames.STATEFUL_TEST_AARCH64_ASAN: CommonJobConfigs.STATEFUL_TEST.with_properties( + required_builds=[BuildNames.PACKAGE_AARCH64_ASAN], + runner_type=Runners.FUNC_TESTER_AARCH64, ), JobNames.STATEFUL_TEST_TSAN: CommonJobConfigs.STATEFUL_TEST.with_properties( required_builds=[BuildNames.PACKAGE_TSAN] @@ -307,7 +307,7 @@ class CI: ), JobNames.STATEFUL_TEST_AARCH64: CommonJobConfigs.STATEFUL_TEST.with_properties( required_builds=[BuildNames.PACKAGE_AARCH64], - runner_type=Runners.FUNC_TESTER_ARM, + runner_type=Runners.FUNC_TESTER_AARCH64, ), JobNames.STATEFUL_TEST_PARALLEL_REPL_RELEASE: CommonJobConfigs.STATEFUL_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE] @@ -335,10 +335,10 @@ class CI: JobNames.STATELESS_TEST_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_ASAN], num_batches=2 ), - JobNames.STATELESS_TEST_ARM_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_ARM_ASAN], + JobNames.STATELESS_TEST_AARCH64_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties( + required_builds=[BuildNames.PACKAGE_AARCH64_ASAN], num_batches=2, - runner_type=Runners.FUNC_TESTER_ARM, + runner_type=Runners.FUNC_TESTER_AARCH64, ), JobNames.STATELESS_TEST_TSAN: CommonJobConfigs.STATELESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_TSAN], num_batches=4 @@ -360,7 +360,7 @@ class CI: ), JobNames.STATELESS_TEST_AARCH64: CommonJobConfigs.STATELESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_AARCH64], - runner_type=Runners.FUNC_TESTER_ARM, + runner_type=Runners.FUNC_TESTER_AARCH64, ), JobNames.STATELESS_TEST_OLD_ANALYZER_S3_REPLICATED_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=2 @@ -432,10 +432,10 @@ class CI: num_batches=6, timeout=9000, # the job timed out with default value (7200) ), - JobNames.INTEGRATION_TEST_ARM: CommonJobConfigs.INTEGRATION_TEST.with_properties( + JobNames.INTEGRATION_TEST_AARCH64: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_AARCH64], num_batches=6, - runner_type=Runners.FUNC_TESTER_ARM, + runner_type=Runners.FUNC_TESTER_AARCH64, ), JobNames.INTEGRATION_TEST: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE], @@ -453,10 +453,10 @@ class CI: required_builds=[BuildNames.PACKAGE_RELEASE], required_on_release_branch=True, ), - JobNames.COMPATIBILITY_TEST_ARM: CommonJobConfigs.COMPATIBILITY_TEST.with_properties( + JobNames.COMPATIBILITY_TEST_AARCH64: CommonJobConfigs.COMPATIBILITY_TEST.with_properties( required_builds=[BuildNames.PACKAGE_AARCH64], required_on_release_branch=True, - runner_type=Runners.STYLE_CHECKER_ARM, + runner_type=Runners.STYLE_CHECKER_AARCH64, ), JobNames.UNIT_TEST: CommonJobConfigs.UNIT_TEST.with_properties( required_builds=[BuildNames.BINARY_RELEASE], @@ -499,22 +499,22 @@ class CI: required_builds=[BuildNames.BINARY_RELEASE], run_by_labels=[Labels.JEPSEN_TEST], run_command="jepsen_check.py keeper", - runner_type=Runners.STYLE_CHECKER_ARM, + runner_type=Runners.STYLE_CHECKER_AARCH64, ), JobNames.JEPSEN_SERVER: JobConfig( required_builds=[BuildNames.BINARY_RELEASE], run_by_labels=[Labels.JEPSEN_TEST], run_command="jepsen_check.py server", - runner_type=Runners.STYLE_CHECKER_ARM, + runner_type=Runners.STYLE_CHECKER_AARCH64, ), JobNames.PERFORMANCE_TEST_AMD64: CommonJobConfigs.PERF_TESTS.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=4 ), - JobNames.PERFORMANCE_TEST_ARM64: CommonJobConfigs.PERF_TESTS.with_properties( + JobNames.PERFORMANCE_TEST_AARCH64: CommonJobConfigs.PERF_TESTS.with_properties( required_builds=[BuildNames.PACKAGE_AARCH64], num_batches=4, run_by_labels=[Labels.PR_PERFORMANCE], - runner_type=Runners.FUNC_TESTER_ARM, + runner_type=Runners.FUNC_TESTER_AARCH64, ), JobNames.SQLANCER: CommonJobConfigs.SQLLANCER_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE], @@ -532,16 +532,16 @@ class CI: JobNames.CLICKBENCH_TEST: CommonJobConfigs.CLICKBENCH_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE], ), - JobNames.CLICKBENCH_TEST_ARM: CommonJobConfigs.CLICKBENCH_TEST.with_properties( + JobNames.CLICKBENCH_TEST_AARCH64: CommonJobConfigs.CLICKBENCH_TEST.with_properties( required_builds=[BuildNames.PACKAGE_AARCH64], - runner_type=Runners.FUNC_TESTER_ARM, + runner_type=Runners.FUNC_TESTER_AARCH64, ), JobNames.LIBFUZZER_TEST: JobConfig( required_builds=[BuildNames.FUZZERS], run_by_labels=[Tags.libFuzzer], - timeout=10800, + timeout=5400, run_command='libfuzzer_test_check.py "$CHECK_NAME"', - runner_type=Runners.STYLE_CHECKER, + runner_type=Runners.FUNC_TESTER, ), JobNames.DOCKER_SERVER: CommonJobConfigs.DOCKER_SERVER.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE, BuildNames.PACKAGE_AARCH64] @@ -572,7 +572,7 @@ class CI: ), JobNames.STYLE_CHECK: JobConfig( run_always=True, - runner_type=Runners.STYLE_CHECKER_ARM, + runner_type=Runners.STYLE_CHECKER_AARCH64, ), JobNames.BUGFIX_VALIDATE: JobConfig( run_by_labels=[Labels.PR_BUGFIX, Labels.PR_CRITICAL_BUGFIX], diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py index dd86dc320c2..fb3e55fdbe3 100644 --- a/tests/ci/ci_definitions.py +++ b/tests/ci/ci_definitions.py @@ -58,11 +58,11 @@ class Runners(metaclass=WithIter): """ BUILDER = "builder" - BUILDER_ARM = "builder-aarch64" + BUILDER_AARCH64 = "builder-aarch64" STYLE_CHECKER = "style-checker" - STYLE_CHECKER_ARM = "style-checker-aarch64" + STYLE_CHECKER_AARCH64 = "style-checker-aarch64" FUNC_TESTER = "func-tester" - FUNC_TESTER_ARM = "func-tester-aarch64" + FUNC_TESTER_AARCH64 = "func-tester-aarch64" FUZZER_UNIT_TESTER = "fuzzer-unit-tester" @@ -78,7 +78,7 @@ class Tags(metaclass=WithIter): # to upload all binaries from build jobs UPLOAD_ALL_ARTIFACTS = "upload_all" CI_SET_SYNC = "ci_set_sync" - CI_SET_ARM = "ci_set_arm" + CI_SET_AARCH64 = "ci_set_aarch64" CI_SET_REQUIRED = "ci_set_required" CI_SET_BUILDS = "ci_set_builds" @@ -106,7 +106,7 @@ class BuildNames(metaclass=WithIter): PACKAGE_MSAN = "package_msan" PACKAGE_DEBUG = "package_debug" PACKAGE_AARCH64 = "package_aarch64" - PACKAGE_ARM_ASAN = "package_aarch64_asan" + PACKAGE_AARCH64_ASAN = "package_aarch64_asan" PACKAGE_RELEASE_COVERAGE = "package_release_coverage" BINARY_RELEASE = "binary_release" BINARY_TIDY = "binary_tidy" @@ -134,14 +134,14 @@ class JobNames(metaclass=WithIter): DOCKER_SERVER = "Docker server image" DOCKER_KEEPER = "Docker keeper image" INSTALL_TEST_AMD = "Install packages (release)" - INSTALL_TEST_ARM = "Install packages (aarch64)" + INSTALL_TEST_AARCH64 = "Install packages (aarch64)" STATELESS_TEST_DEBUG = "Stateless tests (debug)" STATELESS_TEST_RELEASE = "Stateless tests (release)" STATELESS_TEST_RELEASE_COVERAGE = "Stateless tests (coverage)" STATELESS_TEST_AARCH64 = "Stateless tests (aarch64)" STATELESS_TEST_ASAN = "Stateless tests (asan)" - STATELESS_TEST_ARM_ASAN = "Stateless tests (aarch64, asan)" + STATELESS_TEST_AARCH64_ASAN = "Stateless tests (aarch64, asan)" STATELESS_TEST_TSAN = "Stateless tests (tsan)" STATELESS_TEST_MSAN = "Stateless tests (msan)" STATELESS_TEST_UBSAN = "Stateless tests (ubsan)" @@ -158,7 +158,7 @@ class JobNames(metaclass=WithIter): STATEFUL_TEST_RELEASE_COVERAGE = "Stateful tests (coverage)" STATEFUL_TEST_AARCH64 = "Stateful tests (aarch64)" STATEFUL_TEST_ASAN = "Stateful tests (asan)" - STATEFUL_TEST_ARM_ASAN = "Stateful tests (aarch64, asan)" + STATEFUL_TEST_AARCH64_ASAN = "Stateful tests (aarch64, asan)" STATEFUL_TEST_TSAN = "Stateful tests (tsan)" STATEFUL_TEST_MSAN = "Stateful tests (msan)" STATEFUL_TEST_UBSAN = "Stateful tests (ubsan)" @@ -181,7 +181,7 @@ class JobNames(metaclass=WithIter): INTEGRATION_TEST_ASAN = "Integration tests (asan)" INTEGRATION_TEST_ASAN_OLD_ANALYZER = "Integration tests (asan, old analyzer)" INTEGRATION_TEST_TSAN = "Integration tests (tsan)" - INTEGRATION_TEST_ARM = "Integration tests (aarch64)" + INTEGRATION_TEST_AARCH64 = "Integration tests (aarch64)" INTEGRATION_TEST_FLAKY = "Integration tests flaky check (asan)" UPGRADE_TEST_DEBUG = "Upgrade check (debug)" @@ -205,7 +205,7 @@ class JobNames(metaclass=WithIter): JEPSEN_SERVER = "ClickHouse Server Jepsen" PERFORMANCE_TEST_AMD64 = "Performance Comparison (release)" - PERFORMANCE_TEST_ARM64 = "Performance Comparison (aarch64)" + PERFORMANCE_TEST_AARCH64 = "Performance Comparison (aarch64)" # SQL_LOGIC_TEST = "Sqllogic test (release)" @@ -214,10 +214,10 @@ class JobNames(metaclass=WithIter): SQLTEST = "SQLTest" COMPATIBILITY_TEST = "Compatibility check (release)" - COMPATIBILITY_TEST_ARM = "Compatibility check (aarch64)" + COMPATIBILITY_TEST_AARCH64 = "Compatibility check (aarch64)" CLICKBENCH_TEST = "ClickBench (release)" - CLICKBENCH_TEST_ARM = "ClickBench (aarch64)" + CLICKBENCH_TEST_AARCH64 = "ClickBench (aarch64)" LIBFUZZER_TEST = "libFuzzer tests" @@ -387,7 +387,7 @@ class CommonJobConfigs: "./tests/ci/upload_result_helper.py", ], ), - runner_type=Runners.STYLE_CHECKER_ARM, + runner_type=Runners.STYLE_CHECKER_AARCH64, disable_await=True, ) COMPATIBILITY_TEST = JobConfig( @@ -634,8 +634,8 @@ REQUIRED_CHECKS = [ JobNames.STATEFUL_TEST_RELEASE, JobNames.STATELESS_TEST_RELEASE, JobNames.STATELESS_TEST_ASAN, - JobNames.STATELESS_TEST_ARM_ASAN, - JobNames.STATEFUL_TEST_ARM_ASAN, + JobNames.STATELESS_TEST_AARCH64_ASAN, + JobNames.STATEFUL_TEST_AARCH64_ASAN, JobNames.STATELESS_TEST_FLAKY_ASAN, JobNames.STATEFUL_TEST_ASAN, JobNames.STYLE_CHECK, diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index 99f4778fe8f..32e7f6f6a53 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional import requests +from env_helper import GITHUB_REPOSITORY from get_robot_token import get_parameter_from_ssm from pr_info import PRInfo from report import TestResults @@ -211,17 +212,13 @@ def prepare_tests_results_for_clickhouse( report_url: str, check_name: str, ) -> List[dict]: - pull_request_url = "https://github.com/ClickHouse/ClickHouse/commits/master" - base_ref = "master" - head_ref = "master" - base_repo = pr_info.repo_full_name - head_repo = pr_info.repo_full_name + base_ref = pr_info.base_ref + base_repo = pr_info.base_name + head_ref = pr_info.head_ref + head_repo = pr_info.head_name + pull_request_url = f"https://github.com/{GITHUB_REPOSITORY}/commits/{head_ref}" if pr_info.number != 0: pull_request_url = pr_info.pr_html_url - base_ref = pr_info.base_ref - base_repo = pr_info.base_name - head_ref = pr_info.head_ref - head_repo = pr_info.head_name common_properties = { "pull_request_number": pr_info.number, diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index 28323b6786c..ce9eaf78e75 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -315,7 +315,13 @@ def create_ci_report(pr_info: PRInfo, statuses: CommitStatuses) -> str: ) ) return upload_results( - S3Helper(), pr_info.number, pr_info.sha, test_results, [], CI.StatusNames.CI + S3Helper(), + pr_info.number, + pr_info.sha, + pr_info.head_ref, + test_results, + [], + CI.StatusNames.CI, ) diff --git a/tests/ci/compatibility_check.py b/tests/ci/compatibility_check.py index bb0c717160e..38fb2eceb28 100644 --- a/tests/ci/compatibility_check.py +++ b/tests/ci/compatibility_check.py @@ -131,7 +131,7 @@ def main(): check_name = args.check_name or os.getenv("CHECK_NAME") assert check_name check_glibc = True - # currently hardcoded to x86, don't enable for ARM + # currently hardcoded to x86, don't enable for AARCH64 check_distributions = ( "aarch64" not in check_name.lower() and "arm64" not in check_name.lower() ) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index c8dbcd10245..c94aff1ab4d 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -250,7 +250,9 @@ def main(): s3_helper = S3Helper() pr_info = PRInfo() - url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [], NAME) + url = upload_results( + s3_helper, pr_info.number, pr_info.sha, pr_info.head_ref, test_results, [], NAME + ) print(f"::notice ::Report url: {url}") diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py index 6c6a88330ea..772cbbf1b02 100644 --- a/tests/ci/docker_manifests_merge.py +++ b/tests/ci/docker_manifests_merge.py @@ -183,7 +183,9 @@ def main(): pr_info = PRInfo() s3_helper = S3Helper() - url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [], NAME) + url = upload_results( + s3_helper, pr_info.number, pr_info.sha, pr_info.head_ref, test_results, [], NAME + ) print(f"::notice ::Report url: {url}") diff --git a/tests/ci/libfuzzer_test_check.py b/tests/ci/libfuzzer_test_check.py index 8f19dd7d023..2616fbe3f5d 100644 --- a/tests/ci/libfuzzer_test_check.py +++ b/tests/ci/libfuzzer_test_check.py @@ -3,20 +3,37 @@ import argparse import logging import os +import re import sys import zipfile from pathlib import Path from typing import List +from botocore.exceptions import ClientError + from build_download_helper import download_fuzzers from clickhouse_helper import CiLogsCredentials from docker_images_helper import DockerImage, get_docker_image, pull_image -from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH +from env_helper import REPO_COPY, REPORT_PATH, S3_BUILDS_BUCKET, TEMP_PATH from pr_info import PRInfo +from report import FAILURE, SUCCESS, JobReport, TestResult +from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen +TIMEOUT = 60 NO_CHANGES_MSG = "Nothing to run" +s3 = S3Helper() + + +def zipdir(path, ziph): + # ziph is zipfile handle + for root, _, files in os.walk(path): + for file in files: + ziph.write( + os.path.join(root, file), + os.path.relpath(os.path.join(root, file), os.path.join(path, "..")), + ) def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): @@ -59,16 +76,19 @@ def get_run_command( envs = [ # a static link, don't use S3_URL or S3_DOWNLOAD - '-e S3_URL="https://s3.amazonaws.com/clickhouse-datasets"', + '-e S3_URL="https://s3.amazonaws.com"', ] envs += [f"-e {e}" for e in additional_envs] env_str = " ".join(envs) + uid = os.getuid() + gid = os.getgid() return ( f"docker run " f"{ci_logs_args} " + f"--user {uid}:{gid} " f"--workdir=/fuzzers " f"--volume={fuzzers_path}:/fuzzers " f"--volume={repo_path}/tests:/usr/share/clickhouse-test " @@ -85,6 +105,115 @@ def parse_args(): return parser.parse_args() +def download_corpus(path: str): + logging.info("Download corpus...") + + try: + s3.download_file( + bucket=S3_BUILDS_BUCKET, + s3_path="fuzzer/corpus.zip", + local_file_path=path, + ) + except ClientError as e: + if e.response["Error"]["Code"] == "NoSuchKey": + logging.debug("No active corpus exists") + else: + raise + + with zipfile.ZipFile(f"{path}/corpus.zip", "r") as zipf: + zipf.extractall(path) + os.remove(f"{path}/corpus.zip") + + units = 0 + for _, _, files in os.walk(path): + units += len(files) + + logging.info("...downloaded %d units", units) + + +def upload_corpus(path: str): + with zipfile.ZipFile(f"{path}/corpus.zip", "w", zipfile.ZIP_DEFLATED) as zipf: + zipdir(f"{path}/corpus/", zipf) + s3.upload_file( + bucket=S3_BUILDS_BUCKET, + file_path=f"{path}/corpus.zip", + s3_path="fuzzer/corpus.zip", + ) + + +def process_error(path: Path) -> list: + ERROR = r"^==\d+==\s?ERROR: (\S+): (.*)" + # error_source = "" + # error_reason = "" + # test_unit = "" + # TEST_UNIT_LINE = r"artifact_prefix='.*\/'; Test unit written to (.*)" + error_info = [] + is_error = False + + with open(path, "r", encoding="utf-8") as file: + for line in file: + line = line.rstrip("\n") + if is_error: + error_info.append(line) + # match = re.search(TEST_UNIT_LINE, line) + # if match: + # test_unit = match.group(1) + continue + + match = re.search(ERROR, line) + if match: + error_info.append(line) + # error_source = match.group(1) + # error_reason = match.group(2) + is_error = True + + return error_info + + +def read_status(status_path: Path): + result = [] + with open(status_path, "r", encoding="utf-8") as file: + for line in file: + result.append(line.rstrip("\n")) + return result + + +def process_results(result_path: Path): + test_results = [] + oks = 0 + errors = 0 + fails = 0 + for file in result_path.glob("*.status"): + fuzzer = file.stem + file_path = file.parent / fuzzer + file_path_unit = file_path.with_suffix(".unit") + file_path_out = file_path.with_suffix(".out") + file_path_stdout = file_path.with_suffix(".stdout") + status = read_status(file) + result = TestResult(fuzzer, status[0], float(status[2])) + if status[0] == "OK": + oks += 1 + elif status[0] == "ERROR": + errors += 1 + if file_path_out.exists(): + result.set_log_files(f"['{file_path_out}']") + elif file_path_stdout.exists(): + result.set_log_files(f"['{file_path_stdout}']") + else: + fails += 1 + if file_path_out.exists(): + result.set_raw_logs("\n".join(process_error(file_path_out))) + if file_path_unit.exists(): + result.set_log_files(f"['{file_path_unit}']") + elif file_path_out.exists(): + result.set_log_files(f"['{file_path_out}']") + elif file_path_stdout.exists(): + result.set_log_files(f"['{file_path_stdout}']") + test_results.append(result) + + return [oks, errors, fails, test_results] + + def main(): logging.basicConfig(level=logging.INFO) @@ -114,15 +243,18 @@ def main(): fuzzers_path = temp_path / "fuzzers" fuzzers_path.mkdir(parents=True, exist_ok=True) + download_corpus(fuzzers_path) download_fuzzers(check_name, reports_path, fuzzers_path) for file in os.listdir(fuzzers_path): if file.endswith("_fuzzer"): os.chmod(fuzzers_path / file, 0o777) elif file.endswith("_seed_corpus.zip"): - corpus_path = fuzzers_path / (file.removesuffix("_seed_corpus.zip") + ".in") + seed_corpus_path = fuzzers_path / ( + file.removesuffix("_seed_corpus.zip") + ".in" + ) with zipfile.ZipFile(fuzzers_path / file, "r") as zfd: - zfd.extractall(corpus_path) + zfd.extractall(seed_corpus_path) result_path = temp_path / "result_path" result_path.mkdir(parents=True, exist_ok=True) @@ -133,6 +265,8 @@ def main(): check_name, run_by_hash_num, run_by_hash_total ) + additional_envs.append(f"TIMEOUT={TIMEOUT}") + ci_logs_credentials = CiLogsCredentials(Path(temp_path) / "export-logs-config.sh") ci_logs_args = ci_logs_credentials.get_docker_arguments( pr_info, stopwatch.start_time_str, check_name @@ -152,10 +286,25 @@ def main(): retcode = process.wait() if retcode == 0: logging.info("Run successfully") + upload_corpus(fuzzers_path) else: logging.info("Run failed") - sys.exit(0) + results = process_results(result_path) + + success = results[1] == 0 and results[2] == 0 + + JobReport( + description=f"OK: {results[0]}, ERROR: {results[1]}, FAIL: {results[2]}", + test_results=results[3], + status=SUCCESS if success else FAILURE, + start_time=stopwatch.start_time_str, + duration=stopwatch.duration_seconds, + additional_files=[], + ).dump() + + if not success: + sys.exit(1) if __name__ == "__main__": diff --git a/tests/ci/mark_release_ready.py b/tests/ci/mark_release_ready.py index 7ffb3c9a89b..838961bd89f 100755 --- a/tests/ci/mark_release_ready.py +++ b/tests/ci/mark_release_ready.py @@ -9,9 +9,10 @@ from get_robot_token import get_best_robot_token from git_helper import commit as commit_arg from github_helper import GitHub from pr_info import PRInfo -from release import RELEASE_READY_STATUS from report import SUCCESS +RELEASE_READY_STATUS = "Ready for release" + def main(): parser = argparse.ArgumentParser( diff --git a/tests/ci/official_docker.py b/tests/ci/official_docker.py new file mode 100644 index 00000000000..f62002ee1d2 --- /dev/null +++ b/tests/ci/official_docker.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python +"""Plan: +- Create a PR with regenerated Dockerfiles for each release branch +- Generate `library definition file` as described in +https://github.com/docker-library/official-images/blob/master/README.md#library-definition-files +- Create a PR with it to https://github.com/docker-library/official-images, the file +name will be `library/clickhouse`""" + +import argparse +import logging +from dataclasses import dataclass +from os import getpid +from pathlib import Path +from pprint import pformat +from shlex import quote +from shutil import rmtree +from textwrap import fill +from typing import Dict, Iterable, List, Optional, Set + +from env_helper import GITHUB_REPOSITORY, IS_CI +from git_helper import GIT_PREFIX, Git, git_runner +from version_helper import ( + ClickHouseVersion, + get_supported_versions, + get_tagged_versions, + get_version_from_string, +) + +UBUNTU_NAMES = { + "20.04": "focal", + "22.04": "jammy", +} + +if not IS_CI: + GIT_PREFIX = "git" + +DOCKER_LIBRARY_REPOSITORY = "ClickHouse/docker-library" + +DOCKER_LIBRARY_NAME = {"server": "clickhouse"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="The script to handle tasks for docker-library/official-images", + ) + global_args = argparse.ArgumentParser(add_help=False) + global_args.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="set the script verbosity, could be used multiple", + ) + global_args.add_argument( + "--directory", + type=Path, + default=Path("docker/official"), + help="a relative to the reporitory root directory", + ) + global_args.add_argument( + "--image-type", + choices=["server", "keeper"], + default="server", + help="which image type to process", + ) + global_args.add_argument( + "--commit", + action="store_true", + help="if set, the directory `docker/official` will be staged and committed " + "after generating", + ) + dockerfile_glob = "Dockerfile.*" + global_args.add_argument( + "--dockerfile-glob", + default=dockerfile_glob, + help="a glob to collect Dockerfiles in the server of keeper directory", + ) + subparsers = parser.add_subparsers( + title="commands", dest="command", required=True, help="the command to run" + ) + parser_tree = subparsers.add_parser( + "generate-tree", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + help="generates directory `docker/official`", + parents=[global_args], + ) + parser_tree.add_argument( + "--min-version", + type=get_version_from_string, + default=None, + help="if not set, only currently supported versions will be used", + ) + parser_tree.add_argument( + "--docker-branch", + default="", + help="if set, the branch to get the content of `docker/server` directory. When " + "unset, the content of directories is taken from release tags", + ) + parser_tree.add_argument( + "--build-images", + action="store_true", + help="if set, the image will be built for each Dockerfile '--dockerfile-glob' " + "in the result dirs", + ) + parser_tree.add_argument("--clean", default=True, help=argparse.SUPPRESS) + parser_tree.add_argument( + "--no-clean", + dest="clean", + action="store_false", + default=argparse.SUPPRESS, + help="if set, the directory `docker/official` won't be cleaned " + "before generating", + ) + parser_tree.add_argument( + "--fetch-tags", + action="store_true", + help="if set, the tags will be updated before run", + ) + parser_ldf = subparsers.add_parser( + "generate-ldf", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + help="generate docker library definition file", + parents=[global_args], + ) + parser_ldf.add_argument("--check-changed", default=True, help=argparse.SUPPRESS) + parser_ldf.add_argument( + "--no-check-changed", + dest="check_changed", + action="store_false", + default=argparse.SUPPRESS, + help="if set, the directory `docker/official` won't be checked to be " + "uncommitted", + ) + args = parser.parse_args() + return args + + +def get_versions_greater(minimal: ClickHouseVersion) -> Set[ClickHouseVersion]: + "Get the latest patch version for each major.minor >= minimal" + supported = {} # type: Dict[str, ClickHouseVersion] + versions = get_tagged_versions() + for v in versions: + if v < minimal or not v.is_supported: + continue + txt = f"{v.major}.{v.minor}" + if txt in supported: + supported[txt] = max(v, supported[txt]) + continue + supported[txt] = v + return set(supported.values()) + + +def create_versions_dirs( + versions: Iterable[ClickHouseVersion], directory: Path +) -> Dict[ClickHouseVersion, Path]: + assert directory.is_dir() or not directory.exists() + dirs = {} + for v in versions: + version_dir = directory / str(v) + version_dir.mkdir(parents=True, exist_ok=True) + logging.debug("Directory %s for version %s is created", version_dir, v) + dirs[v] = version_dir + return dirs + + +def generate_docker_directories( + version_dirs: Dict[ClickHouseVersion, Path], + docker_branch: str, + dockerfile_glob: str, + build_images: bool = False, + image_type: str = "server", +) -> None: + arg_version = "ARG VERSION=" + start_filter = "#docker-official-library:off" + stop_filter = "#docker-official-library:on" + for version, directory in version_dirs.items(): + branch = docker_branch or version.describe + docker_source = f"{branch}:docker/{image_type}" + logging.debug( + "Unpack directory content from '%s' to %s", + docker_source, + directory, + ) + # We ignore README* files + git_runner( + f"git archive {docker_source} | tar --exclude='README*' -x -C {directory}" + ) + for df in directory.glob(dockerfile_glob): + original_content = df.read_text().splitlines() + content = [] + filtering = False + for line in original_content: + # Change the ARG VERSION= to a proper version + if line.startswith(arg_version): + logging.debug( + "Found '%s' in line %s:%s, setting to version %s", + arg_version, + df.name, + original_content.index(line), + version, + ) + content.append(f'{arg_version}"{version}"') + continue + # Filter out CI-related part from official docker + if line == start_filter: + filtering = True + continue + if line == stop_filter: + filtering = False + continue + if not filtering: + content.append(line) + + df.write_text("\n".join(content) + "\n") + if build_images: + git_runner( + f"docker build --network=host -t '{DOCKER_LIBRARY_NAME[image_type]}:" + f"{version}{df.suffix}' -f '{df}' --progress plain '{directory}'" + ) + + +def path_is_changed(path: Path) -> bool: + "checks if `path` has uncommitted changes" + logging.info("Checking if the path %s is changed", path) + path_dir = path.parent + return bool(git_runner(f"git -C {path_dir} status --porcelain -- '{path}'")) + + +def get_cmdline(width: int = 80) -> str: + "Returns the cmdline split by words with given maximum width" + cmdline = " ".join( + quote(arg) + for arg in Path(f"/proc/{getpid()}/cmdline") + .read_text(encoding="utf-8") + .split("\x00")[:-1] + ) + if width <= 2: + return cmdline + + return " \\\n".join( + fill( + cmdline, + break_long_words=False, + break_on_hyphens=False, + subsequent_indent=r" ", + width=width - 2, + ).split("\n") + ) + + +def generate_tree(args: argparse.Namespace) -> None: + if args.fetch_tags: + # Fetch all tags to not miss the latest versions + git_runner(f"{GIT_PREFIX} fetch --tags --no-recurse-submodules") + if args.min_version: + versions = get_versions_greater(args.min_version) + else: + versions = get_supported_versions() + logging.info( + "The versions to generate:\n %s", + "\n ".join(v.string for v in sorted(versions)), + ) + directory = Path(git_runner.cwd) / args.directory / args.image_type + if args.clean: + try: + logging.info("Removing directory %s before generating", directory) + rmtree(directory) + except FileNotFoundError: + pass + version_dirs = create_versions_dirs(versions, directory) + generate_docker_directories( + version_dirs, + args.docker_branch, + args.dockerfile_glob, + args.build_images, + args.image_type, + ) + if args.commit and path_is_changed(directory): + logging.info("Staging and committing content of %s", directory) + git_runner(f"git -C {directory} add {directory}") + commit_message = "\n".join( + ( + "Re-/Generated tags for official docker library", + "", + "The changed versions:", + *(f" {v.string}" for v in sorted(versions)), + f"The old images were removed: {args.clean}", + "The directory is generated and committed as following:", + f"{get_cmdline()}", + ) + ) + git_runner(f"{GIT_PREFIX} -C {directory} commit -F -", input=commit_message) + + +@dataclass +class TagAttrs: + """A mutable metadata to preserve between generating tags for different versions""" + + # Only one latest can exist + latest: ClickHouseVersion + # Only one can be a major one (the most fresh per a year) + majors: Dict[int, ClickHouseVersion] + # Only one lts version can exist + lts: Optional[ClickHouseVersion] + + +def ldf_header(git: Git, directory: Path) -> List[str]: + "Generates the header for LDF" + script_path = Path(__file__).relative_to(git.root) + script_sha = git_runner(f"git log -1 --format=format:%H -- {script_path}") + repo = f"https://github.com/{GITHUB_REPOSITORY}" + dl_repo = f"https://github.com/{DOCKER_LIBRARY_REPOSITORY}" + fetch_commit = git_runner( + f"git -C {directory} log -1 --format=format:%H -- {directory}" + ) + dl_branch = git_runner(f"git -C {directory} branch --show-current") + return [ + f"# The file is generated by {repo}/blob/{script_sha}/{script_path}", + "", + "Maintainers: Misha f. Shiryaev (@Felixoid),", + " Max Kainov (@mkaynov),", + " Alexander Sapin (@alesapin)", + f"GitRepo: {dl_repo}.git", + f"GitFetch: refs/heads/{dl_branch}", + f"GitCommit: {fetch_commit}", + ] + + +def ldf_tags(version: ClickHouseVersion, distro: str, tag_attrs: TagAttrs) -> str: + """returns the string 'Tags: coma, separated, tags'""" + tags = [] + # without_distro shows that it's the default tags set, without `-jammy` suffix + without_distro = distro in UBUNTU_NAMES.values() + if version == tag_attrs.latest: + if without_distro: + tags.append("latest") + tags.append(distro) + + # The current version gets the `lts` tag when it's the first met version.is_lts + with_lts = tag_attrs.lts in (None, version) and version.is_lts + if with_lts: + tag_attrs.lts = version + if without_distro: + tags.append("lts") + tags.append(f"lts-{distro}") + + # If the tag `22`, `23`, `24` etc. should be included in the tags + with_major = tag_attrs.majors.get(version.major) in (None, version) + if with_major: + tag_attrs.majors[version.major] = version + if without_distro: + tags.append(f"{version.major}") + tags.append(f"{version.major}-{distro}") + + # Add all normal tags + for tag in ( + f"{version.major}.{version.minor}", + f"{version.major}.{version.minor}.{version.patch}", + f"{version.major}.{version.minor}.{version.patch}.{version.tweak}", + ): + if without_distro: + tags.append(tag) + tags.append(f"{tag}-{distro}") + + return f"Tags: {', '.join(tags)}" + + +def generate_ldf(args: argparse.Namespace) -> None: + """Collect all args.dockerfile_glob files from args.directory and generate the + Library Definition File, read about it in + https://github.com/docker-library/official-images/?tab=readme-ov-file#library-definition-files + """ + directory = Path(git_runner.cwd) / args.directory / args.image_type + versions = sorted([get_version_from_string(d.name) for d in directory.iterdir()]) + assert versions, "There are no directories to generate the LDF" + if args.check_changed: + assert not path_is_changed( + directory + ), f"There are uncommitted changes in {directory}" + git = Git(True) + # Support a few repositories, get the git-root for images directory + dir_git_root = ( + args.directory / git_runner(f"git -C {args.directory} rev-parse --show-cdup") + ).absolute() + lines = ldf_header(git, directory) + tag_attrs = TagAttrs(versions[-1], {}, None) + + # We iterate from the most recent to the oldest version + for version in reversed(versions): + tag_dir = directory / str(version) + for file in tag_dir.glob(args.dockerfile_glob): + lines.append("") + distro = file.suffix[1:] + if distro == "ubuntu": + # replace 'ubuntu' by the release name from UBUNTU_NAMES + with open(file, "r", encoding="utf-8") as fd: + for l in fd: + if l.startswith("FROM ubuntu:"): + ubuntu_version = l.split(":")[-1].strip() + distro = UBUNTU_NAMES[ubuntu_version] + break + lines.append(ldf_tags(version, distro, tag_attrs)) + lines.append("Architectures: amd64, arm64v8") + lines.append(f"Directory: {tag_dir.relative_to(dir_git_root)}") + lines.append(f"File: {file.name}") + + # For the last '\n' in join + lines.append("") + ldf_file = ( + Path(git_runner.cwd) / args.directory / DOCKER_LIBRARY_NAME[args.image_type] + ) + ldf_file.write_text("\n".join(lines)) + logging.info("The content of LDF file:\n%s", "\n".join(lines)) + if args.commit and path_is_changed(ldf_file): + logging.info("Starting committing or %s file", ldf_file) + ldf_dir = ldf_file.parent + git_runner(f"git -C {ldf_dir} add {ldf_file}") + commit_message = ( + f"Re-/Generated docker LDF for {args.image_type} image\n\n" + f"The file is generated and committed as following:\n{get_cmdline()}" + ) + git_runner(f"{GIT_PREFIX} -C {ldf_dir} commit -F -", input=commit_message) + + +def main() -> None: + args = parse_args() + log_levels = [logging.CRITICAL, logging.WARN, logging.INFO, logging.DEBUG] + logging.basicConfig(level=log_levels[min(args.verbose, 3)]) + logging.debug("Arguments are %s", pformat(args.__dict__)) + if args.command == "generate-tree": + generate_tree(args) + elif args.command == "generate-ldf": + generate_ldf(args) + + +if __name__ == "__main__": + main() diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 8ed2f972183..d602a2d752a 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -132,6 +132,12 @@ class PRInfo: ref = github_event.get("ref", "refs/heads/master") if ref and ref.startswith("refs/heads/"): ref = ref[11:] + # Default values + self.base_ref = "" # type: str + self.base_name = "" # type: str + self.head_ref = "" # type: str + self.head_name = "" # type: str + self.number = 0 # type: int # workflow completed event, used for PRs only if "action" in github_event and github_event["action"] == "completed": @@ -146,7 +152,7 @@ class PRInfo: if "pull_request" in github_event: # pull request and other similar events self.event_type = EventType.PULL_REQUEST - self.number = github_event["pull_request"]["number"] # type: int + self.number = github_event["pull_request"]["number"] if pr_event_from_api: try: response = get_gh_api( @@ -172,17 +178,13 @@ class PRInfo: self.pr_html_url = f"{repo_prefix}/pull/{self.number}" # master or backport/xx.x/xxxxx - where the PR will be merged - self.base_ref = github_event["pull_request"]["base"]["ref"] # type: str + self.base_ref = github_event["pull_request"]["base"]["ref"] # ClickHouse/ClickHouse - self.base_name = github_event["pull_request"]["base"]["repo"][ - "full_name" - ] # type: str + self.base_name = github_event["pull_request"]["base"]["repo"]["full_name"] # any_branch-name - the name of working branch name - self.head_ref = github_event["pull_request"]["head"]["ref"] # type: str + self.head_ref = github_event["pull_request"]["head"]["ref"] # UserName/ClickHouse or ClickHouse/ClickHouse - self.head_name = github_event["pull_request"]["head"]["repo"][ - "full_name" - ] # type: str + self.head_name = github_event["pull_request"]["head"]["repo"]["full_name"] self.body = github_event["pull_request"]["body"] self.labels = { label["name"] for label in github_event["pull_request"]["labels"] diff --git a/tests/ci/release.py b/tests/ci/release.py deleted file mode 100755 index ed9d60a5cad..00000000000 --- a/tests/ci/release.py +++ /dev/null @@ -1,693 +0,0 @@ -#!/usr/bin/env python3 - -""" -script to create releases for ClickHouse - -The `gh` CLI preferred over the PyGithub to have an easy way to rollback bad -release in command line by simple execution giving rollback commands - -On another hand, PyGithub is used for convenient getting commit's status from API - -To run this script on a freshly installed Ubuntu 22.04 system, it is enough to do the following commands: - -sudo apt install pip -pip install requests boto3 github PyGithub -sudo snap install gh -gh auth login -""" - - -import argparse -import json -import logging -import subprocess -from contextlib import contextmanager -from typing import Any, Final, Iterator, List, Optional, Tuple - -from ci_config import Labels -from git_helper import Git, commit, release_branch -from report import SUCCESS -from version_helper import ( - FILE_WITH_VERSION_PATH, - GENERATED_CONTRIBUTORS, - ClickHouseVersion, - VersionType, - get_abs_path, - get_version_from_repo, - update_cmake_version, - update_contributors, -) - -RELEASE_READY_STATUS = "Ready for release" - - -class Repo: - VALID = ("ssh", "https", "origin") - - def __init__(self, repo: str, protocol: str): - self._repo = repo - self._url = "" - self.url = protocol - - @property - def url(self) -> str: - return self._url - - @url.setter - def url(self, protocol: str) -> None: - if protocol == "ssh": - self._url = f"git@github.com:{self}.git" - elif protocol == "https": - self._url = f"https://github.com/{self}.git" - elif protocol == "origin": - self._url = protocol - else: - raise ValueError(f"protocol must be in {self.VALID}") - - def __str__(self): - return self._repo - - -class Release: - NEW = "new" # type: Final - PATCH = "patch" # type: Final - VALID_TYPE = (NEW, PATCH) # type: Final[Tuple[str, str]] - CMAKE_PATH = get_abs_path(FILE_WITH_VERSION_PATH) - CONTRIBUTORS_PATH = get_abs_path(GENERATED_CONTRIBUTORS) - - def __init__( - self, - repo: Repo, - release_commit: str, - release_type: str, - dry_run: bool, - with_stderr: bool, - ): - self.repo = repo - self._release_commit = "" - self.release_commit = release_commit - self.dry_run = dry_run - self.with_stderr = with_stderr - assert release_type in self.VALID_TYPE - self.release_type = release_type - self._git = Git() - self._version = get_version_from_repo(git=self._git) - self.release_version = self.version - self._release_branch = "" - self._version_new_tag = None # type: Optional[ClickHouseVersion] - self._rollback_stack = [] # type: List[str] - - def run( - self, cmd: str, cwd: Optional[str] = None, dry_run: bool = False, **kwargs: Any - ) -> str: - cwd_text = "" - if cwd: - cwd_text = f" (CWD='{cwd}')" - if dry_run: - logging.info("Would run command%s:\n %s", cwd_text, cmd) - return "" - if not self.with_stderr: - kwargs["stderr"] = subprocess.DEVNULL - - logging.info("Running command%s:\n %s", cwd_text, cmd) - return self._git.run(cmd, cwd, **kwargs) - - def set_release_info(self): - # Fetch release commit and tags in case they don't exist locally - self.run( - f"git fetch {self.repo.url} {self.release_commit} --no-recurse-submodules" - ) - self.run(f"git fetch {self.repo.url} --tags --no-recurse-submodules") - - # Get the actual version for the commit before check - with self._checkout(self.release_commit, True): - self.release_branch = f"{self.version.major}.{self.version.minor}" - self.release_version = get_version_from_repo(git=self._git) - self.release_version.with_description(self.get_stable_release_type()) - - self.read_version() - - def read_version(self): - self._git.update() - self.version = get_version_from_repo(git=self._git) - - def get_stable_release_type(self) -> str: - if self.version.is_lts: - return VersionType.LTS - return VersionType.STABLE - - def check_commit_release_ready(self): - per_page = 100 - page = 1 - while True: - statuses = json.loads( - self.run( - f"gh api 'repos/{self.repo}/commits/{self.release_commit}" - f"/statuses?per_page={per_page}&page={page}'" - ) - ) - - if not statuses: - break - - for status in statuses: - if status["context"] == RELEASE_READY_STATUS: - if not status["state"] == SUCCESS: - raise ValueError( - f"the status {RELEASE_READY_STATUS} is {status['state']}" - ", not success" - ) - - return - - page += 1 - - raise KeyError( - f"the status {RELEASE_READY_STATUS} " - f"is not found for commit {self.release_commit}" - ) - - def check_prerequisites(self): - """ - Check tooling installed in the system, `git` is checked by Git() init - """ - try: - self.run("gh auth status") - except subprocess.SubprocessError: - logging.error( - "The github-cli either not installed or not setup, please follow " - "the instructions on https://github.com/cli/cli#installation and " - "https://cli.github.com/manual/" - ) - raise - - if self.release_type == self.PATCH: - self.check_commit_release_ready() - - def do( - self, check_dirty: bool, check_run_from_master: bool, check_branch: bool - ) -> None: - self.check_prerequisites() - - if check_dirty: - logging.info("Checking if repo is clean") - try: - self.run("git diff HEAD --exit-code") - except subprocess.CalledProcessError: - logging.fatal("Repo contains uncommitted changes") - raise - - if check_run_from_master and self._git.branch != "master": - raise RuntimeError("the script must be launched only from master") - - self.set_release_info() - - if check_branch: - self.check_branch() - - if self.release_type == self.NEW: - with self._checkout(self.release_commit, True): - # Checkout to the commit, it will provide the correct current version - with self.new_release(): - with self.create_release_branch(): - logging.info( - "Publishing release %s from commit %s is done", - self.release_version.describe, - self.release_commit, - ) - - elif self.release_type == self.PATCH: - with self._checkout(self.release_commit, True): - with self.patch_release(): - logging.info( - "Publishing release %s from commit %s is done", - self.release_version.describe, - self.release_commit, - ) - - if self.dry_run: - logging.info("Dry running, clean out possible changes") - rollback = self._rollback_stack.copy() - rollback.reverse() - for cmd in rollback: - self.run(cmd) - return - - self.log_post_workflows() - self.log_rollback() - - def check_no_tags_after(self): - tags_after_commit = self.run(f"git tag --contains={self.release_commit}") - if tags_after_commit: - raise RuntimeError( - f"Commit {self.release_commit} belongs to following tags:\n" - f"{tags_after_commit}\nChoose another commit" - ) - - def check_branch(self): - branch = self.release_branch - if self.release_type == self.NEW: - # Commit to spin up the release must belong to a main branch - branch = "master" - elif self.release_type != self.PATCH: - raise ( - ValueError(f"release_type {self.release_type} not in {self.VALID_TYPE}") - ) - - # Prefetch the branch to have it updated - if self._git.branch == branch: - self.run("git pull --no-recurse-submodules") - else: - self.run( - f"git fetch {self.repo.url} {branch}:{branch} --no-recurse-submodules" - ) - output = self.run(f"git branch --contains={self.release_commit} {branch}") - if branch not in output: - raise RuntimeError( - f"commit {self.release_commit} must belong to {branch} " - f"for {self.release_type} release" - ) - - def _update_cmake_contributors( - self, version: ClickHouseVersion, reset_tweak: bool = True - ) -> None: - if reset_tweak: - desc = version.description - version = version.reset_tweak() - version.with_description(desc) - update_cmake_version(version) - update_contributors(raise_error=True) - if self.dry_run: - logging.info( - "Dry running, resetting the following changes in the repo:\n%s", - self.run(f"git diff '{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'"), - ) - self.run(f"git checkout '{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'") - - def _commit_cmake_contributors( - self, version: ClickHouseVersion, reset_tweak: bool = True - ) -> None: - if reset_tweak: - version = version.reset_tweak() - self.run( - f"git commit '{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}' " - f"-m 'Update autogenerated version to {version.string} and contributors'", - dry_run=self.dry_run, - ) - - @property - def bump_part(self) -> ClickHouseVersion.PART_TYPE: - if self.release_type == Release.NEW: - if self._version.minor >= 12: - return "major" - return "minor" - return "patch" - - @property - def has_rollback(self) -> bool: - return bool(self._rollback_stack) - - def log_rollback(self): - if self.has_rollback: - rollback = self._rollback_stack.copy() - rollback.reverse() - logging.info( - "To rollback the action run the following commands:\n %s", - "\n ".join(rollback), - ) - - def log_post_workflows(self): - logging.info( - "To verify all actions are running good visit the following links:\n %s", - "\n ".join( - f"https://github.com/{self.repo}/actions/workflows/{action}.yml" - for action in ("release", "tags_stable") - ), - ) - - @contextmanager - def create_release_branch(self): - self.check_no_tags_after() - # Create release branch - self.read_version() - assert self._version_new_tag is not None - with self._create_tag( - self._version_new_tag.describe, - self.release_commit, - f"Initial commit for release {self._version_new_tag.major}.{self._version_new_tag.minor}", - ): - with self._create_branch(self.release_branch, self.release_commit): - with self._checkout(self.release_branch, True): - with self._bump_release_branch(): - yield - - @contextmanager - def patch_release(self): - self.check_no_tags_after() - self.read_version() - version_type = self.get_stable_release_type() - self.version.with_description(version_type) - with self._create_gh_release(False): - self.version = self.version.update(self.bump_part) - self.version.with_description(version_type) - self._update_cmake_contributors(self.version) - # Checking out the commit of the branch and not the branch itself, - # then we are able to skip rollback - with self._checkout(f"{self.release_branch}^0", False): - current_commit = self.run("git rev-parse HEAD") - self._commit_cmake_contributors(self.version) - with self._push( - "HEAD", with_rollback_on_fail=False, remote_ref=self.release_branch - ): - # DO NOT PUT ANYTHING ELSE HERE - # The push must be the last action and mean the successful release - self._rollback_stack.append( - f"{self.dry_run_prefix}git push {self.repo.url} " - f"+{current_commit}:{self.release_branch}" - ) - yield - - @contextmanager - def new_release(self): - # Create branch for a version bump - self.read_version() - self.version = self.version.update(self.bump_part) - helper_branch = f"{self.version.major}.{self.version.minor}-prepare" - with self._create_branch(helper_branch, self.release_commit): - with self._checkout(helper_branch, True): - with self._bump_version_in_master(helper_branch): - yield - - @property - def version(self) -> ClickHouseVersion: - return self._version - - @version.setter - def version(self, version: ClickHouseVersion) -> None: - if not isinstance(version, ClickHouseVersion): - raise ValueError(f"version must be ClickHouseVersion, not {type(version)}") - self._version = version - - @property - def release_branch(self) -> str: - return self._release_branch - - @release_branch.setter - def release_branch(self, branch: str) -> None: - self._release_branch = release_branch(branch) - - @property - def release_commit(self) -> str: - return self._release_commit - - @release_commit.setter - def release_commit(self, release_commit: str) -> None: - self._release_commit = commit(release_commit) - - @property - def dry_run_prefix(self) -> str: - if self.dry_run: - return "# " - return "" - - @contextmanager - def _bump_release_branch(self): - # Update only git, original version stays the same - self._git.update() - new_version = self.version.copy() - version_type = self.get_stable_release_type() - pr_labels = f"--label {Labels.RELEASE}" - if version_type == VersionType.LTS: - pr_labels += f" --label {Labels.RELEASE_LTS}" - new_version.with_description(version_type) - self._update_cmake_contributors(new_version) - self._commit_cmake_contributors(new_version) - with self._push(self.release_branch): - with self._create_gh_label( - f"v{self.release_branch}-must-backport", "10dbed" - ): - with self._create_gh_label( - f"v{self.release_branch}-affected", "c2bfff" - ): - # The following command is rolled back by deleting branch - # in self._push - self.run( - f"gh pr create --repo {self.repo} --title " - f"'Release pull request for branch {self.release_branch}' " - f"--head {self.release_branch} {pr_labels} " - "--body 'This PullRequest is a part of ClickHouse release " - "cycle. It is used by CI system only. Do not perform any " - "changes with it.'", - dry_run=self.dry_run, - ) - # Here the release branch part is done. - # We don't create a release itself automatically to have a - # safe window to backport possible bug fixes. - yield - - @contextmanager - def _bump_version_in_master(self, helper_branch: str) -> Iterator[None]: - self.read_version() - self.version = self.version.update(self.bump_part) - self.version.with_description(VersionType.TESTING) - self._update_cmake_contributors(self.version) - self._commit_cmake_contributors(self.version) - # Create a version-new tag - self._version_new_tag = self.version.copy() - self._version_new_tag.tweak = 1 - self._version_new_tag.with_description(VersionType.NEW) - - with self._push(helper_branch): - body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md") - # The following command is rolled back by deleting branch in self._push - self.run( - f"gh pr create --repo {self.repo} --title 'Update version after " - f"release' --head {helper_branch} --body-file '{body_file}' " - "--label 'do not test' --assignee @me", - dry_run=self.dry_run, - ) - # Here the new release part is done - yield - - @contextmanager - def _checkout(self, ref: str, with_checkout_back: bool = False) -> Iterator[None]: - self._git.update() - orig_ref = self._git.branch or self._git.sha - rollback_cmd = "" - if ref not in (self._git.branch, self._git.sha): - self.run(f"git checkout {ref}") - # checkout is not put into rollback_stack intentionally - rollback_cmd = f"git checkout {orig_ref}" - # always update version and git after checked out ref - self.read_version() - try: - yield - except (Exception, KeyboardInterrupt): - logging.warning("Rolling back checked out %s for %s", ref, orig_ref) - self.run(f"git reset --hard; git checkout -f {orig_ref}") - raise - # Normal flow when we need to checkout back - if with_checkout_back and rollback_cmd: - self.run(rollback_cmd) - - @contextmanager - def _create_branch(self, name: str, start_point: str = "") -> Iterator[None]: - self.run(f"git branch {name} {start_point}") - - rollback_cmd = f"git branch -D {name}" - self._rollback_stack.append(rollback_cmd) - try: - yield - except (Exception, KeyboardInterrupt): - logging.warning("Rolling back created branch %s", name) - self.run(rollback_cmd) - raise - - @contextmanager - def _create_gh_label(self, label: str, color_hex: str) -> Iterator[None]: - # API call, https://docs.github.com/en/rest/reference/issues#create-a-label - self.run( - f"gh api repos/{self.repo}/labels -f name={label} -f color={color_hex}", - dry_run=self.dry_run, - ) - rollback_cmd = ( - f"{self.dry_run_prefix}gh api repos/{self.repo}/labels/{label} -X DELETE" - ) - self._rollback_stack.append(rollback_cmd) - try: - yield - except (Exception, KeyboardInterrupt): - logging.warning("Rolling back label %s", label) - self.run(rollback_cmd) - raise - - @contextmanager - def _create_gh_release(self, as_prerelease: bool) -> Iterator[None]: - tag = self.release_version.describe - with self._create_tag(tag, self.release_commit): - # Preserve tag if version is changed - prerelease = "" - if as_prerelease: - prerelease = "--prerelease" - self.run( - f"gh release create {prerelease} --repo {self.repo} " - f"--title 'Release {tag}' '{tag}'", - dry_run=self.dry_run, - ) - rollback_cmd = ( - f"{self.dry_run_prefix}gh release delete --yes " - f"--repo {self.repo} '{tag}'" - ) - self._rollback_stack.append(rollback_cmd) - try: - yield - except (Exception, KeyboardInterrupt): - logging.warning("Rolling back release publishing") - self.run(rollback_cmd) - raise - - @contextmanager - def _create_tag( - self, tag: str, commit: str, tag_message: str = "" - ) -> Iterator[None]: - tag_message = tag_message or f"Release {tag}" - # Create tag even in dry-run - self.run(f"git tag -a -m '{tag_message}' '{tag}' {commit}") - rollback_cmd = f"git tag -d '{tag}'" - self._rollback_stack.append(rollback_cmd) - try: - with self._push(tag): - yield - except (Exception, KeyboardInterrupt): - logging.warning("Rolling back tag %s", tag) - self.run(rollback_cmd) - raise - - @contextmanager - def _push( - self, ref: str, with_rollback_on_fail: bool = True, remote_ref: str = "" - ) -> Iterator[None]: - if remote_ref == "": - remote_ref = ref - - self.run(f"git push {self.repo.url} {ref}:{remote_ref}", dry_run=self.dry_run) - if with_rollback_on_fail: - rollback_cmd = ( - f"{self.dry_run_prefix}git push -d {self.repo.url} {remote_ref}" - ) - self._rollback_stack.append(rollback_cmd) - - try: - yield - except (Exception, KeyboardInterrupt): - if with_rollback_on_fail: - logging.warning("Rolling back pushed ref %s", ref) - self.run(rollback_cmd) - - raise - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description="Script to release a new ClickHouse version, requires `git` and " - "`gh` (github-cli) commands " - "!!! LAUNCH IT ONLY FROM THE MASTER BRANCH !!!", - ) - - parser.add_argument( - "--commit", - required=True, - type=commit, - help="commit create a release", - ) - parser.add_argument( - "--repo", - default="ClickHouse/ClickHouse", - help="repository to create the release", - ) - parser.add_argument( - "--remote-protocol", - "-p", - default="ssh", - choices=Repo.VALID, - help="repo protocol for git commands remote, 'origin' is a special case and " - "uses 'origin' as a remote", - ) - parser.add_argument( - "--type", - required=True, - choices=Release.VALID_TYPE, - dest="release_type", - help="a release type to bump the major.minor.patch version part, " - "new branch is created only for the value 'new'", - ) - parser.add_argument("--with-release-branch", default=True, help=argparse.SUPPRESS) - parser.add_argument("--check-dirty", default=True, help=argparse.SUPPRESS) - parser.add_argument( - "--no-check-dirty", - dest="check_dirty", - action="store_false", - default=argparse.SUPPRESS, - help="(dangerous) if set, skip check repository for uncommitted changes", - ) - parser.add_argument("--check-run-from-master", default=True, help=argparse.SUPPRESS) - parser.add_argument( - "--no-run-from-master", - dest="check_run_from_master", - action="store_false", - default=argparse.SUPPRESS, - help="(for development) if set, the script could run from non-master branch", - ) - parser.add_argument("--check-branch", default=True, help=argparse.SUPPRESS) - parser.add_argument( - "--no-check-branch", - dest="check_branch", - action="store_false", - default=argparse.SUPPRESS, - help="(debug or development only, dangerous) if set, skip the branch check for " - "a run. By default, 'new' type work only for master, and 'patch' " - "works only for a release branches, that name " - "should be the same as '$MAJOR.$MINOR' version, e.g. 22.2", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="do not make any actual changes in the repo, just show what will be done", - ) - parser.add_argument( - "--with-stderr", - action="store_true", - help="if set, the stderr of all subprocess commands will be printed as well", - ) - - return parser.parse_args() - - -def main(): - logging.basicConfig(level=logging.INFO) - args = parse_args() - repo = Repo(args.repo, args.remote_protocol) - release = Release( - repo, args.commit, args.release_type, args.dry_run, args.with_stderr - ) - - try: - release.do(args.check_dirty, args.check_run_from_master, args.check_branch) - except: - if release.has_rollback: - logging.error( - "!!The release process finished with error, read the output carefully!!" - ) - logging.error( - "Probably, rollback finished with error. " - "If you don't see any of the following commands in the output, " - "execute them manually:" - ) - release.log_rollback() - raise - - -if __name__ == "__main__": - assert False, "Script Deprecated, ask ci team for help" - main() diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index 9a40ad1277f..d0aa034258a 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -311,24 +311,32 @@ class S3Helper: def list_prefix( self, s3_prefix_path: str, bucket: str = S3_BUILDS_BUCKET ) -> List[str]: - objects = self.client.list_objects_v2(Bucket=bucket, Prefix=s3_prefix_path) + paginator = self.client.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket, Prefix=s3_prefix_path) result = [] - if "Contents" in objects: - for obj in objects["Contents"]: - result.append(obj["Key"]) + for page in pages: + if "Contents" in page: + for obj in page["Contents"]: + result.append(obj["Key"]) return result def list_prefix_non_recursive( - self, s3_prefix_path: str, bucket: str = S3_BUILDS_BUCKET + self, + s3_prefix_path: str, + bucket: str = S3_BUILDS_BUCKET, + only_dirs: bool = False, ) -> List[str]: - objects = self.client.list_objects_v2(Bucket=bucket, Prefix=s3_prefix_path) + paginator = self.client.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket, Prefix=s3_prefix_path, Delimiter="/") result = [] - if "Contents" in objects: - for obj in objects["Contents"]: - if "/" not in obj["Key"][len(s3_prefix_path) + 1 :]: + for page in pages: + if not only_dirs and "Contents" in page: + for obj in page["Contents"]: result.append(obj["Key"]) - + if "CommonPrefixes" in page: + for obj in page["CommonPrefixes"]: + result.append(obj["Prefix"]) return result def url_if_exists(self, key: str, bucket: str = S3_BUILDS_BUCKET) -> str: diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py index 0e396b827ea..65418310c31 100644 --- a/tests/ci/test_ci_config.py +++ b/tests/ci/test_ci_config.py @@ -36,11 +36,12 @@ class TestCIConfig(unittest.TestCase): elif "binary_" in job.lower() or "package_" in job.lower(): if job.lower() in ( CI.BuildNames.PACKAGE_AARCH64, - CI.BuildNames.PACKAGE_ARM_ASAN, + CI.BuildNames.PACKAGE_AARCH64_ASAN, ): self.assertTrue( - CI.JOB_CONFIGS[job].runner_type in (CI.Runners.BUILDER_ARM,), - f"Job [{job}] must have [{CI.Runners.BUILDER_ARM}] runner", + CI.JOB_CONFIGS[job].runner_type + in (CI.Runners.BUILDER_AARCH64,), + f"Job [{job}] must have [{CI.Runners.BUILDER_AARCH64}] runner", ) else: self.assertTrue( @@ -96,7 +97,7 @@ class TestCIConfig(unittest.TestCase): else: self.assertTrue(CI.JOB_CONFIGS[job].build_config is None) if "asan" in job and "aarch" in job: - expected_builds = [CI.BuildNames.PACKAGE_ARM_ASAN] + expected_builds = [CI.BuildNames.PACKAGE_AARCH64_ASAN] elif "asan" in job: expected_builds = [CI.BuildNames.PACKAGE_ASAN] elif "msan" in job: diff --git a/tests/ci/test_ci_options.py b/tests/ci/test_ci_options.py index 536e18758f8..e1b780387e7 100644 --- a/tests/ci/test_ci_options.py +++ b/tests/ci/test_ci_options.py @@ -10,7 +10,7 @@ from ci_settings import CiSettings _TEST_BODY_1 = """ #### Run only: - [ ] Some Set -- [x] Integration tests (arm64) +- [x] Integration tests (aarch64) - [x] Integration tests - [x] Integration tests - [ ] Integration tests @@ -150,7 +150,7 @@ class TestCIOptions(unittest.TestCase): self.assertFalse(ci_options.no_ci_cache) self.assertTrue(ci_options.no_merge_commit) self.assertTrue(ci_options.woolen_wolfdog) - self.assertEqual(ci_options.ci_sets, ["ci_set_arm"]) + self.assertEqual(ci_options.ci_sets, ["ci_set_aarch64"]) self.assertCountEqual(ci_options.include_keywords, ["foo", "foo_bar"]) self.assertCountEqual(ci_options.exclude_keywords, ["foo", "foo_bar"]) diff --git a/tests/ci/upload_result_helper.py b/tests/ci/upload_result_helper.py index 619ffe937c7..85f4cfd12a0 100644 --- a/tests/ci/upload_result_helper.py +++ b/tests/ci/upload_result_helper.py @@ -64,6 +64,7 @@ def upload_results( s3_client: S3Helper, pr_number: int, commit_sha: str, + branch_name: str, test_results: TestResults, additional_files: Union[Sequence[Path], Sequence[str]], check_name: str, @@ -80,8 +81,7 @@ def upload_results( process_logs(s3_client, additional_files, s3_path_prefix, test_results) ) - branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commits/master" - branch_name = "master" + branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commits/{branch_name}" if pr_number != 0: branch_name = f"PR #{pr_number}" branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/pull/{pr_number}" diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index bf48f3f53dd..0fce2f920d1 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -164,6 +164,11 @@ class ClickHouseVersion: """our X.3 and X.8 are LTS""" return self.minor % 5 == 3 + @property + def is_supported(self) -> bool: + "we can support only versions with VersionType STABLE or LTS" + return self.description in (VersionType.STABLE, VersionType.LTS) + def get_stable_release_type(self) -> str: if self.is_lts: return VersionType.LTS @@ -365,7 +370,7 @@ def get_supported_versions( versions = list(versions) else: # checks that repo is not shallow in background - versions = get_tagged_versions() + versions = [v for v in get_tagged_versions() if v.is_supported] versions.sort() versions.reverse() for version in versions: diff --git a/tests/config/config.d/storage_conf.xml b/tests/config/config.d/storage_conf.xml index 74bad7528c8..fee7ce841a6 100644 --- a/tests/config/config.d/storage_conf.xml +++ b/tests/config/config.d/storage_conf.xml @@ -27,6 +27,7 @@ 0.3 0.15 0.15 + 50 0 diff --git a/tests/docker_scripts/attach_gdb.lib b/tests/docker_scripts/attach_gdb.lib index 4170a19176c..f8a08b5e39d 100644 --- a/tests/docker_scripts/attach_gdb.lib +++ b/tests/docker_scripts/attach_gdb.lib @@ -5,7 +5,8 @@ source /repo/tests/docker_scripts/utils.lib function attach_gdb_to_clickhouse() { - IS_ASAN=$(clickhouse-client --query "SELECT count() FROM system.build_options WHERE name = 'CXX_FLAGS' AND position('sanitize=address' IN value)") + # Use retries to avoid failures due to fault injections + IS_ASAN=$(run_with_retry 5 clickhouse-client --query "SELECT count() FROM system.build_options WHERE name = 'CXX_FLAGS' AND position('sanitize=address' IN value)") if [[ "$IS_ASAN" = "1" ]]; then echo "ASAN build detected. Not using gdb since it disables LeakSanitizer detections" diff --git a/tests/docker_scripts/upgrade_runner.sh b/tests/docker_scripts/upgrade_runner.sh index ece75ebf782..15c5ab69521 100755 --- a/tests/docker_scripts/upgrade_runner.sh +++ b/tests/docker_scripts/upgrade_runner.sh @@ -63,7 +63,7 @@ install_packages previous_release_package_folder function save_settings_clean() { local out=$1 && shift - script -q -c "clickhouse-local -q \"select * from system.settings into outfile '$out'\"" --log-out /dev/null + script -q -c "clickhouse-local --implicit-select 0 -q \"select * from system.settings into outfile '$out'\"" --log-out /dev/null } # We save the (numeric) version of the old server to compare setting changes between the 2 @@ -147,7 +147,6 @@ then FROM new_settings LEFT JOIN old_settings ON new_settings.name = old_settings.name WHERE (new_value != old_value) - AND NOT (startsWith(new_value, 'auto(') AND old_value LIKE '%auto(%') AND (name NOT IN ( SELECT arrayJoin(tupleElement(changes, 'name')) FROM diff --git a/tests/fuzz/build.sh b/tests/fuzz/build.sh index 12f41f6e079..f60336e6b53 100755 --- a/tests/fuzz/build.sh +++ b/tests/fuzz/build.sh @@ -1,5 +1,8 @@ #!/bin/bash -eu +# rename clickhouse +mv $OUT/clickhouse $OUT/clickhouse_fuzzer + # copy fuzzer options and dictionaries cp $SRC/tests/fuzz/*.dict $OUT/ cp $SRC/tests/fuzz/*.options $OUT/ diff --git a/tests/fuzz/clickhouse_fuzzer.options b/tests/fuzz/clickhouse_fuzzer.options new file mode 100644 index 00000000000..a22ba7b3b88 --- /dev/null +++ b/tests/fuzz/clickhouse_fuzzer.options @@ -0,0 +1,2 @@ +[CI] +FUZZER_ARGS = true diff --git a/tests/fuzz/dictionaries/keywords.dict b/tests/fuzz/dictionaries/keywords.dict index abaaf9e53b5..a37675ebcad 100644 --- a/tests/fuzz/dictionaries/keywords.dict +++ b/tests/fuzz/dictionaries/keywords.dict @@ -538,6 +538,7 @@ "WITH ADMIN OPTION" "WITH CHECK" "WITH FILL" +"STALENESS" "WITH GRANT OPTION" "WITH NAME" "WITH REPLACE OPTION" diff --git a/tests/fuzz/runner.py b/tests/fuzz/runner.py index 4d7ac457627..f4c66e00117 100644 --- a/tests/fuzz/runner.py +++ b/tests/fuzz/runner.py @@ -1,26 +1,49 @@ #!/usr/bin/env python3 import configparser +import datetime import logging import os import subprocess from pathlib import Path DEBUGGER = os.getenv("DEBUGGER", "") -FUZZER_ARGS = os.getenv("FUZZER_ARGS", "") +TIMEOUT = int(os.getenv("TIMEOUT", "0")) +OUTPUT = "/test_output" -def run_fuzzer(fuzzer: str): +class Stopwatch: + def __init__(self): + self.reset() + + @property + def duration_seconds(self) -> float: + return (datetime.datetime.utcnow() - self.start_time).total_seconds() + + @property + def start_time_str(self) -> str: + return self.start_time_str_value + + def reset(self) -> None: + self.start_time = datetime.datetime.utcnow() + self.start_time_str_value = self.start_time.strftime("%Y-%m-%d %H:%M:%S") + + +def run_fuzzer(fuzzer: str, timeout: int): logging.info("Running fuzzer %s...", fuzzer) - corpus_dir = f"{fuzzer}.in" - with Path(corpus_dir) as path: + seed_corpus_dir = f"{fuzzer}.in" + with Path(seed_corpus_dir) as path: if not path.exists() or not path.is_dir(): - corpus_dir = "" + seed_corpus_dir = "" + active_corpus_dir = f"corpus/{fuzzer}" + if not os.path.exists(active_corpus_dir): + os.makedirs(active_corpus_dir) options_file = f"{fuzzer}.options" custom_libfuzzer_options = "" fuzzer_arguments = "" + use_fuzzer_args = False with Path(options_file) as path: if path.exists() and path.is_file(): @@ -44,7 +67,9 @@ def run_fuzzer(fuzzer: str): if parser.has_section("libfuzzer"): custom_libfuzzer_options = " ".join( - f"-{key}={value}" for key, value in parser["libfuzzer"].items() + f"-{key}={value}" + for key, value in parser["libfuzzer"].items() + if key not in ("jobs", "exact_artifact_path") ) if parser.has_section("fuzzer_arguments"): @@ -53,19 +78,70 @@ def run_fuzzer(fuzzer: str): for key, value in parser["fuzzer_arguments"].items() ) - cmd_line = f"{DEBUGGER} ./{fuzzer} {FUZZER_ARGS} {corpus_dir}" - if custom_libfuzzer_options: - cmd_line += f" {custom_libfuzzer_options}" - if fuzzer_arguments: - cmd_line += f" {fuzzer_arguments}" + use_fuzzer_args = parser.getboolean("CI", "FUZZER_ARGS", fallback=False) - if not "-dict=" in cmd_line and Path(f"{fuzzer}.dict").exists(): - cmd_line += f" -dict={fuzzer}.dict" + exact_artifact_path = f"{OUTPUT}/{fuzzer}.unit" + status_path = f"{OUTPUT}/{fuzzer}.status" + out_path = f"{OUTPUT}/{fuzzer}.out" + stdout_path = f"{OUTPUT}/{fuzzer}.stdout" - cmd_line += " < /dev/null" + if not "-dict=" in custom_libfuzzer_options and Path(f"{fuzzer}.dict").exists(): + custom_libfuzzer_options += f" -dict={fuzzer}.dict" + custom_libfuzzer_options += f" -exact_artifact_path={exact_artifact_path}" - logging.info("...will execute: %s", cmd_line) - subprocess.check_call(cmd_line, shell=True) + libfuzzer_corpora = f"{active_corpus_dir} {seed_corpus_dir}" + + cmd_line = f"{DEBUGGER} ./{fuzzer} {fuzzer_arguments}" + + env = None + with_fuzzer_args = "" + if use_fuzzer_args: + env = {"FUZZER_ARGS": f"{custom_libfuzzer_options} {libfuzzer_corpora}".strip()} + with_fuzzer_args = f" with FUZZER_ARGS '{env['FUZZER_ARGS']}'" + else: + cmd_line += f" {custom_libfuzzer_options} {libfuzzer_corpora}" + + logging.info("...will execute: '%s'%s", cmd_line, with_fuzzer_args) + + stopwatch = Stopwatch() + try: + with open(out_path, "wb") as out, open(stdout_path, "wb") as stdout: + subprocess.run( + cmd_line.split(), + stdin=subprocess.DEVNULL, + stdout=stdout, + stderr=out, + text=True, + check=True, + shell=False, + errors="replace", + timeout=timeout, + env=env, + ) + except subprocess.CalledProcessError: + logging.info("Fail running %s", fuzzer) + with open(status_path, "w", encoding="utf-8") as status: + status.write( + f"FAIL\n{stopwatch.start_time_str}\n{stopwatch.duration_seconds}\n" + ) + except subprocess.TimeoutExpired: + logging.info("Successful running %s", fuzzer) + with open(status_path, "w", encoding="utf-8") as status: + status.write( + f"OK\n{stopwatch.start_time_str}\n{stopwatch.duration_seconds}\n" + ) + except Exception as e: + logging.info("Unexpected exception running %s: %s", fuzzer, e) + with open(status_path, "w", encoding="utf-8") as status: + status.write( + f"ERROR\n{stopwatch.start_time_str}\n{stopwatch.duration_seconds}\n" + ) + else: + logging.info("Error running %s", fuzzer) + with open(status_path, "w", encoding="utf-8") as status: + status.write( + f"ERROR\n{stopwatch.start_time_str}\n{stopwatch.duration_seconds}\n" + ) def main(): @@ -73,10 +149,14 @@ def main(): subprocess.check_call("ls -al", shell=True) + timeout = 30 if TIMEOUT == 0 else TIMEOUT + with Path() as current: for fuzzer in current.iterdir(): if (current / fuzzer).is_file() and os.access(current / fuzzer, os.X_OK): - run_fuzzer(fuzzer) + run_fuzzer(fuzzer.name, timeout) + + subprocess.check_call(f"ls -al {OUTPUT}", shell=True) if __name__ == "__main__": diff --git a/tests/integration/compose/docker_compose_mysql.yml b/tests/integration/compose/docker_compose_mysql.yml index f45410bde78..91df21165ea 100644 --- a/tests/integration/compose/docker_compose_mysql.yml +++ b/tests/integration/compose/docker_compose_mysql.yml @@ -5,7 +5,7 @@ services: environment: MYSQL_ROOT_PASSWORD: clickhouse MYSQL_ROOT_HOST: ${MYSQL_ROOT_HOST} - DATADIR: /mysql/ + DATADIR: /var/log/mysql/ expose: - ${MYSQL_PORT:-3306} command: --server_id=100 @@ -14,11 +14,11 @@ services: --gtid-mode="ON" --enforce-gtid-consistency --log-error-verbosity=3 - --log-error=/mysql/error.log + --log-error=/var/log/mysql/error.log --general-log=ON - --general-log-file=/mysql/general.log + --general-log-file=/var/log/mysql/general.log volumes: - type: ${MYSQL_LOGS_FS:-tmpfs} source: ${MYSQL_LOGS:-} - target: /mysql/ + target: /var/log/mysql/ user: ${MYSQL_DOCKER_USER} diff --git a/tests/integration/compose/docker_compose_mysql_8_0.yml b/tests/integration/compose/docker_compose_mysql_8_0.yml index e1ff1633bc7..e1e2e241443 100644 --- a/tests/integration/compose/docker_compose_mysql_8_0.yml +++ b/tests/integration/compose/docker_compose_mysql_8_0.yml @@ -4,8 +4,8 @@ services: restart: always environment: MYSQL_ROOT_PASSWORD: clickhouse - MYSQL_ROOT_HOST: ${MYSQL_ROOT_HOST} - DATADIR: /mysql/ + MYSQL_ROOT_HOST: ${MYSQL8_ROOT_HOST} + DATADIR: /var/log/mysql/ expose: - ${MYSQL8_PORT:-3306} command: --server_id=100 --log-bin='mysql-bin-1.log' @@ -13,11 +13,11 @@ services: --default-time-zone='+3:00' --gtid-mode="ON" --enforce-gtid-consistency --log-error-verbosity=3 - --log-error=/mysql/error.log + --log-error=/var/log/mysql/error.log --general-log=ON - --general-log-file=/mysql/general.log + --general-log-file=/var/log/mysql/general.log volumes: - type: ${MYSQL8_LOGS_FS:-tmpfs} source: ${MYSQL8_LOGS:-} - target: /mysql/ + target: /var/log/mysql/ user: ${MYSQL8_DOCKER_USER} diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 7c531cdd493..a0c2e1d1a70 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -744,11 +744,13 @@ class ClickHouseCluster: # available when with_prometheus == True self.with_prometheus = False self.prometheus_writer_host = "prometheus_writer" + self.prometheus_writer_ip = None self.prometheus_writer_port = 9090 self.prometheus_writer_logs_dir = p.abspath( p.join(self.instances_dir, "prometheus_writer/logs") ) self.prometheus_reader_host = "prometheus_reader" + self.prometheus_reader_ip = None self.prometheus_reader_port = 9091 self.prometheus_reader_logs_dir = p.abspath( p.join(self.instances_dir, "prometheus_reader/logs") @@ -1653,6 +1655,7 @@ class ClickHouseCluster: copy_common_configs=True, config_root_name="clickhouse", extra_configs=[], + extra_args="", randomize_settings=True, ) -> "ClickHouseInstance": """Add an instance to the cluster. @@ -1740,6 +1743,7 @@ class ClickHouseCluster: with_postgres_cluster=with_postgres_cluster, with_postgresql_java_client=with_postgresql_java_client, clickhouse_start_command=clickhouse_start_command, + clickhouse_start_extra_args=extra_args, main_config_name=main_config_name, users_config_name=users_config_name, copy_common_configs=copy_common_configs, @@ -2726,6 +2730,16 @@ class ClickHouseCluster: raise Exception("Can't wait LDAP to start") + def wait_prometheus_to_start(self): + self.prometheus_reader_ip = self.get_instance_ip(self.prometheus_reader_host) + self.prometheus_writer_ip = self.get_instance_ip(self.prometheus_writer_host) + self.wait_for_url( + f"http://{self.prometheus_reader_ip}:{self.prometheus_reader_port}/api/v1/query?query=time()" + ) + self.wait_for_url( + f"http://{self.prometheus_writer_ip}:{self.prometheus_writer_port}/api/v1/query?query=time()" + ) + def start(self): pytest_xdist_logging_to_separate_files.setup() logging.info("Running tests in {}".format(self.base_path)) @@ -3081,12 +3095,23 @@ class ClickHouseCluster: f"http://{self.jdbc_bridge_ip}:{self.jdbc_bridge_port}/ping" ) - if self.with_prometheus: + if self.with_prometheus and self.base_prometheus_cmd: os.makedirs(self.prometheus_writer_logs_dir) os.chmod(self.prometheus_writer_logs_dir, stat.S_IRWXU | stat.S_IRWXO) os.makedirs(self.prometheus_reader_logs_dir) os.chmod(self.prometheus_reader_logs_dir, stat.S_IRWXU | stat.S_IRWXO) + prometheus_start_cmd = self.base_prometheus_cmd + common_opts + + logging.info( + "Trying to create Prometheus instances by command %s", + " ".join(map(str, prometheus_start_cmd)), + ) + run_and_check(prometheus_start_cmd) + self.up_called = True + logging.info("Trying to connect to Prometheus...") + self.wait_prometheus_to_start() + clickhouse_start_cmd = self.base_cmd + ["up", "-d", "--no-recreate"] logging.debug( ( @@ -3368,6 +3393,7 @@ class ClickHouseInstance: with_postgres_cluster, with_postgresql_java_client, clickhouse_start_command=CLICKHOUSE_START_COMMAND, + clickhouse_start_extra_args="", main_config_name="config.xml", users_config_name="users.xml", copy_common_configs=True, @@ -3463,11 +3489,18 @@ class ClickHouseInstance: self.users_config_name = users_config_name self.copy_common_configs = copy_common_configs - self.clickhouse_start_command = clickhouse_start_command.replace( + clickhouse_start_command_with_conf = clickhouse_start_command.replace( "{main_config_file}", self.main_config_name ) - self.clickhouse_stay_alive_command = "bash -c \"trap 'pkill tail' INT TERM; {} --daemon; coproc tail -f /dev/null; wait $$!\"".format( - clickhouse_start_command + + self.clickhouse_start_command = "{} -- {}".format( + clickhouse_start_command_with_conf, clickhouse_start_extra_args + ) + self.clickhouse_start_command_in_daemon = "{} --daemon -- {}".format( + clickhouse_start_command_with_conf, clickhouse_start_extra_args + ) + self.clickhouse_stay_alive_command = "bash -c \"trap 'pkill tail' INT TERM; {}; coproc tail -f /dev/null; wait $$!\"".format( + self.clickhouse_start_command_in_daemon ) self.path = p.join(self.cluster.instances_dir, name) @@ -3910,7 +3943,7 @@ class ClickHouseInstance: if pid is None: logging.debug("No clickhouse process running. Start new one.") self.exec_in_container( - ["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], + ["bash", "-c", self.clickhouse_start_command_in_daemon], user=str(os.getuid()), ) if expected_to_fail: @@ -4230,7 +4263,7 @@ class ClickHouseInstance: user="root", ) self.exec_in_container( - ["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], + ["bash", "-c", self.clickhouse_start_command_in_daemon], user=str(os.getuid()), ) @@ -4311,7 +4344,7 @@ class ClickHouseInstance: ] ) self.exec_in_container( - ["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], + ["bash", "-c", self.clickhouse_start_command_in_daemon], user=str(os.getuid()), ) @@ -4699,9 +4732,7 @@ class ClickHouseInstance: entrypoint_cmd = self.clickhouse_start_command if self.stay_alive: - entrypoint_cmd = self.clickhouse_stay_alive_command.replace( - "{main_config_file}", self.main_config_name - ) + entrypoint_cmd = self.clickhouse_stay_alive_command else: entrypoint_cmd = ( "[" diff --git a/tests/queries/0_stateless/03225_alter_to_json_not_supported.reference b/tests/integration/test_deduplicated_attached_part_rename/__init__.py similarity index 100% rename from tests/queries/0_stateless/03225_alter_to_json_not_supported.reference rename to tests/integration/test_deduplicated_attached_part_rename/__init__.py diff --git a/tests/integration/test_deduplicated_attached_part_rename/test.py b/tests/integration/test_deduplicated_attached_part_rename/test.py new file mode 100644 index 00000000000..02fa2c9d4a3 --- /dev/null +++ b/tests/integration/test_deduplicated_attached_part_rename/test.py @@ -0,0 +1,88 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +ch1 = cluster.add_instance( + "ch1", + with_zookeeper=True, + macros={"replica": "node1"}, + stay_alive=True, +) + +database_name = "dedup_attach" + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def q(query): + return ch1.query(database=database_name, sql=query) + + +def test_deduplicated_attached_part_renamed_after_attach(started_cluster): + ch1.query(f"CREATE DATABASE {database_name}") + + q( + "CREATE TABLE dedup (id UInt32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/dedup_attach/dedup/s1', 'r1') ORDER BY id;" + ) + q("INSERT INTO dedup VALUES (1),(2),(3);") + + table_data_path = q( + "SELECT data_paths FROM system.tables WHERE database=currentDatabase() AND table='dedup'" + ).strip("'[]\n") + + ch1.exec_in_container( + [ + "bash", + "-c", + f"cp -r {table_data_path}/all_0_0_0 {table_data_path}/detached/all_0_0_0", + ] + ) + # Part is attached as all_1_1_0 + q("ALTER TABLE dedup ATTACH PART 'all_0_0_0'") + + assert 2 == int( + q( + f"SELECT count() FROM system.parts WHERE database='{database_name}' AND table = 'dedup'" + ).strip() + ) + + ch1.exec_in_container( + [ + "bash", + "-c", + f"cp -r {table_data_path}/all_1_1_0 {table_data_path}/detached/all_1_1_0", + ] + ) + # Part is deduplicated and not attached + q("ALTER TABLE dedup ATTACH PART 'all_1_1_0'") + + assert 2 == int( + q( + f"SELECT count() FROM system.parts WHERE database='{database_name}' AND table = 'dedup'" + ).strip() + ) + assert 1 == int( + q( + f"SELECT count() FROM system.detached_parts WHERE database='{database_name}' AND table = 'dedup'" + ).strip() + ) + # Check that it is not 'attaching_all_1_1_0' + assert ( + "all_1_1_0" + == q( + f"SELECT name FROM system.detached_parts WHERE database='{database_name}' AND table = 'dedup'" + ).strip() + ) + + q("DROP TABLE dedup") + q("SYSTEM DROP REPLICA 'r1' FROM ZKPATH '/clickhouse/tables/dedup_attach/dedup/s1'") + ch1.query(f"DROP DATABASE {database_name}") diff --git a/tests/integration/test_http_handlers_config/test.py b/tests/integration/test_http_handlers_config/test.py index efba4f05748..cf291c6dedd 100644 --- a/tests/integration/test_http_handlers_config/test.py +++ b/tests/integration/test_http_handlers_config/test.py @@ -17,9 +17,10 @@ class SimpleCluster: cluster.start() def add_instance(self, name, config_dir): - script_path = os.path.dirname(os.path.realpath(__file__)) return self.cluster.add_instance( - name, main_configs=[os.path.join(script_path, config_dir, "config.xml")] + name, + main_configs=[os.path.join(config_dir, "config.xml")], + user_configs=["users.d/users.yaml"], ) @@ -96,6 +97,16 @@ def test_dynamic_query_handler(): == res_custom_ct.headers["X-Test-Http-Response-Headers-Even-Multiple"] ) + assert cluster.instance.http_request( + "test_dynamic_handler_auth_with_password?query=select+currentUser()" + ).content, "with_password" + assert cluster.instance.http_request( + "test_dynamic_handler_auth_with_password_fail?query=select+currentUser()" + ).status_code, 403 + assert cluster.instance.http_request( + "test_dynamic_handler_auth_without_password?query=select+currentUser()" + ).content, "without_password" + def test_predefined_query_handler(): with contextlib.closing( @@ -177,6 +188,16 @@ def test_predefined_query_handler(): ) assert b"max_threads\t1\n" == res1.content + assert cluster.instance.http_request( + "test_predefined_handler_auth_with_password" + ).content, "with_password" + assert cluster.instance.http_request( + "test_predefined_handler_auth_with_password_fail" + ).status_code, 403 + assert cluster.instance.http_request( + "test_predefined_handler_auth_without_password" + ).content, "without_password" + def test_fixed_static_handler(): with contextlib.closing( diff --git a/tests/integration/test_http_handlers_config/test_dynamic_handler/config.xml b/tests/integration/test_http_handlers_config/test_dynamic_handler/config.xml index 58fedbd9078..4900219f595 100644 --- a/tests/integration/test_http_handlers_config/test_dynamic_handler/config.xml +++ b/tests/integration/test_http_handlers_config/test_dynamic_handler/config.xml @@ -24,5 +24,32 @@ + + + GET + /test_dynamic_handler_auth_with_password + + dynamic_query_handler + with_password + password + + + + GET + /test_dynamic_handler_auth_with_password_fail + + dynamic_query_handler + with_password + + + + + GET + /test_dynamic_handler_auth_without_password + + dynamic_query_handler + without_password + + diff --git a/tests/integration/test_http_handlers_config/test_predefined_handler/config.xml b/tests/integration/test_http_handlers_config/test_predefined_handler/config.xml index a7804721f12..3c0ee3cd09a 100644 --- a/tests/integration/test_http_handlers_config/test_predefined_handler/config.xml +++ b/tests/integration/test_http_handlers_config/test_predefined_handler/config.xml @@ -33,5 +33,35 @@ INSERT INTO test_table(id, data) SELECT {id:UInt32}, {_request_body:String} + + + GET + /test_predefined_handler_auth_with_password + + predefined_query_handler + with_password + password + SELECT currentUser() + + + + GET + /test_predefined_handler_auth_with_password_fail + + predefined_query_handler + with_password + + SELECT currentUser() + + + + GET + /test_predefined_handler_auth_without_password + + predefined_query_handler + without_password + SELECT currentUser() + + diff --git a/tests/integration/test_http_handlers_config/users.d/users.yaml b/tests/integration/test_http_handlers_config/users.d/users.yaml new file mode 100644 index 00000000000..9ab8a84ae5a --- /dev/null +++ b/tests/integration/test_http_handlers_config/users.d/users.yaml @@ -0,0 +1,7 @@ +users: + with_password: + profile: default + password: password + without_password: + profile: default + no_password: 1 diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index ed80898ebc7..e2fa776a8f0 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -794,3 +794,17 @@ def test_keeper_storage_remove_on_cluster(cluster, ignore, expected_raise): node.query( f"DROP NAMED COLLECTION test_nc ON CLUSTER `replicated_nc_nodes_cluster`" ) + + +@pytest.mark.parametrize( + "instance_name", + [("node"), ("node_with_keeper")], +) +def test_name_escaping(cluster, instance_name): + node = cluster.instances[instance_name] + + node.query("DROP NAMED COLLECTION IF EXISTS `test_!strange/symbols!`;") + node.query("CREATE NAMED COLLECTION `test_!strange/symbols!` AS key1=1, key2=2") + node.restart_clickhouse() + + node.query("DROP NAMED COLLECTION `test_!strange/symbols!`") diff --git a/tests/integration/test_prometheus_protocols/test.py b/tests/integration/test_prometheus_protocols/test.py index e368c841c4e..49bc7817f02 100644 --- a/tests/integration/test_prometheus_protocols/test.py +++ b/tests/integration/test_prometheus_protocols/test.py @@ -20,7 +20,7 @@ node = cluster.add_instance( def execute_query_on_prometheus_writer(query, timestamp): return execute_query_impl( - cluster.get_instance_ip(cluster.prometheus_writer_host), + cluster.prometheus_writer_ip, cluster.prometheus_writer_port, "/api/v1/query", query, @@ -30,7 +30,7 @@ def execute_query_on_prometheus_writer(query, timestamp): def execute_query_on_prometheus_reader(query, timestamp): return execute_query_impl( - cluster.get_instance_ip(cluster.prometheus_reader_host), + cluster.prometheus_reader_ip, cluster.prometheus_reader_port, "/api/v1/query", query, diff --git a/tests/integration/test_quorum_inserts/test.py b/tests/integration/test_quorum_inserts/test.py index eefc4882e8e..5e4a960acdf 100644 --- a/tests/integration/test_quorum_inserts/test.py +++ b/tests/integration/test_quorum_inserts/test.py @@ -1,5 +1,6 @@ import concurrent import time +import uuid import pytest @@ -46,10 +47,10 @@ def started_cluster(): def test_simple_add_replica(started_cluster): - zero.query("DROP TABLE IF EXISTS test_simple ON CLUSTER cluster") + table_name = "test_simple_" + uuid.uuid4().hex create_query = ( - "CREATE TABLE test_simple " + f"CREATE TABLE {table_name} " "(a Int8, d Date) " "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " "PARTITION BY d ORDER BY a" @@ -58,91 +59,78 @@ def test_simple_add_replica(started_cluster): zero.query(create_query) first.query(create_query) - first.query("SYSTEM STOP FETCHES test_simple") + first.query(f"SYSTEM STOP FETCHES {table_name}") zero.query( - "INSERT INTO test_simple VALUES (1, '2011-01-01')", + f"INSERT INTO {table_name} VALUES (1, '2011-01-01')", settings={"insert_quorum": 1}, ) - assert "1\t2011-01-01\n" == zero.query("SELECT * from test_simple") - assert "" == first.query("SELECT * from test_simple") + assert "1\t2011-01-01\n" == zero.query(f"SELECT * from {table_name}") + assert "" == first.query(f"SELECT * from {table_name}") - first.query("SYSTEM START FETCHES test_simple") + first.query(f"SYSTEM START FETCHES {table_name}") - first.query("SYSTEM SYNC REPLICA test_simple", timeout=20) + first.query(f"SYSTEM SYNC REPLICA {table_name}", timeout=20) - assert "1\t2011-01-01\n" == zero.query("SELECT * from test_simple") - assert "1\t2011-01-01\n" == first.query("SELECT * from test_simple") + assert "1\t2011-01-01\n" == zero.query(f"SELECT * from {table_name}") + assert "1\t2011-01-01\n" == first.query(f"SELECT * from {table_name}") second.query(create_query) - second.query("SYSTEM SYNC REPLICA test_simple", timeout=20) + second.query(f"SYSTEM SYNC REPLICA {table_name}", timeout=20) - assert "1\t2011-01-01\n" == zero.query("SELECT * from test_simple") - assert "1\t2011-01-01\n" == first.query("SELECT * from test_simple") - assert "1\t2011-01-01\n" == second.query("SELECT * from test_simple") + assert "1\t2011-01-01\n" == zero.query(f"SELECT * from {table_name}") + assert "1\t2011-01-01\n" == first.query(f"SELECT * from {table_name}") + assert "1\t2011-01-01\n" == second.query(f"SELECT * from {table_name}") - zero.query("DROP TABLE IF EXISTS test_simple ON CLUSTER cluster") + zero.query(f"DROP TABLE IF EXISTS {table_name} ON CLUSTER cluster") def test_drop_replica_and_achieve_quorum(started_cluster): - zero.query( - "DROP TABLE IF EXISTS test_drop_replica_and_achieve_quorum ON CLUSTER cluster" - ) - + table_name = "test_drop_replica_and_achieve_quorum_" + uuid.uuid4().hex create_query = ( - "CREATE TABLE test_drop_replica_and_achieve_quorum " + f"CREATE TABLE {table_name} " "(a Int8, d Date) " "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " "PARTITION BY d ORDER BY a" ) - print("Create Replicated table with two replicas") zero.query(create_query) first.query(create_query) - print("Stop fetches on one replica. Since that, it will be isolated.") - first.query("SYSTEM STOP FETCHES test_drop_replica_and_achieve_quorum") - + first.query(f"SYSTEM STOP FETCHES {table_name}") print("Insert to other replica. This query will fail.") quorum_timeout = zero.query_and_get_error( - "INSERT INTO test_drop_replica_and_achieve_quorum(a,d) VALUES (1, '2011-01-01')", + f"INSERT INTO {table_name}(a,d) VALUES (1, '2011-01-01')", settings={"insert_quorum_timeout": 5000}, ) assert "Timeout while waiting for quorum" in quorum_timeout, "Query must fail." - assert TSV("1\t2011-01-01\n") == TSV( zero.query( - "SELECT * FROM test_drop_replica_and_achieve_quorum", + f"SELECT * FROM {table_name}", settings={"select_sequential_consistency": 0}, ) ) - assert TSV("") == TSV( zero.query( - "SELECT * FROM test_drop_replica_and_achieve_quorum", + f"SELECT * FROM {table_name}", settings={"select_sequential_consistency": 1}, ) ) - # TODO:(Mikhaylov) begin; maybe delete this lines. I want clickhouse to fetch parts and update quorum. print("START FETCHES first replica") - first.query("SYSTEM START FETCHES test_drop_replica_and_achieve_quorum") - + first.query(f"SYSTEM START FETCHES {table_name}") print("SYNC first replica") - first.query("SYSTEM SYNC REPLICA test_drop_replica_and_achieve_quorum", timeout=20) + first.query(f"SYSTEM SYNC REPLICA {table_name}", timeout=20) # TODO:(Mikhaylov) end - print("Add second replica") second.query(create_query) - print("SYNC second replica") - second.query("SYSTEM SYNC REPLICA test_drop_replica_and_achieve_quorum", timeout=20) - + second.query(f"SYSTEM SYNC REPLICA {table_name}", timeout=20) print("Quorum for previous insert achieved.") assert TSV("1\t2011-01-01\n") == TSV( second.query( - "SELECT * FROM test_drop_replica_and_achieve_quorum", + f"SELECT * FROM {table_name}", settings={"select_sequential_consistency": 1}, ) ) @@ -155,7 +143,7 @@ def test_insert_quorum_with_drop_partition(started_cluster, add_new_data): "test_quorum_insert_with_drop_partition_new_data" if add_new_data else "test_quorum_insert_with_drop_partition" - ) + ) + uuid.uuid4().hex zero.query(f"DROP TABLE IF EXISTS {table_name} ON CLUSTER cluster") create_query = ( @@ -218,12 +206,12 @@ def test_insert_quorum_with_move_partition(started_cluster, add_new_data): "test_insert_quorum_with_move_partition_source_new_data" if add_new_data else "test_insert_quorum_with_move_partition_source" - ) + ) + uuid.uuid4().hex destination_table_name = ( "test_insert_quorum_with_move_partition_destination_new_data" if add_new_data else "test_insert_quorum_with_move_partition_destination" - ) + ) + uuid.uuid4().hex zero.query(f"DROP TABLE IF EXISTS {source_table_name} ON CLUSTER cluster") zero.query(f"DROP TABLE IF EXISTS {destination_table_name} ON CLUSTER cluster") @@ -296,10 +284,10 @@ def test_insert_quorum_with_move_partition(started_cluster, add_new_data): def test_insert_quorum_with_ttl(started_cluster): - zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_ttl ON CLUSTER cluster") + table_name = "test_insert_quorum_with_ttl_" + uuid.uuid4().hex create_query = ( - "CREATE TABLE test_insert_quorum_with_ttl " + f"CREATE TABLE {table_name} " "(a Int8, d Date) " "Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') " "PARTITION BY d ORDER BY a " @@ -311,12 +299,12 @@ def test_insert_quorum_with_ttl(started_cluster): zero.query(create_query) first.query(create_query) - print("Stop fetches for test_insert_quorum_with_ttl at first replica.") - first.query("SYSTEM STOP FETCHES test_insert_quorum_with_ttl") + print(f"Stop fetches for {table_name} at first replica.") + first.query(f"SYSTEM STOP FETCHES {table_name}") print("Insert should fail since it can not reach the quorum.") quorum_timeout = zero.query_and_get_error( - "INSERT INTO test_insert_quorum_with_ttl(a,d) VALUES(1, '2011-01-01')", + f"INSERT INTO {table_name}(a,d) VALUES(1, '2011-01-01')", settings={"insert_quorum_timeout": 5000}, ) assert "Timeout while waiting for quorum" in quorum_timeout, "Query must fail." @@ -327,51 +315,49 @@ def test_insert_quorum_with_ttl(started_cluster): time.sleep(10) assert TSV("1\t2011-01-01\n") == TSV( zero.query( - "SELECT * FROM test_insert_quorum_with_ttl", + f"SELECT * FROM {table_name}", settings={"select_sequential_consistency": 0}, ) ) - print("Resume fetches for test_insert_quorum_with_ttl at first replica.") - first.query("SYSTEM START FETCHES test_insert_quorum_with_ttl") + print(f"Resume fetches for {table_name} at first replica.") + first.query(f"SYSTEM START FETCHES {table_name}") print("Sync first replica.") - first.query("SYSTEM SYNC REPLICA test_insert_quorum_with_ttl") + first.query(f"SYSTEM SYNC REPLICA {table_name}") zero.query( - "INSERT INTO test_insert_quorum_with_ttl(a,d) VALUES(1, '2011-01-01')", + f"INSERT INTO {table_name}(a,d) VALUES(1, '2011-01-01')", settings={"insert_quorum_timeout": 5000}, ) print("Inserts should resume.") - zero.query("INSERT INTO test_insert_quorum_with_ttl(a, d) VALUES(2, '2012-02-02')") + zero.query(f"INSERT INTO {table_name}(a, d) VALUES(2, '2012-02-02')") - first.query("OPTIMIZE TABLE test_insert_quorum_with_ttl") - first.query("SYSTEM SYNC REPLICA test_insert_quorum_with_ttl") - zero.query("SYSTEM SYNC REPLICA test_insert_quorum_with_ttl") + first.query(f"OPTIMIZE TABLE {table_name}") + first.query(f"SYSTEM SYNC REPLICA {table_name}") + zero.query(f"SYSTEM SYNC REPLICA {table_name}") assert TSV("2\t2012-02-02\n") == TSV( first.query( - "SELECT * FROM test_insert_quorum_with_ttl", + f"SELECT * FROM {table_name}", settings={"select_sequential_consistency": 0}, ) ) assert TSV("2\t2012-02-02\n") == TSV( first.query( - "SELECT * FROM test_insert_quorum_with_ttl", + f"SELECT * FROM {table_name}", settings={"select_sequential_consistency": 1}, ) ) - zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_ttl ON CLUSTER cluster") + zero.query(f"DROP TABLE IF EXISTS {table_name} ON CLUSTER cluster") -def test_insert_quorum_with_keeper_loss_connection(): - zero.query( - "DROP TABLE IF EXISTS test_insert_quorum_with_keeper_fail ON CLUSTER cluster" - ) +def test_insert_quorum_with_keeper_loss_connection(started_cluster): + table_name = "test_insert_quorum_with_keeper_loss_" + uuid.uuid4().hex create_query = ( - "CREATE TABLE test_insert_quorum_with_keeper_loss" + f"CREATE TABLE {table_name} " "(a Int8, d Date) " "Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') " "ORDER BY a " @@ -380,7 +366,7 @@ def test_insert_quorum_with_keeper_loss_connection(): zero.query(create_query) first.query(create_query) - first.query("SYSTEM STOP FETCHES test_insert_quorum_with_keeper_loss") + first.query(f"SYSTEM STOP FETCHES {table_name}") zero.query("SYSTEM ENABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op") zero.query("SYSTEM ENABLE FAILPOINT replicated_merge_tree_insert_retry_pause") @@ -388,57 +374,60 @@ def test_insert_quorum_with_keeper_loss_connection(): with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: insert_future = executor.submit( lambda: zero.query( - "INSERT INTO test_insert_quorum_with_keeper_loss(a,d) VALUES(1, '2011-01-01')", + f"INSERT INTO {table_name}(a,d) VALUES(1, '2011-01-01')", settings={"insert_quorum_timeout": 150000}, ) ) - pm = PartitionManager() - pm.drop_instance_zk_connections(zero) + with PartitionManager() as pm: + pm.drop_instance_zk_connections(zero) - retries = 0 - zk = cluster.get_kazoo_client("zoo1") - while True: - if ( - zk.exists( - "/clickhouse/tables/test_insert_quorum_with_keeper_loss/replicas/zero/is_active" + retries = 0 + zk = cluster.get_kazoo_client("zoo1") + while True: + if ( + zk.exists( + f"/clickhouse/tables/{table_name}/replicas/zero/is_active" + ) + is None + ): + break + print("replica is still active") + time.sleep(1) + retries += 1 + if retries == 120: + raise Exception("Can not wait cluster replica inactive") + + first.query("SYSTEM ENABLE FAILPOINT finish_set_quorum_failed_parts") + quorum_fail_future = executor.submit( + lambda: first.query( + "SYSTEM WAIT FAILPOINT finish_set_quorum_failed_parts", timeout=300 ) - is None - ): - break - print("replica is still active") - time.sleep(1) - retries += 1 - if retries == 120: - raise Exception("Can not wait cluster replica inactive") - - first.query("SYSTEM ENABLE FAILPOINT finish_set_quorum_failed_parts") - quorum_fail_future = executor.submit( - lambda: first.query( - "SYSTEM WAIT FAILPOINT finish_set_quorum_failed_parts", timeout=300 ) - ) - first.query("SYSTEM START FETCHES test_insert_quorum_with_keeper_loss") + first.query(f"SYSTEM START FETCHES {table_name}") - concurrent.futures.wait([quorum_fail_future]) + concurrent.futures.wait([quorum_fail_future]) - assert quorum_fail_future.exception() is None + assert quorum_fail_future.exception() is None - zero.query("SYSTEM ENABLE FAILPOINT finish_clean_quorum_failed_parts") - clean_quorum_fail_parts_future = executor.submit( - lambda: first.query( - "SYSTEM WAIT FAILPOINT finish_clean_quorum_failed_parts", timeout=300 + zero.query("SYSTEM ENABLE FAILPOINT finish_clean_quorum_failed_parts") + clean_quorum_fail_parts_future = executor.submit( + lambda: first.query( + "SYSTEM WAIT FAILPOINT finish_clean_quorum_failed_parts", + timeout=300, + ) ) - ) - pm.restore_instance_zk_connections(zero) - concurrent.futures.wait([clean_quorum_fail_parts_future]) + pm.restore_instance_zk_connections(zero) + concurrent.futures.wait([clean_quorum_fail_parts_future]) - assert clean_quorum_fail_parts_future.exception() is None + assert clean_quorum_fail_parts_future.exception() is None - zero.query("SYSTEM DISABLE FAILPOINT replicated_merge_tree_insert_retry_pause") - concurrent.futures.wait([insert_future]) - assert insert_future.exception() is not None - assert not zero.contains_in_log("LOGICAL_ERROR") - assert zero.contains_in_log( - "fails to commit and will not retry or clean garbage" - ) + zero.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_insert_retry_pause" + ) + concurrent.futures.wait([insert_future]) + assert insert_future.exception() is not None + assert not zero.contains_in_log("LOGICAL_ERROR") + assert zero.contains_in_log( + "fails to commit and will not retry or clean garbage" + ) diff --git a/tests/integration/test_replicated_s3_zero_copy_drop_partition/test.py b/tests/integration/test_replicated_s3_zero_copy_drop_partition/test.py index 6d2bb0a3b70..7623a24c0ef 100644 --- a/tests/integration/test_replicated_s3_zero_copy_drop_partition/test.py +++ b/tests/integration/test_replicated_s3_zero_copy_drop_partition/test.py @@ -65,12 +65,24 @@ CREATE TABLE test_s3(c1 Int8, c2 Date) ENGINE = ReplicatedMergeTree('/test/table objects_after = get_objects_in_data_path() assert objects_before == objects_after + node1.query("DROP TABLE test_local SYNC") + node1.query("DROP TABLE test_s3 SYNC") def test_drop_complex_columns(started_cluster): + node1 = cluster.instances["node1"] + node1.query( + """ +CREATE TABLE warming_up( +id Int8 +) ENGINE = MergeTree +order by (id) SETTINGS storage_policy = 's3';""" + ) + + # Now we are sure that s3 storage is up and running start_objects = get_objects_in_data_path() print("Objects before", start_objects) - node1 = cluster.instances["node1"] + node1.query( """ CREATE TABLE test_s3_complex_types( @@ -104,3 +116,4 @@ vertical_merge_algorithm_min_columns_to_activate=1;""" end_objects = get_objects_in_data_path() print("Objects after drop", end_objects) assert start_objects == end_objects + node1.query("DROP TABLE warming_up SYNC") diff --git a/tests/integration/test_storage_mysql/test.py b/tests/integration/test_storage_mysql/test.py index 2fc62d7f511..2d34a52c17b 100644 --- a/tests/integration/test_storage_mysql/test.py +++ b/tests/integration/test_storage_mysql/test.py @@ -386,100 +386,6 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32, source Enum8(' conn.close() -def test_mysql_distributed(started_cluster): - table_name = "test_replicas" - - conn1 = get_mysql_conn(started_cluster, started_cluster.mysql8_ip) - conn2 = get_mysql_conn(started_cluster, started_cluster.mysql2_ip) - conn3 = get_mysql_conn(started_cluster, started_cluster.mysql3_ip) - conn4 = get_mysql_conn(started_cluster, started_cluster.mysql4_ip) - - create_mysql_db(conn1, "clickhouse") - create_mysql_db(conn2, "clickhouse") - create_mysql_db(conn3, "clickhouse") - create_mysql_db(conn4, "clickhouse") - - create_mysql_table(conn1, table_name) - create_mysql_table(conn2, table_name) - create_mysql_table(conn3, table_name) - create_mysql_table(conn4, table_name) - - node2.query("DROP TABLE IF EXISTS test_replicas") - - # Storage with with 3 replicas - node2.query( - """ - CREATE TABLE test_replicas - (id UInt32, name String, age UInt32, money UInt32) - ENGINE = MySQL('mysql{2|3|4}:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse'); """ - ) - - # Fill remote tables with different data to be able to check - nodes = [node1, node2, node2, node2] - for i in range(1, 5): - nodes[i - 1].query("DROP TABLE IF EXISTS test_replica{}".format(i)) - nodes[i - 1].query( - """ - CREATE TABLE test_replica{} - (id UInt32, name String, age UInt32, money UInt32) - ENGINE = MySQL('mysql{}:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse');""".format( - i, 80 if i == 1 else i - ) - ) - nodes[i - 1].query( - "INSERT INTO test_replica{} (id, name) SELECT number, 'host{}' from numbers(10) ".format( - i, i - ) - ) - - # test multiple ports parsing - result = node2.query( - """SELECT DISTINCT(name) FROM mysql('mysql{80|2|3}:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse'); """ - ) - assert result == "host1\n" or result == "host2\n" or result == "host3\n" - result = node2.query( - """SELECT DISTINCT(name) FROM mysql('mysql80:3306|mysql2:3306|mysql3:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse'); """ - ) - assert result == "host1\n" or result == "host2\n" or result == "host3\n" - - # check all replicas are traversed - query = "SELECT * FROM (" - for i in range(3): - query += "SELECT name FROM test_replicas UNION DISTINCT " - query += "SELECT name FROM test_replicas) ORDER BY name" - - result = node2.query(query) - assert result == "host2\nhost3\nhost4\n" - - # Storage with with two shards, each has 2 replicas - node2.query("DROP TABLE IF EXISTS test_shards") - - node2.query( - """ - CREATE TABLE test_shards - (id UInt32, name String, age UInt32, money UInt32) - ENGINE = ExternalDistributed('MySQL', 'mysql{80|2}:3306,mysql{3|4}:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse'); """ - ) - - # Check only one replica in each shard is used - result = node2.query("SELECT DISTINCT(name) FROM test_shards ORDER BY name") - assert result == "host1\nhost3\n" - - # check all replicas are traversed - query = "SELECT name FROM (" - for i in range(3): - query += "SELECT name FROM test_shards UNION DISTINCT " - query += "SELECT name FROM test_shards) ORDER BY name" - result = node2.query(query) - assert result == "host1\nhost2\nhost3\nhost4\n" - - # disconnect mysql - started_cluster.pause_container("mysql80") - result = node2.query("SELECT DISTINCT(name) FROM test_shards ORDER BY name") - started_cluster.unpause_container("mysql80") - assert result == "host2\nhost4\n" or result == "host3\nhost4\n" - - def test_external_settings(started_cluster): table_name = "test_external_settings" node1.query(f"DROP TABLE IF EXISTS {table_name}") diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index aaecc7537cf..78bb1167d79 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -449,89 +449,6 @@ def test_concurrent_queries(started_cluster): node1.query("DROP TABLE test.stat;") -def test_postgres_distributed(started_cluster): - cursor0 = started_cluster.postgres_conn.cursor() - cursor1 = started_cluster.postgres2_conn.cursor() - cursor2 = started_cluster.postgres3_conn.cursor() - cursor3 = started_cluster.postgres4_conn.cursor() - cursors = [cursor0, cursor1, cursor2, cursor3] - - for i in range(4): - cursors[i].execute("DROP TABLE IF EXISTS test_replicas") - cursors[i].execute("CREATE TABLE test_replicas (id Integer, name Text)") - cursors[i].execute( - f"""INSERT INTO test_replicas select i, 'host{i+1}' from generate_series(0, 99) as t(i);""" - ) - - # test multiple ports parsing - result = node2.query( - """SELECT DISTINCT(name) FROM postgresql('postgres{1|2|3}:5432', 'postgres', 'test_replicas', 'postgres', 'mysecretpassword'); """ - ) - assert result == "host1\n" or result == "host2\n" or result == "host3\n" - result = node2.query( - """SELECT DISTINCT(name) FROM postgresql('postgres2:5431|postgres3:5432', 'postgres', 'test_replicas', 'postgres', 'mysecretpassword'); """ - ) - assert result == "host3\n" or result == "host2\n" - - # Create storage with with 3 replicas - node2.query("DROP TABLE IF EXISTS test_replicas") - node2.query( - """ - CREATE TABLE test_replicas - (id UInt32, name String) - ENGINE = PostgreSQL('postgres{2|3|4}:5432', 'postgres', 'test_replicas', 'postgres', 'mysecretpassword'); """ - ) - - # Check all replicas are traversed - query = "SELECT name FROM (" - for i in range(3): - query += "SELECT name FROM test_replicas UNION DISTINCT " - query += "SELECT name FROM test_replicas) ORDER BY name" - result = node2.query(query) - assert result == "host2\nhost3\nhost4\n" - - # Create storage with with two two shards, each has 2 replicas - node2.query("DROP TABLE IF EXISTS test_shards") - - node2.query( - """ - CREATE TABLE test_shards - (id UInt32, name String, age UInt32, money UInt32) - ENGINE = ExternalDistributed('PostgreSQL', 'postgres{1|2}:5432,postgres{3|4}:5432', 'postgres', 'test_replicas', 'postgres', 'mysecretpassword'); """ - ) - - # Check only one replica in each shard is used - result = node2.query("SELECT DISTINCT(name) FROM test_shards ORDER BY name") - assert result == "host1\nhost3\n" - - node2.query( - """ - CREATE TABLE test_shards2 - (id UInt32, name String, age UInt32, money UInt32) - ENGINE = ExternalDistributed('PostgreSQL', postgres4, addresses_expr='postgres{1|2}:5432,postgres{3|4}:5432'); """ - ) - - result = node2.query("SELECT DISTINCT(name) FROM test_shards2 ORDER BY name") - assert result == "host1\nhost3\n" - - # Check all replicas are traversed - query = "SELECT name FROM (" - for i in range(3): - query += "SELECT name FROM test_shards UNION DISTINCT " - query += "SELECT name FROM test_shards) ORDER BY name" - result = node2.query(query) - assert result == "host1\nhost2\nhost3\nhost4\n" - - # Disconnect postgres1 - started_cluster.pause_container("postgres1") - result = node2.query("SELECT DISTINCT(name) FROM test_shards ORDER BY name") - started_cluster.unpause_container("postgres1") - assert result == "host2\nhost4\n" or result == "host3\nhost4\n" - node2.query("DROP TABLE test_shards2") - node2.query("DROP TABLE test_shards") - node2.query("DROP TABLE test_replicas") - - def test_datetime_with_timezone(started_cluster): cursor = started_cluster.postgres_conn.cursor() cursor.execute("DROP TABLE IF EXISTS test_timezone") @@ -850,6 +767,7 @@ def test_filter_pushdown(started_cluster): "INSERT INTO test_filter_pushdown.test_table VALUES (1, 10), (1, 110), (2, 0), (3, 33), (4, 0)" ) + node1.query("DROP TABLE IF EXISTS test_filter_pushdown_pg_table") node1.query( """ CREATE TABLE test_filter_pushdown_pg_table (id UInt32, value UInt32) @@ -857,12 +775,14 @@ def test_filter_pushdown(started_cluster): """ ) + node1.query("DROP TABLE IF EXISTS test_filter_pushdown_local_table") node1.query( """ CREATE TABLE test_filter_pushdown_local_table (id UInt32, value UInt32) ENGINE Memory AS SELECT * FROM test_filter_pushdown_pg_table """ ) + node1.query("DROP TABLE IF EXISTS ch_table") node1.query( "CREATE TABLE ch_table (id UInt32, pg_id UInt32) ENGINE MergeTree ORDER BY id" ) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index c495fc1d44f..62afc0b1c1d 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -1000,6 +1000,9 @@ def test_max_set_age(started_cluster): assert "Cannot parse input" in node.query( f"SELECT exception FROM system.s3queue WHERE file_name ilike '%{file_with_error}'" ) + assert "Cannot parse input" in node.query( + f"SELECT exception FROM system.s3queue_log WHERE file_name ilike '%{file_with_error}' ORDER BY processing_end_time DESC LIMIT 1" + ) assert 1 == int( node.query( @@ -1403,8 +1406,8 @@ def test_shards_distributed(started_cluster, mode, processing_threads): # A unique path is necessary for repeatable tests keeper_path = f"/clickhouse/test_{table_name}_{generate_random_string()}" files_path = f"{table_name}_data" - files_to_generate = 300 - row_num = 300 + files_to_generate = 600 + row_num = 1000 total_rows = row_num * files_to_generate shards_num = 2 diff --git a/tests/queries/0_stateless/00755_avg_value_size_hint_passing.sql b/tests/queries/0_stateless/00755_avg_value_size_hint_passing.sql index 1d033b7aa47..c840bba6871 100644 --- a/tests/queries/0_stateless/00755_avg_value_size_hint_passing.sql +++ b/tests/queries/0_stateless/00755_avg_value_size_hint_passing.sql @@ -5,7 +5,7 @@ SET max_block_size = 1000; SET max_memory_usage = 1000000000; INSERT INTO size_hint SELECT arrayMap(x -> 'Hello', range(1000)) FROM numbers(10000); -SET max_memory_usage = 100000000, max_threads = 2; +SET max_memory_usage = 105000000, max_threads = 2; SELECT count(), sum(length(s)) FROM size_hint; DROP TABLE size_hint; diff --git a/tests/queries/0_stateless/01284_fuzz_bits.sql b/tests/queries/0_stateless/01284_fuzz_bits.sql index 95a07c7bd44..1055d2aa580 100644 --- a/tests/queries/0_stateless/01284_fuzz_bits.sql +++ b/tests/queries/0_stateless/01284_fuzz_bits.sql @@ -18,7 +18,7 @@ FROM reinterpretAsUInt8( substring( fuzzBits( - arrayStringConcat(arrayMap(x -> toString('\0'), range(10000))), + materialize(arrayStringConcat(arrayMap(x -> toString('\0'), range(10000)))), 0.3 ), id + 1, diff --git a/tests/queries/0_stateless/01287_max_execution_speed.sql b/tests/queries/0_stateless/01287_max_execution_speed.sql index 0d132999481..89c3050a256 100644 --- a/tests/queries/0_stateless/01287_max_execution_speed.sql +++ b/tests/queries/0_stateless/01287_max_execution_speed.sql @@ -1,5 +1,8 @@ -- Tags: no-fasttest, no-debug, no-tsan, no-msan, no-asan +SET max_rows_to_read=0; +SET max_bytes_to_read=0; + SET min_execution_speed = 100000000000, timeout_before_checking_execution_speed = 0; SELECT count() FROM system.numbers; -- { serverError TOO_SLOW } SET min_execution_speed = 0; diff --git a/tests/queries/0_stateless/01655_plan_optimizations.reference b/tests/queries/0_stateless/01655_plan_optimizations.reference index edf93b4b39f..7fc7556e85b 100644 --- a/tests/queries/0_stateless/01655_plan_optimizations.reference +++ b/tests/queries/0_stateless/01655_plan_optimizations.reference @@ -82,12 +82,12 @@ Filter column: notEquals(__table1.y, 0_UInt8) 9 10 > one condition of filter should be pushed down after aggregating, other two conditions are ANDed Filter column -FUNCTION and(minus(s, 8) :: 5, minus(s, 4) :: 2) -> and(notEquals(y, 0), minus(s, 8), minus(s, 4)) +FUNCTION and(minus(s, 8) :: 3, minus(s, 4) :: 5) -> and(notEquals(y, 0), minus(s, 8), minus(s, 4)) Aggregating Filter column: notEquals(y, 0) > (analyzer) one condition of filter should be pushed down after aggregating, other two conditions are ANDed Filter column -FUNCTION and(minus(__table1.s, 8_UInt8) :: 1, minus(__table1.s, 4_UInt8) :: 2) -> and(notEquals(__table1.y, 0_UInt8), minus(__table1.s, 8_UInt8), minus(__table1.s, 4_UInt8)) +FUNCTION and(minus(__table1.s, 8_UInt8) :: 3, minus(__table1.s, 4_UInt8) :: 5) -> and(notEquals(__table1.y, 0_UInt8), minus(__table1.s, 8_UInt8), minus(__table1.s, 4_UInt8)) Aggregating Filter column: notEquals(__table1.y, 0_UInt8) 0 1 @@ -163,7 +163,6 @@ Filter column: notEquals(__table1.y, 2_UInt8) > filter is pushed down before CreatingSets CreatingSets Filter -Filter 1 3 > one condition of filter is pushed down before LEFT JOIN diff --git a/tests/queries/0_stateless/01655_plan_optimizations.sh b/tests/queries/0_stateless/01655_plan_optimizations.sh index 42cdac8c01f..04ab9bbd11c 100755 --- a/tests/queries/0_stateless/01655_plan_optimizations.sh +++ b/tests/queries/0_stateless/01655_plan_optimizations.sh @@ -89,14 +89,14 @@ $CLICKHOUSE_CLIENT --enable_analyzer=0 --convert_query_to_cnf=0 -q " select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y ) where y != 0 and s - 8 and s - 4 settings enable_optimize_predicate_expression=0" | - grep -o "Aggregating\|Filter column\|Filter column: notEquals(y, 0)\|FUNCTION and(minus(s, 8) :: 5, minus(s, 4) :: 2) -> and(notEquals(y, 0), minus(s, 8), minus(s, 4))" + grep -o "Aggregating\|Filter column\|Filter column: notEquals(y, 0)\|FUNCTION and(minus(s, 8) :: 3, minus(s, 4) :: 5) -> and(notEquals(y, 0), minus(s, 8), minus(s, 4))" echo "> (analyzer) one condition of filter should be pushed down after aggregating, other two conditions are ANDed" $CLICKHOUSE_CLIENT --enable_analyzer=1 --convert_query_to_cnf=0 -q " explain actions = 1 select s, y from ( select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y ) where y != 0 and s - 8 and s - 4 settings enable_optimize_predicate_expression=0" | - grep -o "Aggregating\|Filter column\|Filter column: notEquals(__table1.y, 0_UInt8)\|FUNCTION and(minus(__table1.s, 8_UInt8) :: 1, minus(__table1.s, 4_UInt8) :: 2) -> and(notEquals(__table1.y, 0_UInt8), minus(__table1.s, 8_UInt8), minus(__table1.s, 4_UInt8))" + grep -o "Aggregating\|Filter column\|Filter column: notEquals(__table1.y, 0_UInt8)\|FUNCTION and(minus(__table1.s, 8_UInt8) :: 3, minus(__table1.s, 4_UInt8) :: 5) -> and(notEquals(__table1.y, 0_UInt8), minus(__table1.s, 8_UInt8), minus(__table1.s, 4_UInt8))" $CLICKHOUSE_CLIENT -q " select s, y from ( select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y diff --git a/tests/queries/0_stateless/01786_explain_merge_tree.reference b/tests/queries/0_stateless/01786_explain_merge_tree.reference index 3a015d32539..f02dbcb59c9 100644 --- a/tests/queries/0_stateless/01786_explain_merge_tree.reference +++ b/tests/queries/0_stateless/01786_explain_merge_tree.reference @@ -86,11 +86,17 @@ ReadType: InOrder Parts: 1 Granules: 3 + Virtual row conversions + Actions: INPUT :: 0 -> x UInt32 : 0 + Positions: 0 ----------------- ReadFromMergeTree (default.test_index) ReadType: InReverseOrder Parts: 1 Granules: 3 + Virtual row conversions + Actions: INPUT :: 0 -> x UInt32 : 0 + Positions: 0 ReadFromMergeTree (default.idx) Indexes: PrimaryKey @@ -174,11 +180,19 @@ ReadType: InOrder Parts: 1 Granules: 3 + Virtual row conversions + Actions: INPUT : 0 -> x UInt32 : 0 + ALIAS x :: 0 -> __table1.x UInt32 : 1 + Positions: 1 ----------------- ReadFromMergeTree (default.test_index) ReadType: InReverseOrder Parts: 1 Granules: 3 + Virtual row conversions + Actions: INPUT : 0 -> x UInt32 : 0 + ALIAS x :: 0 -> __table1.x UInt32 : 1 + Positions: 1 ReadFromMergeTree (default.idx) Indexes: PrimaryKey diff --git a/tests/queries/0_stateless/01786_explain_merge_tree.sh b/tests/queries/0_stateless/01786_explain_merge_tree.sh index 828012f56bc..9fb764dcd38 100755 --- a/tests/queries/0_stateless/01786_explain_merge_tree.sh +++ b/tests/queries/0_stateless/01786_explain_merge_tree.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) for i in $(seq 0 1) do - CH_CLIENT="$CLICKHOUSE_CLIENT --optimize_move_to_prewhere=1 --convert_query_to_cnf=0 --optimize_read_in_order=1 --enable_analyzer=$i" + CH_CLIENT="$CLICKHOUSE_CLIENT --optimize_move_to_prewhere=1 --convert_query_to_cnf=0 --optimize_read_in_order=1 --read_in_order_use_virtual_row=1 --enable_analyzer=$i" $CH_CLIENT -q "drop table if exists test_index" $CH_CLIENT -q "drop table if exists idx" diff --git a/tests/queries/0_stateless/01825_new_type_json_10.sql b/tests/queries/0_stateless/01825_new_type_json_10.sql index f586cc4477b..9aac35e2c88 100644 --- a/tests/queries/0_stateless/01825_new_type_json_10.sql +++ b/tests/queries/0_stateless/01825_new_type_json_10.sql @@ -1,6 +1,7 @@ -- Tags: no-fasttest SET allow_experimental_json_type = 1; +SET allow_suspicious_types_in_order_by = 1; DROP TABLE IF EXISTS t_json_10; CREATE TABLE t_json_10 (o JSON) ENGINE = Memory; diff --git a/tests/queries/0_stateless/01825_new_type_json_11.sh b/tests/queries/0_stateless/01825_new_type_json_11.sh index f448b7433ab..e9b90af4499 100755 --- a/tests/queries/0_stateless/01825_new_type_json_11.sh +++ b/tests/queries/0_stateless/01825_new_type_json_11.sh @@ -57,8 +57,8 @@ $CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(obj)) as $CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(obj.key_1[]))) as path FROM t_json_11 order by path;" $CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(arrayJoin(obj.key_1[].key_3[])))) as path FROM t_json_11 order by path;" $CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(arrayJoin(arrayJoin(obj.key_1[].key_3[].key_4[]))))) as path FROM t_json_11 order by path;" -$CLICKHOUSE_CLIENT -q "SELECT obj FROM t_json_11 ORDER BY obj.id FORMAT JSONEachRow" -$CLICKHOUSE_CLIENT -q "SELECT obj.key_1[].key_3 FROM t_json_11 ORDER BY obj.id FORMAT JSONEachRow" -$CLICKHOUSE_CLIENT -q "SELECT obj.key_1[].key_3[].key_4[].key_5, obj.key_1[].key_3[].key_7 FROM t_json_11 ORDER BY obj.id" +$CLICKHOUSE_CLIENT -q "SELECT obj FROM t_json_11 ORDER BY obj.id FORMAT JSONEachRow" --allow_suspicious_types_in_order_by 1 +$CLICKHOUSE_CLIENT -q "SELECT obj.key_1[].key_3 FROM t_json_11 ORDER BY obj.id FORMAT JSONEachRow" --allow_suspicious_types_in_order_by 1 +$CLICKHOUSE_CLIENT -q "SELECT obj.key_1[].key_3[].key_4[].key_5, obj.key_1[].key_3[].key_7 FROM t_json_11 ORDER BY obj.id" --allow_suspicious_types_in_order_by 1 $CLICKHOUSE_CLIENT -q "DROP TABLE t_json_11;" diff --git a/tests/queries/0_stateless/01825_new_type_json_12.sh b/tests/queries/0_stateless/01825_new_type_json_12.sh index d7c938d7cd1..fd5b9fddd75 100755 --- a/tests/queries/0_stateless/01825_new_type_json_12.sh +++ b/tests/queries/0_stateless/01825_new_type_json_12.sh @@ -47,8 +47,8 @@ $CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(obj)) as $CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(obj.key_0[]))) as path FROM t_json_12 order by path;" $CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(arrayJoin(obj.key_0[].key_1[])))) as path FROM t_json_12 order by path;" $CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(arrayJoin(arrayJoin(obj.key_0[].key_1[].key_3[]))))) as path FROM t_json_12 order by path;" -$CLICKHOUSE_CLIENT -q "SELECT obj FROM t_json_12 ORDER BY obj.id FORMAT JSONEachRow" --output_format_json_named_tuples_as_objects 1 +$CLICKHOUSE_CLIENT -q "SELECT obj FROM t_json_12 ORDER BY obj.id FORMAT JSONEachRow" --output_format_json_named_tuples_as_objects 1 --allow_suspicious_types_in_order_by 1 $CLICKHOUSE_CLIENT -q "SELECT obj.key_0[].key_1[].key_3[].key_4, obj.key_0[].key_1[].key_3[].key_5, \ - obj.key_0[].key_1[].key_3[].key_6, obj.key_0[].key_1[].key_3[].key_7 FROM t_json_12 ORDER BY obj.id" + obj.key_0[].key_1[].key_3[].key_6, obj.key_0[].key_1[].key_3[].key_7 FROM t_json_12 ORDER BY obj.id" --allow_suspicious_types_in_order_by 1 $CLICKHOUSE_CLIENT -q "DROP TABLE t_json_12;" diff --git a/tests/queries/0_stateless/01825_new_type_json_13.sh b/tests/queries/0_stateless/01825_new_type_json_13.sh index 316e6890d5e..116665e58e3 100755 --- a/tests/queries/0_stateless/01825_new_type_json_13.sh +++ b/tests/queries/0_stateless/01825_new_type_json_13.sh @@ -39,12 +39,12 @@ EOF $CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(obj)) as path FROM t_json_13 order by path;" $CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(obj.key1[]))) as path FROM t_json_13 order by path;" -$CLICKHOUSE_CLIENT -q "SELECT obj FROM t_json_13 ORDER BY obj.id FORMAT JSONEachRow" --output_format_json_named_tuples_as_objects 1 +$CLICKHOUSE_CLIENT -q "SELECT obj FROM t_json_13 ORDER BY obj.id FORMAT JSONEachRow" --output_format_json_named_tuples_as_objects 1 --allow_suspicious_types_in_order_by 1 $CLICKHOUSE_CLIENT -q "SELECT \ obj.key_1.key_2.key_3.key_8, \ obj.key_1.key_2.key_3.key_4.key_5, \ obj.key_1.key_2.key_3.key_4.key_6, \ obj.key_1.key_2.key_3.key_4.key_7 \ -FROM t_json_13 ORDER BY obj.id" +FROM t_json_13 ORDER BY obj.id" --allow_suspicious_types_in_order_by 1 $CLICKHOUSE_CLIENT -q "DROP TABLE t_json_13;" diff --git a/tests/queries/0_stateless/01825_new_type_json_6.sh b/tests/queries/0_stateless/01825_new_type_json_6.sh index 6b9a7e71f50..a2102636c42 100755 --- a/tests/queries/0_stateless/01825_new_type_json_6.sh +++ b/tests/queries/0_stateless/01825_new_type_json_6.sh @@ -54,6 +54,6 @@ EOF $CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(data)) as path FROM t_json_6 order by path;" $CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(data.out[]))) as path FROM t_json_6 order by path;" $CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(arrayJoin(data.out[].outputs[])))) as path FROM t_json_6 order by path;" -$CLICKHOUSE_CLIENT -q "SELECT data.key, data.out[].type, data.out[].value, data.out[].outputs[].index, data.out[].outputs[].n FROM t_json_6 ORDER BY data.key" +$CLICKHOUSE_CLIENT -q "SELECT data.key, data.out[].type, data.out[].value, data.out[].outputs[].index, data.out[].outputs[].n FROM t_json_6 ORDER BY data.key" --allow_suspicious_types_in_order_by 1 $CLICKHOUSE_CLIENT -q "DROP TABLE t_json_6;" diff --git a/tests/queries/0_stateless/01825_new_type_json_7.sh b/tests/queries/0_stateless/01825_new_type_json_7.sh index 36483175df6..b6ea46f5ff8 100755 --- a/tests/queries/0_stateless/01825_new_type_json_7.sh +++ b/tests/queries/0_stateless/01825_new_type_json_7.sh @@ -25,6 +25,6 @@ cat <&1 | grep -q "Code: 62. DB::Exception: Syntax error: failed at position 48 ('test'): test.native.zst FORMAT Native. Expected string literal." && echo 'OK' || echo 'FAIL' ||: - +$CLICKHOUSE_LOCAL --query="SELECT number FROM system.numbers INTO OUTFILE test.native.zst FORMAT Native" 2>&1 | grep -q "Code: 62. DB::Exception: Syntax error: failed at position 48 ('test'): test.native.zst FORMAT Native." && echo 'OK' || echo 'FAIL' ||: diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 2ea62444cff..ef5a2c6665f 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -485,6 +485,8 @@ CREATE TABLE system.parts `data_version` UInt64, `primary_key_bytes_in_memory` UInt64, `primary_key_bytes_in_memory_allocated` UInt64, + `index_granularity_bytes_in_memory` UInt64, + `index_granularity_bytes_in_memory_allocated` UInt64, `is_frozen` UInt8, `database` String, `table` String, diff --git a/tests/queries/0_stateless/02125_query_views_log.sql b/tests/queries/0_stateless/02125_query_views_log.sql index ba50902ebea..08e9c73a165 100644 --- a/tests/queries/0_stateless/02125_query_views_log.sql +++ b/tests/queries/0_stateless/02125_query_views_log.sql @@ -1,3 +1,5 @@ +SET output_format_pretty_single_large_number_tip_threshold = 0; + drop table if exists src; drop table if exists dst; drop table if exists mv1; diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference index d608364e01b..cb96a7167da 100644 --- a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference @@ -172,7 +172,8 @@ ExpressionTransform (Expression) ExpressionTransform (ReadFromMergeTree) - MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 + VirtualRowTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 2020-10-10 00:00:00 0.01 2020-10-10 00:00:00 0.01 2020-10-10 00:00:00 0.01 @@ -186,7 +187,8 @@ ExpressionTransform (Expression) ExpressionTransform (ReadFromMergeTree) - MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 + VirtualRowTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 2020-10-10 00:00:00 0.01 2020-10-10 00:00:00 0.01 2020-10-10 00:00:00 0.01 diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql index 7bbdecf5501..4cc05203b6a 100644 --- a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql @@ -2,6 +2,7 @@ SET max_threads=0; SET optimize_read_in_order=1; SET optimize_trivial_insert_select = 1; SET read_in_order_two_level_merge_threshold=100; +SET read_in_order_use_virtual_row = 1; DROP TABLE IF EXISTS t_read_in_order; diff --git a/tests/queries/0_stateless/02184_hash_functions_and_ip_types.sql b/tests/queries/0_stateless/02184_hash_functions_and_ip_types.sql index e7d1909cae6..22b59a16255 100644 --- a/tests/queries/0_stateless/02184_hash_functions_and_ip_types.sql +++ b/tests/queries/0_stateless/02184_hash_functions_and_ip_types.sql @@ -1,5 +1,6 @@ -- Tags: no-fasttest +SET output_format_pretty_single_large_number_tip_threshold = 0; SET enable_analyzer = 1; SELECT diff --git a/tests/queries/0_stateless/03231_dynamic_not_safe_primary_key.reference b/tests/queries/0_stateless/02354_vector_search_adaptive_index_granularity.reference similarity index 100% rename from tests/queries/0_stateless/03231_dynamic_not_safe_primary_key.reference rename to tests/queries/0_stateless/02354_vector_search_adaptive_index_granularity.reference diff --git a/tests/queries/0_stateless/02354_vector_search_adaptive_index_granularity.sql b/tests/queries/0_stateless/02354_vector_search_adaptive_index_granularity.sql new file mode 100644 index 00000000000..208b5b7a874 --- /dev/null +++ b/tests/queries/0_stateless/02354_vector_search_adaptive_index_granularity.sql @@ -0,0 +1,20 @@ +-- Tags: no-fasttest, no-ordinary-database + +-- Tests that vector similarity indexes cannot be created with index_granularity_bytes = 0 + +SET allow_experimental_vector_similarity_index = 1; + +DROP TABLE IF EXISTS tab; + +-- If adaptive index granularity is disabled, certain vector search queries with PREWHERE run into LOGICAL_ERRORs. +-- SET allow_experimental_vector_similarity_index = 1; +-- CREATE TABLE tab (`id` Int32, `vec` Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance') GRANULARITY 100000000) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity_bytes = 0; +-- INSERT INTO tab SELECT number, [toFloat32(number), 0.] FROM numbers(10000); +-- WITH [1., 0.] AS reference_vec SELECT id, L2Distance(vec, reference_vec) FROM tab PREWHERE toLowCardinality(10) ORDER BY L2Distance(vec, reference_vec) ASC LIMIT 100; +-- As a workaround, force enabled adaptive index granularity for now (it is the default anyways). +CREATE TABLE tab(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance')) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity_bytes = 0; -- { serverError INVALID_SETTING_VALUE } + +CREATE TABLE tab(id Int32, vec Array(Float32)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity_bytes = 0; +ALTER TABLE tab ADD INDEX vec_idx1(vec) TYPE vector_similarity('hnsw', 'cosineDistance'); -- { serverError INVALID_SETTING_VALUE } + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/02354_vector_search_and_other_skipping_indexes.reference b/tests/queries/0_stateless/02354_vector_search_and_other_skipping_indexes.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02354_vector_search_and_other_skipping_indexes.sql b/tests/queries/0_stateless/02354_vector_search_and_other_skipping_indexes.sql new file mode 100644 index 00000000000..386d3b6e26e --- /dev/null +++ b/tests/queries/0_stateless/02354_vector_search_and_other_skipping_indexes.sql @@ -0,0 +1,20 @@ +-- Tags: no-fasttest, no-ordinary-database + +SET allow_experimental_vector_similarity_index = 1; + +-- Usage of vector similarity index and further skipping indexes on the same table (issue #71381) + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab( + val String, + vec Array(Float32), + INDEX ann_idx vec TYPE vector_similarity('hnsw', 'cosineDistance'), + INDEX set_idx val TYPE set(100) +) +ENGINE = MergeTree() +ORDER BY tuple(); + +INSERT INTO tab VALUES ('hello world', [0.0]); + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/02354_vector_search_bugs.sql b/tests/queries/0_stateless/02354_vector_search_bugs.sql deleted file mode 100644 index d55bdb88a76..00000000000 --- a/tests/queries/0_stateless/02354_vector_search_bugs.sql +++ /dev/null @@ -1,119 +0,0 @@ --- Tags: no-fasttest, no-ordinary-database - --- Tests various bugs and special cases for vector indexes. - -SET allow_experimental_vector_similarity_index = 1; -SET enable_analyzer = 1; -- 0 vs. 1 produce slightly different error codes, make it future-proof - -DROP TABLE IF EXISTS tab; - -SELECT 'Rejects INSERTs of Arrays with different sizes'; - -CREATE TABLE tab(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance')) ENGINE = MergeTree ORDER BY id; -INSERT INTO tab values (0, [2.2, 2.3]) (1, [3.1, 3.2, 3.3]); -- { serverError INCORRECT_DATA } -DROP TABLE tab; - -SELECT 'Issue #52258: Empty Arrays or Arrays with default values are rejected'; - -CREATE TABLE tab (id UInt64, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance')) ENGINE = MergeTree() ORDER BY id; -INSERT INTO tab VALUES (1, []); -- { serverError INCORRECT_DATA } -INSERT INTO tab (id) VALUES (1); -- { serverError INCORRECT_DATA } -DROP TABLE tab; - -SELECT 'It is possible to create parts with different Array vector sizes but there will be an error at query time'; - -CREATE TABLE tab(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance')) ENGINE = MergeTree ORDER BY id; -SYSTEM STOP MERGES tab; -INSERT INTO tab values (0, [2.2, 2.3]) (1, [3.1, 3.2]); -INSERT INTO tab values (2, [2.2, 2.3, 2.4]) (3, [3.1, 3.2, 3.3]); - -WITH [0.0, 2.0] AS reference_vec -SELECT id, vec, L2Distance(vec, reference_vec) -FROM tab -ORDER BY L2Distance(vec, reference_vec) -LIMIT 3; -- { serverError SIZES_OF_ARRAYS_DONT_MATCH } - -DROP TABLE tab; - -SELECT 'Correctness of index with > 1 mark'; - -CREATE TABLE tab(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance')) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8192; -INSERT INTO tab SELECT number, [toFloat32(number), 0.0] from numbers(10000); - -WITH [1.0, 0.0] AS reference_vec -SELECT id, vec, L2Distance(vec, reference_vec) -FROM tab -ORDER BY L2Distance(vec, reference_vec) -LIMIT 1; - -WITH [9000.0, 0.0] AS reference_vec -SELECT id, vec, L2Distance(vec, reference_vec) -FROM tab -ORDER BY L2Distance(vec, reference_vec) -LIMIT 1; - -DROP TABLE tab; - -SELECT 'Issue #69085: Reference vector computed by a subquery'; - -CREATE TABLE tab(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'cosineDistance', 'f16', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; -INSERT INTO tab VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); - --- works -EXPLAIN indexes = 1 -WITH [0., 2.] AS reference_vec -SELECT - id, - vec, - cosineDistance(vec, reference_vec) AS distance -FROM tab -ORDER BY distance -LIMIT 1 -SETTINGS enable_analyzer = 0; - --- does not work -EXPLAIN indexes = 1 -WITH ( - SELECT vec - FROM tab - LIMIT 1 -) AS reference_vec -SELECT - id, - vec, - cosineDistance(vec, reference_vec) AS distance -FROM tab -ORDER BY distance -LIMIT 1 -SETTINGS enable_analyzer = 0; - --- does not work as well -EXPLAIN indexes = 1 -WITH ( - SELECT [0., 2.] -) AS reference_vec -SELECT - id, - vec, - cosineDistance(vec, reference_vec) AS distance -FROM tab -ORDER BY distance -LIMIT 1 -SETTINGS enable_analyzer = 0; - -DROP TABLE tab; - -SELECT 'index_granularity_bytes = 0 is disallowed'; - --- If adaptive index granularity is disabled, certain vector search queries with PREWHERE run into LOGICAL_ERRORs. --- SET allow_experimental_vector_similarity_index = 1; --- CREATE TABLE tab (`id` Int32, `vec` Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance') GRANULARITY 100000000) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity_bytes = 0; --- INSERT INTO tab SELECT number, [toFloat32(number), 0.] FROM numbers(10000); --- WITH [1., 0.] AS reference_vec SELECT id, L2Distance(vec, reference_vec) FROM tab PREWHERE toLowCardinality(10) ORDER BY L2Distance(vec, reference_vec) ASC LIMIT 100; --- As a workaround, force enabled adaptive index granularity for now (it is the default anyways). -CREATE TABLE tab(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance')) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity_bytes = 0; -- { serverError INVALID_SETTING_VALUE } - -CREATE TABLE tab(id Int32, vec Array(Float32)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity_bytes = 0; -ALTER TABLE tab ADD INDEX vec_idx1(vec) TYPE vector_similarity('hnsw', 'cosineDistance'); -- { serverError INVALID_SETTING_VALUE } - -DROP TABLE tab; diff --git a/tests/queries/0_stateless/02354_vector_search_different_array_sizes.reference b/tests/queries/0_stateless/02354_vector_search_different_array_sizes.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02354_vector_search_different_array_sizes.sql b/tests/queries/0_stateless/02354_vector_search_different_array_sizes.sql new file mode 100644 index 00000000000..41b9d7869e4 --- /dev/null +++ b/tests/queries/0_stateless/02354_vector_search_different_array_sizes.sql @@ -0,0 +1,24 @@ +-- Tags: no-fasttest, no-ordinary-database + +SET allow_experimental_vector_similarity_index = 1; +SET enable_analyzer = 1; -- 0 vs. 1 produce slightly different error codes, make it future-proof + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance')) ENGINE = MergeTree ORDER BY id; + +-- Vector similarity indexes reject INSERTs of Arrays with different sizes +INSERT INTO tab values (0, [2.2, 2.3]) (1, [3.1, 3.2, 3.3]); -- { serverError INCORRECT_DATA } + +-- It is possible to create parts with different Array vector sizes but there will be an error at query time +SYSTEM STOP MERGES tab; +INSERT INTO tab values (0, [2.2, 2.3]) (1, [3.1, 3.2]); +INSERT INTO tab values (2, [2.2, 2.3, 2.4]) (3, [3.1, 3.2, 3.3]); + +WITH [0.0, 2.0] AS reference_vec +SELECT id, vec, L2Distance(vec, reference_vec) +FROM tab +ORDER BY L2Distance(vec, reference_vec) +LIMIT 3; -- { serverError SIZES_OF_ARRAYS_DONT_MATCH } + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/02354_vector_search_empty_arrays_or_default_values.reference b/tests/queries/0_stateless/02354_vector_search_empty_arrays_or_default_values.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02354_vector_search_empty_arrays_or_default_values.sql b/tests/queries/0_stateless/02354_vector_search_empty_arrays_or_default_values.sql new file mode 100644 index 00000000000..e24b1a527be --- /dev/null +++ b/tests/queries/0_stateless/02354_vector_search_empty_arrays_or_default_values.sql @@ -0,0 +1,13 @@ +-- Tags: no-fasttest, no-ordinary-database + +SET allow_experimental_vector_similarity_index = 1; + +-- Vector similarity indexes must reject empty Arrays or Arrays with default values (issue #52258) + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab (id UInt64, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance')) ENGINE = MergeTree() ORDER BY id; +INSERT INTO tab VALUES (1, []); -- { serverError INCORRECT_DATA } +INSERT INTO tab (id) VALUES (1); -- { serverError INCORRECT_DATA } + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/02354_vector_search_expansion_search.sql b/tests/queries/0_stateless/02354_vector_search_expansion_search.sql index fcbe9ee42b9..427148b829f 100644 --- a/tests/queries/0_stateless/02354_vector_search_expansion_search.sql +++ b/tests/queries/0_stateless/02354_vector_search_expansion_search.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest, long, no-asan, no-asan, no-ubsan, no-debug +-- Tags: no-fasttest, long, no-asan, no-ubsan, no-debug -- ^^ Disable test for slow builds: generating data takes time but a sufficiently large data set -- is necessary for different hnsw_candidate_list_size_for_search settings to make a difference @@ -14,7 +14,7 @@ CREATE TABLE tab(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similar -- Generate random values but with a fixed seed (conceptually), so that the data is deterministic. -- Unfortunately, no random functions in ClickHouse accepts a seed. Instead, abuse the numbers table + hash functions to provide -- deterministic randomness. -INSERT INTO tab SELECT number, [sipHash64(number)/18446744073709551615, wyHash64(number)/18446744073709551615] FROM numbers(370000); -- 18446744073709551615 is the biggest UInt64 +INSERT INTO tab SELECT number, [sipHash64(number)/18446744073709551615, wyHash64(number)/18446744073709551615] FROM numbers(660000); -- 18446744073709551615 is the biggest UInt64 -- hnsw_candidate_list_size_for_search = 0 is illegal WITH [0.5, 0.5] AS reference_vec diff --git a/tests/queries/0_stateless/02354_vector_search_legacy_index_creation_syntax.reference b/tests/queries/0_stateless/02354_vector_search_legacy_index_creation_syntax.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02354_vector_search_legacy_index_creation_syntax.sql b/tests/queries/0_stateless/02354_vector_search_legacy_index_creation_syntax.sql new file mode 100644 index 00000000000..e5dbc6aa6a9 --- /dev/null +++ b/tests/queries/0_stateless/02354_vector_search_legacy_index_creation_syntax.sql @@ -0,0 +1,13 @@ +-- Tags: no-fasttest, no-ordinary-database + +-- Tests the legacy syntax to create vector similarity indexes before #70616. +-- Support for this syntax can be removed after mid-2025. + +SET allow_experimental_vector_similarity_index = 1; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'cosineDistance', 'f32', 42, 99, 113)) ENGINE = MergeTree ORDER BY id; -- Note the 6th parameter: 133 + +DROP TABLE tab; + diff --git a/tests/queries/0_stateless/02354_vector_search_multiple_marks.reference b/tests/queries/0_stateless/02354_vector_search_multiple_marks.reference new file mode 100644 index 00000000000..117bf2cead8 --- /dev/null +++ b/tests/queries/0_stateless/02354_vector_search_multiple_marks.reference @@ -0,0 +1,2 @@ +1 [1,0] 0 +9000 [9000,0] 0 diff --git a/tests/queries/0_stateless/02354_vector_search_multiple_marks.sql b/tests/queries/0_stateless/02354_vector_search_multiple_marks.sql new file mode 100644 index 00000000000..fb99dd2361c --- /dev/null +++ b/tests/queries/0_stateless/02354_vector_search_multiple_marks.sql @@ -0,0 +1,25 @@ +-- Tags: no-fasttest, no-ordinary-database + +-- Tests correctness of vector similarity index with > 1 mark + +SET allow_experimental_vector_similarity_index = 1; +SET enable_analyzer = 0; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance')) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8192; +INSERT INTO tab SELECT number, [toFloat32(number), 0.0] from numbers(10000); + +WITH [1.0, 0.0] AS reference_vec +SELECT id, vec, L2Distance(vec, reference_vec) +FROM tab +ORDER BY L2Distance(vec, reference_vec) +LIMIT 1; + +WITH [9000.0, 0.0] AS reference_vec +SELECT id, vec, L2Distance(vec, reference_vec) +FROM tab +ORDER BY L2Distance(vec, reference_vec) +LIMIT 1; + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/02354_vector_search_queries.reference b/tests/queries/0_stateless/02354_vector_search_queries.reference index 223a18b57bf..cf80f46f53c 100644 --- a/tests/queries/0_stateless/02354_vector_search_queries.reference +++ b/tests/queries/0_stateless/02354_vector_search_queries.reference @@ -67,7 +67,7 @@ Expression (Projection) Condition: true Parts: 1/1 Granules: 4/4 --- Non-default quantization +-- Test all distance metrics x all quantization 1 [2,3.2] 2.3323807824711897 4 [2.4,5.2] 3.9999999046325727 2 [4.2,3.4] 4.427188573446585 @@ -75,7 +75,7 @@ Expression (Projection) Limit (preliminary LIMIT (without OFFSET)) Sorting (Sorting for ORDER BY) Expression (Before ORDER BY) - ReadFromMergeTree (default.tab_f64) + ReadFromMergeTree (default.tab_l2_f64) Indexes: PrimaryKey Condition: true @@ -93,7 +93,7 @@ Expression (Projection) Limit (preliminary LIMIT (without OFFSET)) Sorting (Sorting for ORDER BY) Expression (Before ORDER BY) - ReadFromMergeTree (default.tab_f32) + ReadFromMergeTree (default.tab_l2_f32) Indexes: PrimaryKey Condition: true @@ -111,7 +111,7 @@ Expression (Projection) Limit (preliminary LIMIT (without OFFSET)) Sorting (Sorting for ORDER BY) Expression (Before ORDER BY) - ReadFromMergeTree (default.tab_f16) + ReadFromMergeTree (default.tab_l2_f16) Indexes: PrimaryKey Condition: true @@ -129,7 +129,7 @@ Expression (Projection) Limit (preliminary LIMIT (without OFFSET)) Sorting (Sorting for ORDER BY) Expression (Before ORDER BY) - ReadFromMergeTree (default.tab_bf16) + ReadFromMergeTree (default.tab_l2_bf16) Indexes: PrimaryKey Condition: true @@ -147,7 +147,97 @@ Expression (Projection) Limit (preliminary LIMIT (without OFFSET)) Sorting (Sorting for ORDER BY) Expression (Before ORDER BY) - ReadFromMergeTree (default.tab_i8) + ReadFromMergeTree (default.tab_l2_i8) + Indexes: + PrimaryKey + Condition: true + Parts: 1/1 + Granules: 4/4 + Skip + Name: idx + Description: vector_similarity GRANULARITY 2 + Parts: 1/1 + Granules: 3/4 +6 [1,9.3] 0.005731362878640178 +4 [2.4,5.2] 0.09204062768384846 +1 [2,3.2] 0.15200169244542905 +Expression (Projection) + Limit (preliminary LIMIT (without OFFSET)) + Sorting (Sorting for ORDER BY) + Expression (Before ORDER BY) + ReadFromMergeTree (default.tab_cos_f64) + Indexes: + PrimaryKey + Condition: true + Parts: 1/1 + Granules: 4/4 + Skip + Name: idx + Description: vector_similarity GRANULARITY 2 + Parts: 1/1 + Granules: 3/4 +6 [1,9.3] 0.005731362878640178 +4 [2.4,5.2] 0.09204062768384846 +1 [2,3.2] 0.15200169244542905 +Expression (Projection) + Limit (preliminary LIMIT (without OFFSET)) + Sorting (Sorting for ORDER BY) + Expression (Before ORDER BY) + ReadFromMergeTree (default.tab_cos_f32) + Indexes: + PrimaryKey + Condition: true + Parts: 1/1 + Granules: 4/4 + Skip + Name: idx + Description: vector_similarity GRANULARITY 2 + Parts: 1/1 + Granules: 3/4 +6 [1,9.3] 0.005731362878640178 +4 [2.4,5.2] 0.09204062768384846 +1 [2,3.2] 0.15200169244542905 +Expression (Projection) + Limit (preliminary LIMIT (without OFFSET)) + Sorting (Sorting for ORDER BY) + Expression (Before ORDER BY) + ReadFromMergeTree (default.tab_cos_f16) + Indexes: + PrimaryKey + Condition: true + Parts: 1/1 + Granules: 4/4 + Skip + Name: idx + Description: vector_similarity GRANULARITY 2 + Parts: 1/1 + Granules: 3/4 +6 [1,9.3] 0.005731362878640178 +4 [2.4,5.2] 0.09204062768384846 +1 [2,3.2] 0.15200169244542905 +Expression (Projection) + Limit (preliminary LIMIT (without OFFSET)) + Sorting (Sorting for ORDER BY) + Expression (Before ORDER BY) + ReadFromMergeTree (default.tab_cos_bf16) + Indexes: + PrimaryKey + Condition: true + Parts: 1/1 + Granules: 4/4 + Skip + Name: idx + Description: vector_similarity GRANULARITY 2 + Parts: 1/1 + Granules: 3/4 +6 [1,9.3] 0.005731362878640178 +4 [2.4,5.2] 0.09204062768384846 +1 [2,3.2] 0.15200169244542905 +Expression (Projection) + Limit (preliminary LIMIT (without OFFSET)) + Sorting (Sorting for ORDER BY) + Expression (Before ORDER BY) + ReadFromMergeTree (default.tab_cos_i8) Indexes: PrimaryKey Condition: true diff --git a/tests/queries/0_stateless/02354_vector_search_queries.sql b/tests/queries/0_stateless/02354_vector_search_queries.sql index 71b8a1e520a..0941f9a43d6 100644 --- a/tests/queries/0_stateless/02354_vector_search_queries.sql +++ b/tests/queries/0_stateless/02354_vector_search_queries.sql @@ -81,88 +81,181 @@ SETTINGS max_limit_for_ann_queries = 2; -- LIMIT 3 > 2 --> don't use the ann ind DROP TABLE tab; -SELECT '-- Non-default quantization'; -CREATE TABLE tab_f64(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance', 'f64', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; -CREATE TABLE tab_f32(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance', 'f32', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; -CREATE TABLE tab_f16(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance', 'f16', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; -CREATE TABLE tab_bf16(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance', 'bf16', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; -CREATE TABLE tab_i8(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance', 'i8', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; -INSERT INTO tab_f64 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); -INSERT INTO tab_f32 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); -INSERT INTO tab_f16 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); -INSERT INTO tab_bf16 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); -INSERT INTO tab_i8 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); +SELECT '-- Test all distance metrics x all quantization'; + +DROP TABLE IF EXISTS tab_l2_f64; +DROP TABLE IF EXISTS tab_l2_f32; +DROP TABLE IF EXISTS tab_l2_f16; +DROP TABLE IF EXISTS tab_l2_bf16; +DROP TABLE IF EXISTS tab_l2_i8; +DROP TABLE IF EXISTS tab_cos_f64; +DROP TABLE IF EXISTS tab_cos_f32; +DROP TABLE IF EXISTS tab_cos_f16; +DROP TABLE IF EXISTS tab_cos_bf16; +DROP TABLE IF EXISTS tab_cos_i8; + +CREATE TABLE tab_l2_f64(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance', 'f64', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; +CREATE TABLE tab_l2_f32(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance', 'f32', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; +CREATE TABLE tab_l2_f16(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance', 'f16', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; +CREATE TABLE tab_l2_bf16(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance', 'bf16', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; +CREATE TABLE tab_l2_i8(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance', 'i8', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; +CREATE TABLE tab_cos_f64(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'cosineDistance', 'f64', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; +CREATE TABLE tab_cos_f32(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'cosineDistance', 'f32', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; +CREATE TABLE tab_cos_f16(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'cosineDistance', 'f16', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; +CREATE TABLE tab_cos_bf16(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'cosineDistance', 'bf16', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; +CREATE TABLE tab_cos_i8(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'cosineDistance', 'i8', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; + +INSERT INTO tab_l2_f64 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); +INSERT INTO tab_l2_f32 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); +INSERT INTO tab_l2_f16 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); +INSERT INTO tab_l2_bf16 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); +INSERT INTO tab_l2_i8 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); +INSERT INTO tab_cos_f64 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); +INSERT INTO tab_cos_f32 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); +INSERT INTO tab_cos_f16 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); +INSERT INTO tab_cos_bf16 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); +INSERT INTO tab_cos_i8 VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); WITH [0.0, 2.0] AS reference_vec SELECT id, vec, L2Distance(vec, reference_vec) -FROM tab_f64 +FROM tab_l2_f64 ORDER BY L2Distance(vec, reference_vec) LIMIT 3; EXPLAIN indexes = 1 WITH [0.0, 2.0] AS reference_vec SELECT id, vec, L2Distance(vec, reference_vec) -FROM tab_f64 +FROM tab_l2_f64 ORDER BY L2Distance(vec, reference_vec) LIMIT 3; WITH [0.0, 2.0] AS reference_vec SELECT id, vec, L2Distance(vec, reference_vec) -FROM tab_f32 +FROM tab_l2_f32 ORDER BY L2Distance(vec, reference_vec) LIMIT 3; EXPLAIN indexes = 1 WITH [0.0, 2.0] AS reference_vec SELECT id, vec, L2Distance(vec, reference_vec) -FROM tab_f32 +FROM tab_l2_f32 ORDER BY L2Distance(vec, reference_vec) LIMIT 3; WITH [0.0, 2.0] AS reference_vec SELECT id, vec, L2Distance(vec, reference_vec) -FROM tab_f16 +FROM tab_l2_f16 ORDER BY L2Distance(vec, reference_vec) LIMIT 3; EXPLAIN indexes = 1 WITH [0.0, 2.0] AS reference_vec SELECT id, vec, L2Distance(vec, reference_vec) -FROM tab_f16 +FROM tab_l2_f16 ORDER BY L2Distance(vec, reference_vec) LIMIT 3; WITH [0.0, 2.0] AS reference_vec SELECT id, vec, L2Distance(vec, reference_vec) -FROM tab_bf16 +FROM tab_l2_bf16 ORDER BY L2Distance(vec, reference_vec) LIMIT 3; EXPLAIN indexes = 1 WITH [0.0, 2.0] AS reference_vec SELECT id, vec, L2Distance(vec, reference_vec) -FROM tab_bf16 +FROM tab_l2_bf16 ORDER BY L2Distance(vec, reference_vec) LIMIT 3; WITH [0.0, 2.0] AS reference_vec SELECT id, vec, L2Distance(vec, reference_vec) -FROM tab_i8 +FROM tab_l2_i8 ORDER BY L2Distance(vec, reference_vec) LIMIT 3; EXPLAIN indexes = 1 WITH [0.0, 2.0] AS reference_vec SELECT id, vec, L2Distance(vec, reference_vec) -FROM tab_i8 +FROM tab_l2_i8 ORDER BY L2Distance(vec, reference_vec) LIMIT 3; -DROP TABLE tab_f64; -DROP TABLE tab_f32; -DROP TABLE tab_f16; -DROP TABLE tab_bf16; -DROP TABLE tab_i8; +WITH [0.0, 2.0] AS reference_vec +SELECT id, vec, cosineDistance(vec, reference_vec) +FROM tab_cos_f64 +ORDER BY cosineDistance(vec, reference_vec) +LIMIT 3; + +EXPLAIN indexes = 1 +WITH [0.0, 2.0] AS reference_vec +SELECT id, vec, cosineDistance(vec, reference_vec) +FROM tab_cos_f64 +ORDER BY cosineDistance(vec, reference_vec) +LIMIT 3; + +WITH [0.0, 2.0] AS reference_vec +SELECT id, vec, cosineDistance(vec, reference_vec) +FROM tab_cos_f32 +ORDER BY cosineDistance(vec, reference_vec) +LIMIT 3; + +EXPLAIN indexes = 1 +WITH [0.0, 2.0] AS reference_vec +SELECT id, vec, cosineDistance(vec, reference_vec) +FROM tab_cos_f32 +ORDER BY cosineDistance(vec, reference_vec) +LIMIT 3; + +WITH [0.0, 2.0] AS reference_vec +SELECT id, vec, cosineDistance(vec, reference_vec) +FROM tab_cos_f16 +ORDER BY cosineDistance(vec, reference_vec) +LIMIT 3; + +EXPLAIN indexes = 1 +WITH [0.0, 2.0] AS reference_vec +SELECT id, vec, cosineDistance(vec, reference_vec) +FROM tab_cos_f16 +ORDER BY cosineDistance(vec, reference_vec) +LIMIT 3; + +WITH [0.0, 2.0] AS reference_vec +SELECT id, vec, cosineDistance(vec, reference_vec) +FROM tab_cos_bf16 +ORDER BY cosineDistance(vec, reference_vec) +LIMIT 3; + +EXPLAIN indexes = 1 +WITH [0.0, 2.0] AS reference_vec +SELECT id, vec, cosineDistance(vec, reference_vec) +FROM tab_cos_bf16 +ORDER BY cosineDistance(vec, reference_vec) +LIMIT 3; + +WITH [0.0, 2.0] AS reference_vec +SELECT id, vec, cosineDistance(vec, reference_vec) +FROM tab_cos_i8 +ORDER BY cosineDistance(vec, reference_vec) +LIMIT 3; + +EXPLAIN indexes = 1 +WITH [0.0, 2.0] AS reference_vec +SELECT id, vec, cosineDistance(vec, reference_vec) +FROM tab_cos_i8 +ORDER BY cosineDistance(vec, reference_vec) +LIMIT 3; + +DROP TABLE tab_l2_f64; +DROP TABLE tab_l2_f32; +DROP TABLE tab_l2_f16; +DROP TABLE tab_l2_bf16; +DROP TABLE tab_l2_i8; +DROP TABLE tab_cos_f64; +DROP TABLE tab_cos_f32; +DROP TABLE tab_cos_f16; +DROP TABLE tab_cos_bf16; +DROP TABLE tab_cos_i8; SELECT '-- Index on Array(Float64) column'; CREATE TABLE tab(id Int32, vec Array(Float64), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance') GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; diff --git a/tests/queries/0_stateless/02354_vector_search_bugs.reference b/tests/queries/0_stateless/02354_vector_search_subquery.reference similarity index 72% rename from tests/queries/0_stateless/02354_vector_search_bugs.reference rename to tests/queries/0_stateless/02354_vector_search_subquery.reference index 9b610cf543a..3b4e2d9ef17 100644 --- a/tests/queries/0_stateless/02354_vector_search_bugs.reference +++ b/tests/queries/0_stateless/02354_vector_search_subquery.reference @@ -1,10 +1,3 @@ -Rejects INSERTs of Arrays with different sizes -Issue #52258: Empty Arrays or Arrays with default values are rejected -It is possible to create parts with different Array vector sizes but there will be an error at query time -Correctness of index with > 1 mark -1 [1,0] 0 -9000 [9000,0] 0 -Issue #69085: Reference vector computed by a subquery Expression (Projection) Limit (preliminary LIMIT (without OFFSET)) Sorting (Sorting for ORDER BY) @@ -40,4 +33,3 @@ Expression (Projection) Condition: true Parts: 1/1 Granules: 4/4 -index_granularity_bytes = 0 is disallowed diff --git a/tests/queries/0_stateless/02354_vector_search_subquery.sql b/tests/queries/0_stateless/02354_vector_search_subquery.sql new file mode 100644 index 00000000000..65ad0dbcd97 --- /dev/null +++ b/tests/queries/0_stateless/02354_vector_search_subquery.sql @@ -0,0 +1,52 @@ +-- Tags: no-fasttest, no-ordinary-database + +SET allow_experimental_vector_similarity_index = 1; +SET enable_analyzer = 0; + +-- Reference vector for vector search is computed by a subquery (issue #69085) + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'cosineDistance', 'f16', 0, 0) GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; +INSERT INTO tab VALUES (0, [4.6, 2.3]), (1, [2.0, 3.2]), (2, [4.2, 3.4]), (3, [5.3, 2.9]), (4, [2.4, 5.2]), (5, [5.3, 2.3]), (6, [1.0, 9.3]), (7, [5.5, 4.7]), (8, [6.4, 3.5]), (9, [5.3, 2.5]), (10, [6.4, 3.4]), (11, [6.4, 3.2]); + +-- works +EXPLAIN indexes = 1 +WITH [0., 2.] AS reference_vec +SELECT + id, + vec, + cosineDistance(vec, reference_vec) AS distance +FROM tab +ORDER BY distance +LIMIT 1; + +-- does not work +EXPLAIN indexes = 1 +WITH ( + SELECT vec + FROM tab + LIMIT 1 +) AS reference_vec +SELECT + id, + vec, + cosineDistance(vec, reference_vec) AS distance +FROM tab +ORDER BY distance +LIMIT 1; + +-- does not work as well +EXPLAIN indexes = 1 +WITH ( + SELECT [0., 2.] +) AS reference_vec +SELECT + id, + vec, + cosineDistance(vec, reference_vec) AS distance +FROM tab +ORDER BY distance +LIMIT 1; + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/02421_new_type_json_async_insert.sh b/tests/queries/0_stateless/02421_new_type_json_async_insert.sh index b23470a4179..3c863d83f2d 100755 --- a/tests/queries/0_stateless/02421_new_type_json_async_insert.sh +++ b/tests/queries/0_stateless/02421_new_type_json_async_insert.sh @@ -17,5 +17,5 @@ $CLICKHOUSE_CLIENT --async_insert=1 --wait_for_async_insert=1 -q 'INSERT INTO t_ wait -$CLICKHOUSE_CLIENT -q "SELECT data.k1 FROM t_json_async_insert ORDER BY data.k1" +$CLICKHOUSE_CLIENT -q "SELECT data.k1 FROM t_json_async_insert ORDER BY data.k1" --allow_suspicious_types_in_order_by 1 $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_async_insert" diff --git a/tests/queries/0_stateless/02494_query_cache_normalize_ast.sql b/tests/queries/0_stateless/02494_query_cache_normalize_ast.sql index 1dbb3ef8158..cb53c4db7de 100644 --- a/tests/queries/0_stateless/02494_query_cache_normalize_ast.sql +++ b/tests/queries/0_stateless/02494_query_cache_normalize_ast.sql @@ -7,7 +7,7 @@ SYSTEM DROP QUERY CACHE; -- Run query whose result gets cached in the query cache. -- Besides "use_query_cache", pass two more knobs (one QC-specific knob and one non-QC-specific knob). We just care -- *that* they are passed and not about their effect. -SELECT 1 SETTINGS use_query_cache = true, query_cache_store_results_of_queries_with_nondeterministic_functions = true, max_threads = 16; +SELECT 1 SETTINGS use_query_cache = true, query_cache_nondeterministic_function_handling = 'save', max_threads = 16; -- Check that entry in QC exists SELECT COUNT(*) FROM system.query_cache; diff --git a/tests/queries/0_stateless/02496_remove_redundant_sorting.reference b/tests/queries/0_stateless/02496_remove_redundant_sorting.reference index 7824fd8cba9..00db41e8ac5 100644 --- a/tests/queries/0_stateless/02496_remove_redundant_sorting.reference +++ b/tests/queries/0_stateless/02496_remove_redundant_sorting.reference @@ -332,13 +332,12 @@ SETTINGS optimize_aggregators_of_group_by_keys=0 -- avoid removing any() as it d Expression (Projection) Sorting (Sorting for ORDER BY) Expression (Before ORDER BY) - Filter ((WHERE + (Projection + Before ORDER BY))) - Filter (HAVING) - Aggregating - Expression ((Before GROUP BY + Projection)) - Sorting (Sorting for ORDER BY) - Expression ((Before ORDER BY + (Projection + Before ORDER BY))) - ReadFromSystemNumbers + Filter (((WHERE + (Projection + Before ORDER BY)) + HAVING)) + Aggregating + Expression ((Before GROUP BY + Projection)) + Sorting (Sorting for ORDER BY) + Expression ((Before ORDER BY + (Projection + Before ORDER BY))) + ReadFromSystemNumbers -- execute 1 2 diff --git a/tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.reference b/tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.reference index 9bb0c022752..a382e14ce03 100644 --- a/tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.reference +++ b/tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.reference @@ -28,21 +28,17 @@ WHERE type_1 = \'all\' (Expression) ExpressionTransform × 2 (Filter) - FilterTransform × 2 - (Filter) - FilterTransform × 2 - (Filter) - FilterTransform × 2 - (Aggregating) - ExpressionTransform × 2 - AggregatingTransform × 2 - Copy 1 → 2 - (Expression) - ExpressionTransform - (Expression) - ExpressionTransform - (ReadFromMergeTree) - MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 + FilterTransform × 6 + (Aggregating) + ExpressionTransform × 2 + AggregatingTransform × 2 + Copy 1 → 2 + (Expression) + ExpressionTransform + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 (Expression) ExpressionTransform × 2 (Filter) @@ -68,14 +64,10 @@ ExpressionTransform × 2 ExpressionTransform × 2 AggregatingTransform × 2 Copy 1 → 2 - (Filter) - FilterTransform - (Filter) - FilterTransform - (Expression) - ExpressionTransform - (ReadFromMergeTree) - MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 (Expression) ExpressionTransform × 2 (Aggregating) diff --git a/tests/queries/0_stateless/02718_parquet_metadata_format.reference b/tests/queries/0_stateless/02718_parquet_metadata_format.reference index 1f55c29da56..815968aeba5 100644 --- a/tests/queries/0_stateless/02718_parquet_metadata_format.reference +++ b/tests/queries/0_stateless/02718_parquet_metadata_format.reference @@ -78,7 +78,8 @@ "distinct_count": null, "min": "0", "max": "999" - } + }, + "have_bloom_filter": false }, { "name": "str", @@ -92,7 +93,8 @@ "distinct_count": null, "min": "Hello0", "max": "Hello999" - } + }, + "have_bloom_filter": false }, { "name": "mod", @@ -106,7 +108,8 @@ "distinct_count": null, "min": "0", "max": "8" - } + }, + "have_bloom_filter": false } ] }, @@ -128,7 +131,8 @@ "distinct_count": null, "min": "0", "max": "999" - } + }, + "have_bloom_filter": false }, { "name": "str", @@ -142,7 +146,8 @@ "distinct_count": null, "min": "Hello0", "max": "Hello999" - } + }, + "have_bloom_filter": false }, { "name": "mod", @@ -156,7 +161,8 @@ "distinct_count": null, "min": "0", "max": "8" - } + }, + "have_bloom_filter": false } ] } @@ -223,3 +229,55 @@ } 1 1 +{ + "num_columns": "1", + "num_rows": "5", + "num_row_groups": "1", + "format_version": "1.0", + "metadata_size": "267", + "total_uncompressed_size": "105", + "total_compressed_size": "128", + "columns": [ + { + "name": "ipv6", + "path": "ipv6", + "max_definition_level": "0", + "max_repetition_level": "0", + "physical_type": "FIXED_LEN_BYTE_ARRAY", + "logical_type": "None", + "compression": "GZIP", + "total_uncompressed_size": "105", + "total_compressed_size": "128", + "space_saved": "-21.9%", + "encodings": [ + "PLAIN", + "BIT_PACKED" + ] + } + ], + "row_groups": [ + { + "num_columns": "1", + "num_rows": "5", + "total_uncompressed_size": "105", + "total_compressed_size": "128", + "columns": [ + { + "name": "ipv6", + "path": "ipv6", + "total_compressed_size": "128", + "total_uncompressed_size": "105", + "have_statistics": true, + "statistics": { + "num_values": "5", + "null_count": "0", + "distinct_count": null, + "min": "27 32 150 125 17 250 66 31 157 44 75 218 51 50 19 144 ", + "max": "154 31 90 141 15 7 68 47 190 29 121 145 188 162 234 154 " + }, + "have_bloom_filter": true + } + ] + } + ] +} diff --git a/tests/queries/0_stateless/02718_parquet_metadata_format.sh b/tests/queries/0_stateless/02718_parquet_metadata_format.sh index 94d7f453850..c6371cff7a3 100755 --- a/tests/queries/0_stateless/02718_parquet_metadata_format.sh +++ b/tests/queries/0_stateless/02718_parquet_metadata_format.sh @@ -17,3 +17,4 @@ $CLICKHOUSE_LOCAL -q "select some_column from file('$CURDIR/data_parquet/02718_d $CLICKHOUSE_LOCAL -q "select num_columns from file('$CURDIR/data_parquet/02718_data.parquet', ParquetMetadata, 'num_columns Array(UInt32)')" 2>&1 | grep -c "BAD_ARGUMENTS" +$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_parquet/ipv6_bloom_filter.gz.parquet', ParquetMetadata) format JSONEachRow" | python3 -m json.tool diff --git a/tests/queries/0_stateless/02751_ip_types_aggregate_functions_states.sql.j2 b/tests/queries/0_stateless/02751_ip_types_aggregate_functions_states.sql.j2 index 7d030d4be2d..602b98e576b 100644 --- a/tests/queries/0_stateless/02751_ip_types_aggregate_functions_states.sql.j2 +++ b/tests/queries/0_stateless/02751_ip_types_aggregate_functions_states.sql.j2 @@ -1,5 +1,7 @@ -- Tags: no-parallel, no-fasttest +SET output_format_pretty_single_large_number_tip_threshold = 0; + {# this test checks backward compatibility of aggregate functions States against IPv4, IPv6 types #} {% set ip4_generator = "select num::UInt32::IPv4 ip from (select arrayJoin(range(999999999, number)) as num from numbers(999999999,50)) order by ip" %} diff --git a/tests/queries/0_stateless/02751_multiquery_with_argument.sh b/tests/queries/0_stateless/02751_multiquery_with_argument.sh index 4021194656b..4378786c145 100755 --- a/tests/queries/0_stateless/02751_multiquery_with_argument.sh +++ b/tests/queries/0_stateless/02751_multiquery_with_argument.sh @@ -9,7 +9,7 @@ $CLICKHOUSE_LOCAL "SELECT 101;" $CLICKHOUSE_LOCAL "SELECT 102;SELECT 103;" # Invalid SQL. -$CLICKHOUSE_LOCAL "SELECT 200; S" 2>&1 | grep -o 'Syntax error' +$CLICKHOUSE_LOCAL --implicit-select 0 "SELECT 200; S" 2>&1 | grep -o 'Syntax error' $CLICKHOUSE_LOCAL "; SELECT 201;" 2>&1 | grep -o 'Empty query' $CLICKHOUSE_LOCAL "; S; SELECT 202" 2>&1 | grep -o 'Empty query' diff --git a/tests/queries/0_stateless/02766_bitshift_with_const_arguments.sql b/tests/queries/0_stateless/02766_bitshift_with_const_arguments.sql index 91e8624057c..6b2961f0555 100644 --- a/tests/queries/0_stateless/02766_bitshift_with_const_arguments.sql +++ b/tests/queries/0_stateless/02766_bitshift_with_const_arguments.sql @@ -10,7 +10,7 @@ DROP TABLE IF EXISTS t1; CREATE TABLE t0 (vkey UInt32, pkey UInt32, c0 UInt32) engine = TinyLog; CREATE TABLE t1 (vkey UInt32) ENGINE = AggregatingMergeTree ORDER BY vkey; INSERT INTO t0 VALUES (15, 25000, 58); -SELECT ref_5.pkey AS c_2_c2392_6 FROM t0 AS ref_5 WHERE 'J[' < multiIf(ref_5.pkey IN ( SELECT 1 ), bitShiftLeft(multiIf(ref_5.c0 > NULL, '1', ')'), 40), NULL); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT ref_5.pkey AS c_2_c2392_6 FROM t0 AS ref_5 WHERE 'J[' < multiIf(ref_5.pkey IN ( SELECT 1 ), bitShiftLeft(multiIf(ref_5.c0 > NULL, '1', ')'), 40), NULL); DROP TABLE t0; DROP TABLE t1; diff --git a/tests/queries/0_stateless/02771_multiple_query_arguments.sh b/tests/queries/0_stateless/02771_multiple_query_arguments.sh index ae6e23eb61a..fcc1394573a 100755 --- a/tests/queries/0_stateless/02771_multiple_query_arguments.sh +++ b/tests/queries/0_stateless/02771_multiple_query_arguments.sh @@ -18,4 +18,4 @@ $CLICKHOUSE_LOCAL --query "SELECT 202;" --query "SELECT 202;" $CLICKHOUSE_LOCAL --query "SELECT 303" --query "SELECT 303; SELECT 303" $CLICKHOUSE_LOCAL --query "" --query "" $CLICKHOUSE_LOCAL --query "SELECT 303" --query 2>&1 | grep -o 'Bad arguments' -$CLICKHOUSE_LOCAL --query "SELECT 303" --query "SELE" 2>&1 | grep -o 'Syntax error' +$CLICKHOUSE_LOCAL --implicit-select 0 --query "SELECT 303" --query "SELE" 2>&1 | grep -o 'Syntax error' diff --git a/tests/queries/0_stateless/02800_clickhouse_local_default_settings.reference b/tests/queries/0_stateless/02800_clickhouse_local_default_settings.reference index 0f18d1a3897..54c6f7ce397 100644 --- a/tests/queries/0_stateless/02800_clickhouse_local_default_settings.reference +++ b/tests/queries/0_stateless/02800_clickhouse_local_default_settings.reference @@ -1,2 +1,3 @@ allow_introspection_functions 1 storage_file_read_method mmap +implicit_select 1 diff --git a/tests/queries/0_stateless/02812_from_to_utc_timestamp.reference b/tests/queries/0_stateless/02812_from_to_utc_timestamp.reference index 4da8a9784dd..fb92bdda821 100644 --- a/tests/queries/0_stateless/02812_from_to_utc_timestamp.reference +++ b/tests/queries/0_stateless/02812_from_to_utc_timestamp.reference @@ -3,3 +3,10 @@ 3 2023-03-16 12:22:33 2023-03-16 10:22:33.000 2023-03-16 03:22:33 2023-03-16 19:22:33.123 2024-02-24 10:22:33 2024-02-24 12:22:33 2024-10-24 09:22:33 2024-10-24 13:22:33 +2024-10-24 16:22:33 2024-10-24 06:22:33 +leap year: 2024-02-29 16:22:33 2024-02-29 06:22:33 +non-leap year: 2023-03-01 16:22:33 2023-03-01 06:22:33 +leap year: 2024-02-29 04:22:33 2024-02-29 19:22:33 +non-leap year: 2023-03-01 04:22:33 2023-02-28 19:22:33 +timezone with half-hour offset: 2024-02-29 00:52:33 2024-02-29 21:52:33 +jump over a year: 2024-01-01 04:01:01 2023-12-31 20:01:01 diff --git a/tests/queries/0_stateless/02812_from_to_utc_timestamp.sh b/tests/queries/0_stateless/02812_from_to_utc_timestamp.sh index 835dab8af57..9eb4484ace0 100755 --- a/tests/queries/0_stateless/02812_from_to_utc_timestamp.sh +++ b/tests/queries/0_stateless/02812_from_to_utc_timestamp.sh @@ -15,4 +15,13 @@ $CLICKHOUSE_CLIENT -q "select x, to_utc_timestamp(toDateTime('2023-03-16 11:22:3 # timestamp convert between DST timezone and UTC $CLICKHOUSE_CLIENT -q "select to_utc_timestamp(toDateTime('2024-02-24 11:22:33'), 'Europe/Madrid'), from_utc_timestamp(toDateTime('2024-02-24 11:22:33'), 'Europe/Madrid')" $CLICKHOUSE_CLIENT -q "select to_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'Europe/Madrid'), from_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'Europe/Madrid')" -$CLICKHOUSE_CLIENT -q "drop table test_tbl" \ No newline at end of file +$CLICKHOUSE_CLIENT -q "select to_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'EST'), from_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'EST')" + +$CLICKHOUSE_CLIENT -q "select 'leap year:', to_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'EST'), from_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'EST')" +$CLICKHOUSE_CLIENT -q "select 'non-leap year:', to_utc_timestamp(toDateTime('2023-02-29 11:22:33'), 'EST'), from_utc_timestamp(toDateTime('2023-02-29 11:22:33'), 'EST')" +$CLICKHOUSE_CLIENT -q "select 'leap year:', to_utc_timestamp(toDateTime('2024-02-28 23:22:33'), 'EST'), from_utc_timestamp(toDateTime('2024-03-01 00:22:33'), 'EST')" +$CLICKHOUSE_CLIENT -q "select 'non-leap year:', to_utc_timestamp(toDateTime('2023-02-28 23:22:33'), 'EST'), from_utc_timestamp(toDateTime('2023-03-01 00:22:33'), 'EST')" +$CLICKHOUSE_CLIENT -q "select 'timezone with half-hour offset:', to_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'Australia/Adelaide'), from_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'Australia/Adelaide')" +$CLICKHOUSE_CLIENT -q "select 'jump over a year:', to_utc_timestamp(toDateTime('2023-12-31 23:01:01'), 'EST'), from_utc_timestamp(toDateTime('2024-01-01 01:01:01'), 'EST')" + +$CLICKHOUSE_CLIENT -q "drop table test_tbl" diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views_1.reference b/tests/queries/0_stateless/02932_refreshable_materialized_views_1.reference index 3ec0d3b9ee2..b50ea042e86 100644 --- a/tests/queries/0_stateless/02932_refreshable_materialized_views_1.reference +++ b/tests/queries/0_stateless/02932_refreshable_materialized_views_1.reference @@ -1,14 +1,14 @@ <1: created view> a -CREATE MATERIALIZED VIEW default.a\nREFRESH EVERY 2 SECOND\n(\n `x` UInt64\n)\nENGINE = Memory\nAS SELECT number AS x\nFROM numbers(2)\nUNION ALL\nSELECT rand64() AS x +CREATE MATERIALIZED VIEW default.a\nREFRESH EVERY 2 SECOND\n(\n `x` UInt64\n)\nENGINE = Memory\nDEFINER = default SQL SECURITY DEFINER\nAS SELECT number AS x\nFROM numbers(2)\nUNION ALL\nSELECT rand64() AS x <2: refreshed> 3 1 1 <3: time difference at least> 1000 <4.1: fake clock> Scheduled 2050-01-01 00:00:01 2050-01-01 00:00:02 1 3 3 3 0 <4.5: altered> Scheduled 2050-01-01 00:00:01 2052-01-01 00:00:00 -CREATE MATERIALIZED VIEW default.a\nREFRESH EVERY 2 YEAR\n(\n `x` UInt64\n)\nENGINE = Memory\nAS SELECT x * 2 AS x\nFROM default.src +CREATE MATERIALIZED VIEW default.a\nREFRESH EVERY 2 YEAR\n(\n `x` UInt64\n)\nENGINE = Memory\nDEFINER = default SQL SECURITY DEFINER\nAS SELECT x * 2 AS x\nFROM default.src <5: no refresh> 3 <6: refreshed> 2 <7: refreshed> Scheduled 2052-02-03 04:05:06 2054-01-01 00:00:00 -CREATE MATERIALIZED VIEW default.b\nREFRESH EVERY 2 YEAR DEPENDS ON default.a\n(\n `y` Int32\n)\nENGINE = MergeTree\nORDER BY y\nSETTINGS index_granularity = 8192\nAS SELECT x * 10 AS y\nFROM default.a +CREATE MATERIALIZED VIEW default.b\nREFRESH EVERY 2 YEAR DEPENDS ON default.a\n(\n `y` Int32\n)\nENGINE = MergeTree\nORDER BY y\nSETTINGS index_granularity = 8192\nDEFINER = default SQL SECURITY DEFINER\nAS SELECT x * 10 AS y\nFROM default.a <7.5: created dependent> 2052-11-11 11:11:11 <8: refreshed> 20 <9: refreshed> a Scheduled 2054-01-01 00:00:00 @@ -26,4 +26,4 @@ CREATE MATERIALIZED VIEW default.b\nREFRESH EVERY 2 YEAR DEPENDS ON default.a\n( <17: chain-refreshed> a Scheduled 2062-01-01 00:00:00 <17: chain-refreshed> b Scheduled 2062-01-01 00:00:00 <18: removed dependency> b Scheduled 2062-03-03 03:03:03 2062-03-03 03:03:03 2064-01-01 00:00:00 -CREATE MATERIALIZED VIEW default.b\nREFRESH EVERY 2 YEAR\n(\n `y` Int32\n)\nENGINE = MergeTree\nORDER BY y\nSETTINGS index_granularity = 8192\nAS SELECT x * 10 AS y\nFROM default.a +CREATE MATERIALIZED VIEW default.b\nREFRESH EVERY 2 YEAR\n(\n `y` Int32\n)\nENGINE = MergeTree\nORDER BY y\nSETTINGS index_granularity = 8192\nDEFINER = default SQL SECURITY DEFINER\nAS SELECT x * 10 AS y\nFROM default.a diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views_2.reference b/tests/queries/0_stateless/02932_refreshable_materialized_views_2.reference index 3eeab4f574e..8dcc3d55603 100644 --- a/tests/queries/0_stateless/02932_refreshable_materialized_views_2.reference +++ b/tests/queries/0_stateless/02932_refreshable_materialized_views_2.reference @@ -7,9 +7,9 @@ <25: rename during refresh> f Running <27: cancelled> f Scheduled cancelled <28: drop during refresh> 0 0 -CREATE MATERIALIZED VIEW default.g\nREFRESH EVERY 1 WEEK OFFSET 3 DAY 4 HOUR RANDOMIZE FOR 4 DAY 1 HOUR\n(\n `x` Int64\n)\nENGINE = Memory\nAS SELECT 42 AS x +CREATE MATERIALIZED VIEW default.g\nREFRESH EVERY 1 WEEK OFFSET 3 DAY 4 HOUR RANDOMIZE FOR 4 DAY 1 HOUR\n(\n `x` Int64\n)\nENGINE = Memory\nDEFINER = default SQL SECURITY DEFINER\nAS SELECT 42 AS x <29: randomize> 1 1 -CREATE MATERIALIZED VIEW default.h\nREFRESH EVERY 1 SECOND TO default.dest\n(\n `x` Int64\n)\nAS SELECT x * 10 AS x\nFROM default.src +CREATE MATERIALIZED VIEW default.h\nREFRESH EVERY 1 SECOND TO default.dest\n(\n `x` Int64\n)\nDEFINER = default SQL SECURITY DEFINER\nAS SELECT x * 10 AS x\nFROM default.src <30: to existing table> 10 <31: to existing table> 10 <31: to existing table> 20 diff --git a/tests/queries/0_stateless/02961_higher_order_constant_expressions.reference b/tests/queries/0_stateless/02961_higher_order_constant_expressions.reference new file mode 100644 index 00000000000..058d23ad850 --- /dev/null +++ b/tests/queries/0_stateless/02961_higher_order_constant_expressions.reference @@ -0,0 +1,8 @@ +[1,2,3] 1 +[2,3,4] 1 +[2,4,6] 1 +[5,7,9] 1 +[1,1,1] 1 +[1,2,3] 0 +[0,0,0] 0 +3 1 diff --git a/tests/queries/0_stateless/02961_higher_order_constant_expressions.sql b/tests/queries/0_stateless/02961_higher_order_constant_expressions.sql new file mode 100644 index 00000000000..23b0b72f48f --- /dev/null +++ b/tests/queries/0_stateless/02961_higher_order_constant_expressions.sql @@ -0,0 +1,13 @@ +SET enable_analyzer = 1; + +SELECT arrayMap(x -> x, [1, 2, 3]) AS x, isConstant(x); +SELECT arrayMap(x -> x + 1, [1, 2, 3]) AS x, isConstant(x); +SELECT arrayMap(x -> x + x, [1, 2, 3]) AS x, isConstant(x); +SELECT arrayMap((x, y) -> x + y, [1, 2, 3], [4, 5, 6]) AS x, isConstant(x); +SELECT arrayMap(x -> 1, [1, 2, 3]) AS x, isConstant(x); +SELECT arrayMap(x -> x + number, [1, 2, 3]) AS x, isConstant(x) FROM numbers(1); +SELECT arrayMap(x -> number, [1, 2, 3]) AS x, isConstant(x) FROM numbers(1); +SELECT arrayMax([1, 2, 3]) AS x, isConstant(x); + +-- Does not work yet: +-- SELECT [1, 2, 3] IN arrayMap(x -> x, [1, 2, 3]); diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.reference b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.reference index e1bf9c27a81..7475cc7a97e 100644 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.reference +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.reference @@ -28,3 +28,19 @@ SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.` SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done + +simple (global) join with analyzer and parallel replicas with local plan +4200000 4200000 4200000 -1400000 +4200006 4200006 4200006 -1400002 +4200012 4200012 4200012 -1400004 +4200018 4200018 4200018 -1400006 +4200024 4200024 4200024 -1400008 +4200030 4200030 4200030 -1400010 +4200036 4200036 4200036 -1400012 +4200042 4200042 4200042 -1400014 +4200048 4200048 4200048 -1400016 +4200054 4200054 4200054 -1400018 +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` (stage: WithMergeableState) + DefaultCoordinator: Coordination done +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) + DefaultCoordinator: Coordination done diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh index b4271c3d29b..a6e755ebc35 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh @@ -27,6 +27,8 @@ inner join (select key, value from num_2) r on l.key = r.key order by l.key limit 10 offset 700000 SETTINGS allow_experimental_analyzer=1" +PARALLEL_REPLICAS_SETTINGS="enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 0" + ############## echo echo "simple (global) join with analyzer and parallel replicas" @@ -35,17 +37,31 @@ $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2) r on l.key = r.key order by l.key limit 10 offset 700000 -SETTINGS allow_experimental_analyzer=1, allow_experimental_parallel_reading_from_replicas = 2, -max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_local_plan=0" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2) r on l.key = r.key order by l.key limit 10 offset 700000 -SETTINGS allow_experimental_analyzer=1, allow_experimental_parallel_reading_from_replicas = 2, send_logs_level='trace', -max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" 2>&1 | +SETTINGS enable_analyzer=1, send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_local_plan=0" 2>&1 | +grep "executeQuery\|.*Coordinator: Coordination done" | +grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | +sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' + +echo +echo "simple (global) join with analyzer and parallel replicas with local plan" + +$CLICKHOUSE_CLIENT -q " +select * from (select key, value from num_1) l +inner join (select key, value from num_2) r on l.key = r.key +order by l.key limit 10 offset 700000 +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_local_plan=1" + +$CLICKHOUSE_CLIENT -q " +select * from (select key, value from num_1) l +inner join (select key, value from num_2) r on l.key = r.key +order by l.key limit 10 offset 700000 +SETTINGS enable_analyzer=1, send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_local_plan=1" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.reference b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.reference index 297ec311f3e..f17d9aea3d5 100644 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.reference +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.reference @@ -11,7 +11,6 @@ simple (local) join with analyzer and parallel replicas 4200048 4200048 4200048 -1400016 4200054 4200054 4200054 -1400018 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done simple (local) join with analyzer and parallel replicas and full sorting merge join @@ -26,7 +25,6 @@ simple (local) join with analyzer and parallel replicas and full sorting merge j 4200048 4200048 4200048 -1400016 4200054 4200054 4200054 -1400018 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer @@ -53,5 +51,4 @@ nested join with analyzer and parallel replicas, both local 420336 420336 420336 -140112 420378 420378 420378 -140126 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh index ed13bf3321b..4768e308f1e 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh @@ -17,6 +17,8 @@ insert into num_1 select number * 2, toString(number * 2) from numbers(1e7); insert into num_2 select number * 3, -number from numbers(1.5e6); " +PARALLEL_REPLICAS_SETTINGS="allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 1, parallel_replicas_local_plan=1" + ############## echo echo "simple (local) join with analyzer and parallel replicas" @@ -25,17 +27,13 @@ $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2) r on l.key = r.key order by l.key limit 10 offset 700000 -SETTINGS allow_experimental_analyzer=1, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1" +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2) r on l.key = r.key order by l.key limit 10 offset 700000 -SETTINGS allow_experimental_analyzer=1, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1" 2>&1 | +SETTINGS enable_analyzer=1, send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' @@ -49,17 +47,13 @@ $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2) r on l.key = r.key order by l.key limit 10 offset 700000 -SETTINGS allow_experimental_analyzer=1, join_algorithm='full_sorting_merge', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1" +SETTINGS enable_analyzer=1, join_algorithm='full_sorting_merge', $PARALLEL_REPLICAS_SETTINGS" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2) r on l.key = r.key order by l.key limit 10 offset 700000 -SETTINGS allow_experimental_analyzer=1, join_algorithm='full_sorting_merge', send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1" 2>&1 | +SETTINGS enable_analyzer=1, join_algorithm='full_sorting_merge', send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' @@ -74,7 +68,7 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS allow_experimental_analyzer=1" +SETTINGS enable_analyzer=1" ############## @@ -86,18 +80,14 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS allow_experimental_analyzer=1, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1" +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS allow_experimental_analyzer=1, join_algorithm='full_sorting_merge', send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1" 2>&1 | +SETTINGS enable_analyzer=1, join_algorithm='full_sorting_merge', send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' diff --git a/tests/queries/0_stateless/02989_variant_comparison.sql b/tests/queries/0_stateless/02989_variant_comparison.sql index e0dcbc97c27..4d09933fb7b 100644 --- a/tests/queries/0_stateless/02989_variant_comparison.sql +++ b/tests/queries/0_stateless/02989_variant_comparison.sql @@ -1,4 +1,5 @@ set allow_experimental_variant_type=1; +set allow_suspicious_types_in_order_by=1; create table test (v1 Variant(String, UInt64, Array(UInt32)), v2 Variant(String, UInt64, Array(UInt32))) engine=Memory; diff --git a/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference b/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference index 79ebc7a5c0c..db8d26ccfea 100644 --- a/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference +++ b/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference @@ -83,7 +83,7 @@ QUERY id: 0 FUNCTION id: 4, function_name: tuple, function_type: ordinary, result_type: Tuple(Nullable(UInt64)) ARGUMENTS LIST id: 5, nodes: 1 - FUNCTION id: 6, function_name: sum, function_type: aggregate, nulls_action : IGNORE_NULLS, result_type: Nullable(UInt64) + FUNCTION id: 6, function_name: sum, function_type: aggregate, result_type: Nullable(UInt64) ARGUMENTS LIST id: 7, nodes: 1 FUNCTION id: 8, function_name: if, function_type: ordinary, result_type: Nullable(UInt8) diff --git a/tests/queries/0_stateless/03031_clickhouse_local_input.reference b/tests/queries/0_stateless/03031_clickhouse_local_input.reference index a6feeef100d..bbb57da94ce 100644 --- a/tests/queries/0_stateless/03031_clickhouse_local_input.reference +++ b/tests/queries/0_stateless/03031_clickhouse_local_input.reference @@ -1,7 +1,15 @@ -# foo +# foo (pipe) +foo +# foo (file) foo # !foo # bar bar # defaults bam +# inferred destination table structure +foo +# direct +foo +# infile +foo diff --git a/tests/queries/0_stateless/03031_clickhouse_local_input.sh b/tests/queries/0_stateless/03031_clickhouse_local_input.sh index e2f9cf48108..f271a5184fd 100755 --- a/tests/queries/0_stateless/03031_clickhouse_local_input.sh +++ b/tests/queries/0_stateless/03031_clickhouse_local_input.sh @@ -4,17 +4,40 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -tmp_file="$CUR_DIR/$CLICKHOUSE_DATABASE.txt" -echo '# foo' +tmp_file="$CUR_DIR/03031_$CLICKHOUSE_DATABASE.txt" +tmp_input="$CUR_DIR/03031_${CLICKHOUSE_DATABASE}_in.txt" + +echo '# foo (pipe)' $CLICKHOUSE_LOCAL --engine_file_truncate_on_insert=1 -q "insert into function file('$tmp_file', 'LineAsString', 'x String') select * from input('x String') format LineAsString" << "$tmp_input" +$CLICKHOUSE_LOCAL --engine_file_truncate_on_insert=1 -q "insert into function file('$tmp_file', 'LineAsString', 'x String') select * from input('x String') format LineAsString" <"$tmp_input" +cat "$tmp_file" + echo '# !foo' $CLICKHOUSE_LOCAL --engine_file_truncate_on_insert=1 -q "insert into function file('$tmp_file', 'LineAsString', 'x String') select * from input('x String') where x != 'foo' format LineAsString" << 8192 * 2 +ORDER BY x ASC +LIMIT 4 +SETTINGS max_block_size = 8192, +read_in_order_two_level_merge_threshold = 0, --force preliminary merge +max_threads = 1, +optimize_read_in_order = 1, +log_comment = 'preliminary merge with filter'; + +SYSTEM FLUSH LOGS; + +SELECT read_rows +FROM system.query_log +WHERE current_database = currentDatabase() +AND log_comment = 'preliminary merge with filter' +AND type = 'QueryFinish' +ORDER BY query_start_time DESC +LIMIT 1; + +SELECT '========'; +-- Expecting 2 virtual rows + one chunk (8192) for result + one extra chunk for next consumption in merge transform (8192), +-- both chunks come from the same part. +SELECT x +FROM t +ORDER BY x ASC +LIMIT 4 +SETTINGS max_block_size = 8192, +read_in_order_two_level_merge_threshold = 5, --avoid preliminary merge +max_threads = 1, +optimize_read_in_order = 1, +log_comment = 'no preliminary merge, no filter'; + +SYSTEM FLUSH LOGS; + +SELECT read_rows +FROM system.query_log +WHERE current_database = currentDatabase() +AND log_comment = 'no preliminary merge, no filter' +AND type = 'QueryFinish' +ORDER BY query_start_time DESC +LIMIT 1; + +SELECT '========'; +-- Expecting 2 virtual rows + two chunks (8192*2) get filtered out + one chunk for result (8192), +-- all chunks come from the same part. +SELECT k +FROM t +WHERE k > 8192 * 2 +ORDER BY x ASC +LIMIT 4 +SETTINGS max_block_size = 8192, +read_in_order_two_level_merge_threshold = 5, --avoid preliminary merge +max_threads = 1, +optimize_read_in_order = 1, +log_comment = 'no preliminary merge, with filter'; + +SYSTEM FLUSH LOGS; + +SELECT read_rows +FROM system.query_log +WHERE current_database = currentDatabase() +AND log_comment = 'no preliminary merge, with filter' +AND type = 'QueryFinish' +ORDER BY query_start_time DESC +LIMIT 1; + +DROP TABLE t; + +SELECT '========'; +-- from 02149_read_in_order_fixed_prefix +DROP TABLE IF EXISTS fixed_prefix; + +CREATE TABLE fixed_prefix(a UInt32, b UInt32) +ENGINE = MergeTree ORDER BY (a, b) +SETTINGS index_granularity = 3; + +SYSTEM STOP MERGES fixed_prefix; + +INSERT INTO fixed_prefix VALUES (0, 100), (1, 2), (1, 3), (1, 4), (2, 5); +INSERT INTO fixed_prefix VALUES (0, 100), (1, 2), (1, 3), (1, 4), (2, 5); + +SELECT a, b +FROM fixed_prefix +WHERE a = 1 +ORDER BY b +SETTINGS max_threads = 1, +optimize_read_in_order = 1, +read_in_order_two_level_merge_threshold = 0; --force preliminary merge + +SELECT a, b +FROM fixed_prefix +WHERE a = 1 +ORDER BY b +SETTINGS max_threads = 1, +optimize_read_in_order = 1, +read_in_order_two_level_merge_threshold = 5; --avoid preliminary merge + +DROP TABLE fixed_prefix; + +SELECT '========'; +DROP TABLE IF EXISTS function_pk; + +CREATE TABLE function_pk +( + `A` Int64, + `B` Int64 +) +ENGINE = MergeTree ORDER BY (A, -B) +SETTINGS index_granularity = 1; + +SYSTEM STOP MERGES function_pk; + +INSERT INTO function_pk values(1,1); +INSERT INTO function_pk values(1,3); +INSERT INTO function_pk values(1,2); + +SELECT * +FROM function_pk +ORDER BY (A,-B) ASC +limit 3 +SETTINGS max_threads = 1, +optimize_read_in_order = 1, +read_in_order_two_level_merge_threshold = 5; --avoid preliminary merge + +DROP TABLE function_pk; + +-- modified from 02317_distinct_in_order_optimization +SELECT '-- test distinct ----'; + +DROP TABLE IF EXISTS distinct_in_order SYNC; + +CREATE TABLE distinct_in_order +( + `a` int, + `b` int, + `c` int +) +ENGINE = MergeTree +ORDER BY (a, b) +SETTINGS index_granularity = 8192, +index_granularity_bytes = '10Mi'; + +SYSTEM STOP MERGES distinct_in_order; + +INSERT INTO distinct_in_order SELECT + number % number, + number % 5, + number % 10 +FROM numbers(1, 1000000); + +SELECT DISTINCT a +FROM distinct_in_order +ORDER BY a ASC +SETTINGS read_in_order_two_level_merge_threshold = 0, +optimize_read_in_order = 1, +max_threads = 2; + +DROP TABLE distinct_in_order; \ No newline at end of file diff --git a/tests/queries/0_stateless/03031_read_in_order_optimization_with_virtual_row_explain.reference b/tests/queries/0_stateless/03031_read_in_order_optimization_with_virtual_row_explain.reference new file mode 100644 index 00000000000..33ef6b19222 --- /dev/null +++ b/tests/queries/0_stateless/03031_read_in_order_optimization_with_virtual_row_explain.reference @@ -0,0 +1,25 @@ +(Expression) +ExpressionTransform + (Sorting) + MergingSortedTransform 4 → 1 + (Expression) + ExpressionTransform × 4 + (ReadFromMergeTree) + ExpressionTransform × 5 + VirtualRowTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + VirtualRowTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 + VirtualRowTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + VirtualRowTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 + VirtualRowTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 + ExpressionTransform + VirtualRowTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 diff --git a/tests/queries/0_stateless/03031_read_in_order_optimization_with_virtual_row_explain.sql b/tests/queries/0_stateless/03031_read_in_order_optimization_with_virtual_row_explain.sql new file mode 100644 index 00000000000..8e3f37b37b8 --- /dev/null +++ b/tests/queries/0_stateless/03031_read_in_order_optimization_with_virtual_row_explain.sql @@ -0,0 +1,26 @@ +-- Tags: no-random-merge-tree-settings, no-object-storage + +SET optimize_read_in_order = 1, merge_tree_min_rows_for_concurrent_read = 1000, read_in_order_use_virtual_row = 1; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab +( + `t` DateTime +) +ENGINE = MergeTree +ORDER BY t +SETTINGS index_granularity = 1; + +SYSTEM STOP MERGES tab; + +INSERT INTO tab SELECT toDateTime('2024-01-10') + number FROM numbers(10000); +INSERT INTO tab SELECT toDateTime('2024-01-30') + number FROM numbers(10000); +INSERT INTO tab SELECT toDateTime('2024-01-20') + number FROM numbers(10000); + +EXPLAIN PIPELINE +SELECT * +FROM tab +ORDER BY t ASC +SETTINGS read_in_order_two_level_merge_threshold = 0, max_threads = 4, read_in_order_use_buffering = 0 +FORMAT tsv; \ No newline at end of file diff --git a/tests/queries/0_stateless/03031_read_in_order_optimization_with_virtual_row_special.reference b/tests/queries/0_stateless/03031_read_in_order_optimization_with_virtual_row_special.reference new file mode 100644 index 00000000000..b03759364cf --- /dev/null +++ b/tests/queries/0_stateless/03031_read_in_order_optimization_with_virtual_row_special.reference @@ -0,0 +1,2 @@ +dist +src diff --git a/tests/queries/0_stateless/03031_read_in_order_optimization_with_virtual_row_special.sql b/tests/queries/0_stateless/03031_read_in_order_optimization_with_virtual_row_special.sql new file mode 100644 index 00000000000..52aa71437db --- /dev/null +++ b/tests/queries/0_stateless/03031_read_in_order_optimization_with_virtual_row_special.sql @@ -0,0 +1,21 @@ +-- Tags: no-parallel + +-- modified from test_01155_ordinary, to test special optimization path for virtual row +DROP DATABASE IF EXISTS test_03031; + +CREATE DATABASE test_03031; + +USE test_03031; + +SET read_in_order_use_virtual_row = 1; + +CREATE TABLE src (s String) ENGINE = MergeTree() ORDER BY s; +INSERT INTO src(s) VALUES ('before moving tables'); +CREATE TABLE dist (s String) ENGINE = Distributed(test_shard_localhost, test_03031, src); + +SET enable_analyzer=0; +SELECT _table FROM merge('test_03031', '') ORDER BY _table, s; + +DROP TABLE src; +DROP TABLE dist; +DROP DATABASE test_03031; \ No newline at end of file diff --git a/tests/queries/0_stateless/03035_dynamic_sorting.sql b/tests/queries/0_stateless/03035_dynamic_sorting.sql index 6c584f57b1e..43d6568a14a 100644 --- a/tests/queries/0_stateless/03035_dynamic_sorting.sql +++ b/tests/queries/0_stateless/03035_dynamic_sorting.sql @@ -1,4 +1,5 @@ set allow_experimental_dynamic_type = 1; +set allow_suspicious_types_in_order_by=1; drop table if exists test; create table test (d1 Dynamic(max_types=2), d2 Dynamic(max_types=2)) engine=Memory; diff --git a/tests/queries/0_stateless/03036_dynamic_read_shared_subcolumns_small.sql.j2 b/tests/queries/0_stateless/03036_dynamic_read_shared_subcolumns_small.sql.j2 index dde4f3f53c3..d6732d91e74 100644 --- a/tests/queries/0_stateless/03036_dynamic_read_shared_subcolumns_small.sql.j2 +++ b/tests/queries/0_stateless/03036_dynamic_read_shared_subcolumns_small.sql.j2 @@ -1,6 +1,7 @@ set allow_experimental_variant_type = 1; set use_variant_as_common_type = 1; set allow_experimental_dynamic_type = 1; +set allow_suspicious_types_in_order_by = 1; drop table if exists test; diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2 b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2 index 3253d7a6c68..daf85077160 100644 --- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2 +++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2 @@ -1,6 +1,7 @@ set allow_experimental_variant_type = 1; set use_variant_as_common_type = 1; set allow_experimental_dynamic_type = 1; +set allow_suspicious_types_in_order_by = 1; drop table if exists test; diff --git a/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.reference b/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.reference index 80f4e309505..d0a3e7b02ae 100644 --- a/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.reference +++ b/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.reference @@ -163,17 +163,21 @@ Positions: 4 2 0 1 Filter (( + (JOIN actions + Change column names to column identifiers))) Header: __table1.id UInt64 __table1.value String - Filter column: and(equals(__table1.id, 5_UInt8), equals(__table1.id, 6_UInt8)) (removed) + AND column: equals(__table1.id, 5_UInt8) Actions: INPUT : 0 -> id UInt64 : 0 - INPUT : 1 -> value String : 1 + COLUMN Const(UInt8) -> 5_UInt8 UInt8 : 1 + FUNCTION equals(id : 0, 5_UInt8 :: 1) -> equals(__table1.id, 5_UInt8) UInt8 : 2 + Positions: 2 0 2 + Filter column: and(equals(__table1.id, 5_UInt8), equals(__table1.id, 6_UInt8)) (removed) + Actions: INPUT : 2 -> value String : 0 + INPUT : 1 -> id UInt64 : 1 COLUMN Const(UInt8) -> 6_UInt8 UInt8 : 2 - COLUMN Const(UInt8) -> 5_UInt8 UInt8 : 3 - ALIAS id : 0 -> __table1.id UInt64 : 4 - ALIAS value :: 1 -> __table1.value String : 5 - FUNCTION equals(id : 0, 6_UInt8 :: 2) -> equals(__table1.id, 6_UInt8) UInt8 : 1 - FUNCTION equals(id :: 0, 5_UInt8 :: 3) -> equals(__table1.id, 5_UInt8) UInt8 : 2 - FUNCTION and(equals(__table1.id, 5_UInt8) :: 2, equals(__table1.id, 6_UInt8) :: 1) -> and(equals(__table1.id, 5_UInt8), equals(__table1.id, 6_UInt8)) UInt8 : 3 - Positions: 3 4 5 + INPUT : 0 -> equals(__table1.id, 5_UInt8) UInt8 : 3 + ALIAS value :: 0 -> __table1.value String : 4 + ALIAS id : 1 -> __table1.id UInt64 : 0 + FUNCTION equals(id :: 1, 6_UInt8 :: 2) -> equals(__table1.id, 6_UInt8) UInt8 : 5 + FUNCTION and(equals(__table1.id, 5_UInt8) :: 3, equals(__table1.id, 6_UInt8) :: 5) -> and(equals(__table1.id, 5_UInt8), equals(__table1.id, 6_UInt8)) UInt8 : 2 + Positions: 2 0 4 ReadFromMergeTree (default.test_table_1) Header: id UInt64 value String @@ -183,17 +187,21 @@ Positions: 4 2 0 1 Filter (( + (JOIN actions + Change column names to column identifiers))) Header: __table2.id UInt64 __table2.value String - Filter column: and(equals(__table2.id, 6_UInt8), equals(__table2.id, 5_UInt8)) (removed) + AND column: equals(__table2.id, 6_UInt8) Actions: INPUT : 0 -> id UInt64 : 0 - INPUT : 1 -> value String : 1 + COLUMN Const(UInt8) -> 6_UInt8 UInt8 : 1 + FUNCTION equals(id : 0, 6_UInt8 :: 1) -> equals(__table2.id, 6_UInt8) UInt8 : 2 + Positions: 2 0 2 + Filter column: and(equals(__table2.id, 6_UInt8), equals(__table2.id, 5_UInt8)) (removed) + Actions: INPUT : 2 -> value String : 0 + INPUT : 1 -> id UInt64 : 1 COLUMN Const(UInt8) -> 5_UInt8 UInt8 : 2 - COLUMN Const(UInt8) -> 6_UInt8 UInt8 : 3 - ALIAS id : 0 -> __table2.id UInt64 : 4 - ALIAS value :: 1 -> __table2.value String : 5 - FUNCTION equals(id : 0, 5_UInt8 :: 2) -> equals(__table2.id, 5_UInt8) UInt8 : 1 - FUNCTION equals(id :: 0, 6_UInt8 :: 3) -> equals(__table2.id, 6_UInt8) UInt8 : 2 - FUNCTION and(equals(__table2.id, 6_UInt8) :: 2, equals(__table2.id, 5_UInt8) :: 1) -> and(equals(__table2.id, 6_UInt8), equals(__table2.id, 5_UInt8)) UInt8 : 3 - Positions: 3 4 5 + INPUT : 0 -> equals(__table2.id, 6_UInt8) UInt8 : 3 + ALIAS value :: 0 -> __table2.value String : 4 + ALIAS id : 1 -> __table2.id UInt64 : 0 + FUNCTION equals(id :: 1, 5_UInt8 :: 2) -> equals(__table2.id, 5_UInt8) UInt8 : 5 + FUNCTION and(equals(__table2.id, 6_UInt8) :: 3, equals(__table2.id, 5_UInt8) :: 5) -> and(equals(__table2.id, 6_UInt8), equals(__table2.id, 5_UInt8)) UInt8 : 2 + Positions: 2 0 4 ReadFromMergeTree (default.test_table_2) Header: id UInt64 value String @@ -656,17 +664,21 @@ Positions: 4 2 0 1 __table1.value String __table2.value String __table2.id UInt64 - Filter column: and(equals(__table1.id, 5_UInt8), equals(__table2.id, 6_UInt8)) (removed) + AND column: equals(__table1.id, 5_UInt8) Actions: INPUT : 0 -> __table1.id UInt64 : 0 - INPUT :: 1 -> __table1.value String : 1 - INPUT :: 2 -> __table2.value String : 2 - INPUT : 3 -> __table2.id UInt64 : 3 - COLUMN Const(UInt8) -> 5_UInt8 UInt8 : 4 - COLUMN Const(UInt8) -> 6_UInt8 UInt8 : 5 - FUNCTION equals(__table1.id : 0, 5_UInt8 :: 4) -> equals(__table1.id, 5_UInt8) UInt8 : 6 - FUNCTION equals(__table2.id : 3, 6_UInt8 :: 5) -> equals(__table2.id, 6_UInt8) UInt8 : 4 - FUNCTION and(equals(__table1.id, 5_UInt8) :: 6, equals(__table2.id, 6_UInt8) :: 4) -> and(equals(__table1.id, 5_UInt8), equals(__table2.id, 6_UInt8)) UInt8 : 5 - Positions: 5 0 1 2 3 + COLUMN Const(UInt8) -> 5_UInt8 UInt8 : 1 + FUNCTION equals(__table1.id : 0, 5_UInt8 :: 1) -> equals(__table1.id, 5_UInt8) UInt8 : 2 + Positions: 2 0 2 + Filter column: and(equals(__table1.id, 5_UInt8), equals(__table2.id, 6_UInt8)) (removed) + Actions: INPUT :: 1 -> __table1.id UInt64 : 0 + INPUT :: 2 -> __table1.value String : 1 + INPUT :: 3 -> __table2.value String : 2 + INPUT : 4 -> __table2.id UInt64 : 3 + COLUMN Const(UInt8) -> 6_UInt8 UInt8 : 4 + INPUT : 0 -> equals(__table1.id, 5_UInt8) UInt8 : 5 + FUNCTION equals(__table2.id : 3, 6_UInt8 :: 4) -> equals(__table2.id, 6_UInt8) UInt8 : 6 + FUNCTION and(equals(__table1.id, 5_UInt8) :: 5, equals(__table2.id, 6_UInt8) :: 6) -> and(equals(__table1.id, 5_UInt8), equals(__table2.id, 6_UInt8)) UInt8 : 4 + Positions: 4 0 1 2 3 Join (JOIN FillRightFirst) Header: __table1.id UInt64 __table1.value String diff --git a/tests/queries/0_stateless/03096_variant_in_primary_key.reference b/tests/queries/0_stateless/03096_variant_in_primary_key.reference deleted file mode 100644 index c701d7d3c26..00000000000 --- a/tests/queries/0_stateless/03096_variant_in_primary_key.reference +++ /dev/null @@ -1,4 +0,0 @@ -1 str_1 -1 str_2 -1 1 -1 2 diff --git a/tests/queries/0_stateless/03096_variant_in_primary_key.sql b/tests/queries/0_stateless/03096_variant_in_primary_key.sql deleted file mode 100644 index 48fbc821bcc..00000000000 --- a/tests/queries/0_stateless/03096_variant_in_primary_key.sql +++ /dev/null @@ -1,7 +0,0 @@ -set allow_experimental_variant_type=1; -drop table if exists test; -create table test (id UInt64, v Variant(UInt64, String)) engine=MergeTree order by (id, v); -insert into test values (1, 1), (1, 'str_1'), (1, 2), (1, 'str_2'); -select * from test; -drop table test; - diff --git a/tests/queries/0_stateless/03141_wildcard_grants.sql b/tests/queries/0_stateless/03141_wildcard_grants.sql index 45962d9b929..e71fa531134 100644 --- a/tests/queries/0_stateless/03141_wildcard_grants.sql +++ b/tests/queries/0_stateless/03141_wildcard_grants.sql @@ -19,4 +19,6 @@ REVOKE SELECT ON team*.* FROM user_03141; SHOW GRANTS FOR user_03141; SELECT '---'; +GRANT SELECT(bar) ON foo.test* TO user_03141; -- { clientError SYNTAX_ERROR } + DROP USER user_03141; diff --git a/tests/queries/0_stateless/03150_dynamic_type_mv_insert.sql b/tests/queries/0_stateless/03150_dynamic_type_mv_insert.sql index 71d5dd4abd1..0e5119a38e0 100644 --- a/tests/queries/0_stateless/03150_dynamic_type_mv_insert.sql +++ b/tests/queries/0_stateless/03150_dynamic_type_mv_insert.sql @@ -1,4 +1,5 @@ SET allow_experimental_dynamic_type=1; +SET allow_suspicious_types_in_order_by=1; DROP TABLE IF EXISTS null_table; CREATE TABLE null_table diff --git a/tests/queries/0_stateless/03151_dynamic_type_scale_max_types.sql b/tests/queries/0_stateless/03151_dynamic_type_scale_max_types.sql index e476d34a1db..f00c1492e40 100644 --- a/tests/queries/0_stateless/03151_dynamic_type_scale_max_types.sql +++ b/tests/queries/0_stateless/03151_dynamic_type_scale_max_types.sql @@ -1,5 +1,6 @@ -SET allow_experimental_dynamic_type=1; -set min_compress_block_size = 585572, max_compress_block_size = 373374, max_block_size = 60768, max_joined_block_size_rows = 18966, max_insert_threads = 5, max_threads = 50, max_read_buffer_size = 708232, connect_timeout_with_failover_ms = 2000, connect_timeout_with_failover_secure_ms = 3000, idle_connection_timeout = 36000, use_uncompressed_cache = true, stream_like_engine_allow_direct_select = true, replication_wait_for_inactive_replica_timeout = 30, compile_aggregate_expressions = false, min_count_to_compile_aggregate_expression = 0, compile_sort_description = false, group_by_two_level_threshold = 1000000, group_by_two_level_threshold_bytes = 12610083, enable_memory_bound_merging_of_aggregation_results = false, min_chunk_bytes_for_parallel_parsing = 18769830, merge_tree_coarse_index_granularity = 12, min_bytes_to_use_direct_io = 10737418240, min_bytes_to_use_mmap_io = 10737418240, log_queries = true, insert_quorum_timeout = 60000, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.05000000074505806, http_response_buffer_size = 294986, fsync_metadata = true, http_send_timeout = 60., http_receive_timeout = 60., opentelemetry_start_trace_probability = 0.10000000149011612, max_bytes_before_external_group_by = 1, max_bytes_before_external_sort = 10737418240, max_bytes_before_remerge_sort = 1326536545, max_untracked_memory = 1048576, memory_profiler_step = 1048576, log_comment = '03151_dynamic_type_scale_max_types.sql', send_logs_level = 'fatal', prefer_localhost_replica = false, optimize_read_in_order = false, optimize_aggregation_in_order = true, aggregation_in_order_max_block_bytes = 27069500, read_in_order_two_level_merge_threshold = 75, allow_introspection_functions = true, database_atomic_wait_for_drop_and_detach_synchronously = true, remote_filesystem_read_method = 'read', local_filesystem_read_prefetch = true, remote_filesystem_read_prefetch = false, merge_tree_compact_parts_min_granules_to_multibuffer_read = 119, async_insert_busy_timeout_max_ms = 5000, read_from_filesystem_cache_if_exists_otherwise_bypass_cache = true, filesystem_cache_segments_batch_size = 10, use_page_cache_for_disks_without_file_cache = true, page_cache_inject_eviction = true, allow_prefetched_read_pool_for_remote_filesystem = false, filesystem_prefetch_step_marks = 50, filesystem_prefetch_min_bytes_for_single_read_task = 16777216, filesystem_prefetch_max_memory_usage = 134217728, filesystem_prefetches_limit = 10, optimize_sorting_by_input_stream_properties = false, allow_experimental_dynamic_type = true, session_timezone = 'Africa/Khartoum', prefer_warmed_unmerged_parts_seconds = 2; +SET allow_experimental_dynamic_type = 1; +SET allow_suspicious_types_in_order_by = 1; +SET optimize_read_in_order = 1; drop table if exists to_table; diff --git a/tests/queries/0_stateless/03158_dynamic_type_from_variant.sql b/tests/queries/0_stateless/03158_dynamic_type_from_variant.sql index a18f985f217..429ac21b5eb 100644 --- a/tests/queries/0_stateless/03158_dynamic_type_from_variant.sql +++ b/tests/queries/0_stateless/03158_dynamic_type_from_variant.sql @@ -1,5 +1,6 @@ SET allow_experimental_dynamic_type=1; SET allow_experimental_variant_type=1; +SET allow_suspicious_types_in_order_by=1; CREATE TABLE test_variable (v Variant(String, UInt32, IPv6, Bool, DateTime64)) ENGINE = Memory; CREATE TABLE test_dynamic (d Dynamic) ENGINE = Memory; diff --git a/tests/queries/0_stateless/03159_dynamic_type_all_types.sql b/tests/queries/0_stateless/03159_dynamic_type_all_types.sql index 28b679e2214..cf8ba687d3f 100644 --- a/tests/queries/0_stateless/03159_dynamic_type_all_types.sql +++ b/tests/queries/0_stateless/03159_dynamic_type_all_types.sql @@ -3,7 +3,7 @@ SET allow_experimental_dynamic_type=1; SET allow_experimental_variant_type=1; SET allow_suspicious_low_cardinality_types=1; - +SET allow_suspicious_types_in_order_by=1; CREATE TABLE t (d Dynamic(max_types=254)) ENGINE = Memory; -- Integer types: signed and unsigned integers (UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256) diff --git a/tests/queries/0_stateless/03162_dynamic_type_nested.sql b/tests/queries/0_stateless/03162_dynamic_type_nested.sql index 94007459a9e..59c22491957 100644 --- a/tests/queries/0_stateless/03162_dynamic_type_nested.sql +++ b/tests/queries/0_stateless/03162_dynamic_type_nested.sql @@ -1,4 +1,5 @@ SET allow_experimental_dynamic_type=1; +SET allow_suspicious_types_in_order_by=1; CREATE TABLE t (d Dynamic) ENGINE = Memory; diff --git a/tests/queries/0_stateless/03163_dynamic_as_supertype.sql b/tests/queries/0_stateless/03163_dynamic_as_supertype.sql index baba637eea4..e859fbd1815 100644 --- a/tests/queries/0_stateless/03163_dynamic_as_supertype.sql +++ b/tests/queries/0_stateless/03163_dynamic_as_supertype.sql @@ -1,4 +1,5 @@ SET allow_experimental_dynamic_type=1; +SET allow_suspicious_types_in_order_by=1; SELECT if(number % 2, number::Dynamic(max_types=3), ('str_' || toString(number))::Dynamic(max_types=2)) AS d, toTypeName(d), dynamicType(d) FROM numbers(4); CREATE TABLE dynamic_test_1 (d Dynamic(max_types=3)) ENGINE = Memory; INSERT INTO dynamic_test_1 VALUES ('str_1'), (42::UInt64); diff --git a/tests/queries/0_stateless/03198_bit_shift_throws_error_for_out_of_bounds.reference b/tests/queries/0_stateless/03198_bit_shift_throws_error_for_out_of_bounds.reference index 33b8cd6ee26..1fda82a9747 100644 --- a/tests/queries/0_stateless/03198_bit_shift_throws_error_for_out_of_bounds.reference +++ b/tests/queries/0_stateless/03198_bit_shift_throws_error_for_out_of_bounds.reference @@ -1,3 +1,9 @@ -- bitShiftRight +0 + +\0\0\0\0\0\0\0\0 -- bitShiftLeft +0 + +\0\0\0\0\0\0\0\0 OK diff --git a/tests/queries/0_stateless/03198_bit_shift_throws_error_for_out_of_bounds.sql b/tests/queries/0_stateless/03198_bit_shift_throws_error_for_out_of_bounds.sql index aec01753673..340cc1292e4 100644 --- a/tests/queries/0_stateless/03198_bit_shift_throws_error_for_out_of_bounds.sql +++ b/tests/queries/0_stateless/03198_bit_shift_throws_error_for_out_of_bounds.sql @@ -1,17 +1,17 @@ SELECT '-- bitShiftRight'; SELECT bitShiftRight(1, -1); -- { serverError ARGUMENT_OUT_OF_BOUND } -SELECT bitShiftRight(toUInt8(1), 8 + 1); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT bitShiftRight(toUInt8(1), 8 + 1); SELECT bitShiftRight('hola', -1); -- { serverError ARGUMENT_OUT_OF_BOUND } -SELECT bitShiftRight('hola', 4 * 8 + 1); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT bitShiftRight('hola', 4 * 8 + 1); SELECT bitShiftRight(toFixedString('hola', 8), -1); -- { serverError ARGUMENT_OUT_OF_BOUND } -SELECT bitShiftRight(toFixedString('hola', 8), 8 * 8 + 1); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT bitShiftRight(toFixedString('hola', 8), 8 * 8 + 1); SELECT '-- bitShiftLeft'; SELECT bitShiftLeft(1, -1); -- { serverError ARGUMENT_OUT_OF_BOUND } -SELECT bitShiftLeft(toUInt8(1), 8 + 1); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT bitShiftLeft(toUInt8(1), 8 + 1); SELECT bitShiftLeft('hola', -1); -- { serverError ARGUMENT_OUT_OF_BOUND } -SELECT bitShiftLeft('hola', 4 * 8 + 1); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT bitShiftLeft('hola', 4 * 8 + 1); SELECT bitShiftLeft(toFixedString('hola', 8), -1); -- { serverError ARGUMENT_OUT_OF_BOUND } -SELECT bitShiftLeft(toFixedString('hola', 8), 8 * 8 + 1); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT bitShiftLeft(toFixedString('hola', 8), 8 * 8 + 1); SELECT 'OK'; \ No newline at end of file diff --git a/tests/queries/0_stateless/03199_atomic_clickhouse_local.reference b/tests/queries/0_stateless/03199_atomic_clickhouse_local.reference new file mode 100644 index 00000000000..1975397394b --- /dev/null +++ b/tests/queries/0_stateless/03199_atomic_clickhouse_local.reference @@ -0,0 +1,6 @@ +123 +Hello +['Hello','world'] +Hello +Hello +['Hello','world'] diff --git a/tests/queries/0_stateless/03199_atomic_clickhouse_local.sh b/tests/queries/0_stateless/03199_atomic_clickhouse_local.sh new file mode 100755 index 00000000000..edaa83b8f95 --- /dev/null +++ b/tests/queries/0_stateless/03199_atomic_clickhouse_local.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_LOCAL} -n " +CREATE TABLE test (x UInt8) ORDER BY x; +INSERT INTO test VALUES (123); +SELECT * FROM test; +CREATE OR REPLACE TABLE test (s String) ORDER BY s; +INSERT INTO test VALUES ('Hello'); +SELECT * FROM test; +RENAME TABLE test TO test2; +CREATE OR REPLACE TABLE test (s Array(String)) ORDER BY s; +INSERT INTO test VALUES (['Hello', 'world']); +SELECT * FROM test; +SELECT * FROM test2; +EXCHANGE TABLES test AND test2; +SELECT * FROM test; +SELECT * FROM test2; +DROP TABLE test; +DROP TABLE test2; +" diff --git a/tests/queries/0_stateless/03199_merge_filters_bug.sql b/tests/queries/0_stateless/03199_merge_filters_bug.sql index ed2ec2ea217..696856c9121 100644 --- a/tests/queries/0_stateless/03199_merge_filters_bug.sql +++ b/tests/queries/0_stateless/03199_merge_filters_bug.sql @@ -1,3 +1,5 @@ +set allow_reorder_prewhere_conditions=0; + drop table if exists t1; drop table if exists t2; @@ -49,7 +51,23 @@ tmp1 AS fs1 FROM t2 LEFT JOIN tmp1 USING (fs1) - WHERE (fs1 IN ('test')) SETTINGS enable_multiple_prewhere_read_steps = 0; + WHERE (fs1 IN ('test')) SETTINGS enable_multiple_prewhere_read_steps = 0, query_plan_merge_filters=0; + +WITH +tmp1 AS +( + SELECT + CAST(s1, 'FixedString(10)') AS fs1, + s2 AS sector, + s3 + FROM t1 + WHERE (s3 != 'test') +) + SELECT + fs1 + FROM t2 + LEFT JOIN tmp1 USING (fs1) + WHERE (fs1 IN ('test')) SETTINGS enable_multiple_prewhere_read_steps = 1, query_plan_merge_filters=1; optimize table t1 final; @@ -67,4 +85,20 @@ tmp1 AS fs1 FROM t2 LEFT JOIN tmp1 USING (fs1) - WHERE (fs1 IN ('test')); + WHERE (fs1 IN ('test')) SETTINGS enable_multiple_prewhere_read_steps = 0, query_plan_merge_filters=0; + +WITH +tmp1 AS +( + SELECT + CAST(s1, 'FixedString(10)') AS fs1, + s2 AS sector, + s3 + FROM t1 + WHERE (s3 != 'test') +) + SELECT + fs1 + FROM t2 + LEFT JOIN tmp1 USING (fs1) + WHERE (fs1 IN ('test')) SETTINGS enable_multiple_prewhere_read_steps = 1, query_plan_merge_filters=1; diff --git a/tests/queries/0_stateless/03203_system_query_metric_log.reference b/tests/queries/0_stateless/03203_system_query_metric_log.reference index 940b0c4e178..fa8e27a7e90 100644 --- a/tests/queries/0_stateless/03203_system_query_metric_log.reference +++ b/tests/queries/0_stateless/03203_system_query_metric_log.reference @@ -23,8 +23,8 @@ --Interval 123: check that the SleepFunctionCalls, SleepFunctionMilliseconds and ProfileEvent_SleepFunctionElapsedMicroseconds are correct 1 --Check that a query_metric_log_interval=0 disables the collection -0 +1 -Check that a query which execution time is less than query_metric_log_interval is never collected -0 +1 --Check that there is a final event when queries finish -3 +1 diff --git a/tests/queries/0_stateless/03203_system_query_metric_log.sh b/tests/queries/0_stateless/03203_system_query_metric_log.sh index bf94be79d7c..abcd14c8e5d 100755 --- a/tests/queries/0_stateless/03203_system_query_metric_log.sh +++ b/tests/queries/0_stateless/03203_system_query_metric_log.sh @@ -84,17 +84,17 @@ check_log 123 # query_metric_log_interval=0 disables the collection altogether $CLICKHOUSE_CLIENT -m -q """ SELECT '--Check that a query_metric_log_interval=0 disables the collection'; - SELECT count() FROM system.query_metric_log WHERE event_date >= yesterday() AND query_id = '${query_prefix}_0' + SELECT count() == 0 FROM system.query_metric_log WHERE event_date >= yesterday() AND query_id = '${query_prefix}_0' """ # a quick query that takes less than query_metric_log_interval is never collected $CLICKHOUSE_CLIENT -m -q """ SELECT '-Check that a query which execution time is less than query_metric_log_interval is never collected'; - SELECT count() FROM system.query_metric_log WHERE event_date >= yesterday() AND query_id = '${query_prefix}_fast' + SELECT count() == 0 FROM system.query_metric_log WHERE event_date >= yesterday() AND query_id = '${query_prefix}_fast' """ # a query that takes more than query_metric_log_interval is collected including the final row $CLICKHOUSE_CLIENT -m -q """ SELECT '--Check that there is a final event when queries finish'; - SELECT count() FROM system.query_metric_log WHERE event_date >= yesterday() AND query_id = '${query_prefix}_1000' + SELECT count() > 2 FROM system.query_metric_log WHERE event_date >= yesterday() AND query_id = '${query_prefix}_1000' """ diff --git a/tests/queries/0_stateless/03214_json_typed_dynamic_path.sql b/tests/queries/0_stateless/03214_json_typed_dynamic_path.sql index 1f6a025825a..eee3d70b8da 100644 --- a/tests/queries/0_stateless/03214_json_typed_dynamic_path.sql +++ b/tests/queries/0_stateless/03214_json_typed_dynamic_path.sql @@ -1,6 +1,7 @@ -- Tags: no-fasttest set allow_experimental_json_type = 1; +set allow_experimental_dynamic_type = 1; drop table if exists test; create table test (json JSON(a Dynamic)) engine=MergeTree order by tuple() settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1; insert into test select '{"a" : 42}'; diff --git a/tests/queries/0_stateless/03222_ignore_nulls_query_tree_elimination.reference b/tests/queries/0_stateless/03222_ignore_nulls_query_tree_elimination.reference new file mode 100644 index 00000000000..1f242fa6f00 --- /dev/null +++ b/tests/queries/0_stateless/03222_ignore_nulls_query_tree_elimination.reference @@ -0,0 +1,3 @@ +3 +3 +3 diff --git a/tests/queries/0_stateless/03222_ignore_nulls_query_tree_elimination.sql b/tests/queries/0_stateless/03222_ignore_nulls_query_tree_elimination.sql new file mode 100644 index 00000000000..72f9781ed45 --- /dev/null +++ b/tests/queries/0_stateless/03222_ignore_nulls_query_tree_elimination.sql @@ -0,0 +1,51 @@ +#!/usr/bin/env -S ${HOME}/clickhouse-client --queries-file + +DROP TABLE IF EXISTS with_fill_date__fuzz_0; + +CREATE TABLE with_fill_date__fuzz_0 +( + `d` Date, + `d32` Nullable(Int32), + `d33` Int32 +) +ENGINE = Memory; + + +INSERT INTO with_fill_date__fuzz_0 VALUES (toDate('2020-03-03'), 1, 3), (toDate('2020-03-03'), NULL, 3), (toDate('2020-02-05'), 1, 1); + + +SELECT count() +FROM with_fill_date__fuzz_0 +ORDER BY + count(), + count() IGNORE NULLS, + max(d) +WITH FILL STEP toIntervalDay(10) +; + + +SELECT count() +FROM with_fill_date__fuzz_0 +ORDER BY + any(d32) RESPECT NULLS, + any_respect_nulls(d32), + max(d) +WITH FILL STEP toIntervalDay(10) +; + + +SELECT count() +FROM with_fill_date__fuzz_0 +ORDER BY + any(d32), + any(d32) IGNORE NULLS, + any(d32) RESPECT NULLS, + any_respect_nulls(d32) IGNORE NULLS, + any_respect_nulls(d32), + sum(d33), + sum(d33) IGNORE NULLS, + max(d) +WITH FILL STEP toIntervalDay(10) +; + + diff --git a/tests/queries/0_stateless/03224_arrayUnion.reference b/tests/queries/0_stateless/03224_arrayUnion.reference index b900b6cdb0a..9b871234d27 100644 --- a/tests/queries/0_stateless/03224_arrayUnion.reference +++ b/tests/queries/0_stateless/03224_arrayUnion.reference @@ -41,3 +41,13 @@ [1,2,3,4,5,10,20] ------- [1,2,3] +------- +[10,-2,1] ['hello','hi'] [3,2,1,NULL] +------- +------- +[1] +------- +[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256] +199999 +------- +[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19] diff --git a/tests/queries/0_stateless/03224_arrayUnion.sql b/tests/queries/0_stateless/03224_arrayUnion.sql index dedbacad906..14a9192f39a 100644 --- a/tests/queries/0_stateless/03224_arrayUnion.sql +++ b/tests/queries/0_stateless/03224_arrayUnion.sql @@ -35,4 +35,23 @@ SELECT arraySort(arrayUnion([NULL, NULL, NULL, 1], [1, NULL, NULL], [1, 2, 3, NU select '-------'; SELECT arraySort(arrayUnion([1, 1, 1, 2, 3], [2, 2, 4], [5, 10, 20])); select '-------'; -SELECT arraySort(arrayUnion([1, 2], [1, 3], [])), +SELECT arraySort(arrayUnion([1, 2], [1, 3], [])); +select '-------'; +-- example from docs +SELECT + arrayUnion([-2, 1], [10, 1], [-2], []) as num_example, + arrayUnion(['hi'], [], ['hello', 'hi']) as str_example, + arrayUnion([1, 3, NULL], [2, 3, NULL]) as null_example; +select '-------'; +--mix of types +SELECT arrayUnion([1], [-2], [1.1, 'hi'], [NULL, 'hello', []]); -- {serverError NO_COMMON_TYPE} +select '-------'; +SELECT arrayUnion([1]); +SELECT arrayUnion(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} +select '-------'; +--bigger arrays +SELECT arraySort(arrayUnion(range(1, 256), range(2, 257))); +SELECT length(arrayUnion(range(1, 100000), range(9999, 200000))); +select '-------'; +--bigger number of arguments +SELECT arraySort(arrayUnion([1, 2], [1, 3], [1, 4], [1, 5], [1, 6], [1, 7], [1, 8], [1, 9], [1, 10], [1, 11], [1, 12], [1, 13], [1, 14], [1, 15], [1, 16], [1, 17], [1, 18], [1, 19])); diff --git a/tests/queries/0_stateless/03225_alter_to_json_not_supported.sql b/tests/queries/0_stateless/03225_alter_to_json_not_supported.sql deleted file mode 100644 index 398494d56de..00000000000 --- a/tests/queries/0_stateless/03225_alter_to_json_not_supported.sql +++ /dev/null @@ -1,15 +0,0 @@ -set allow_experimental_json_type = 1; - -drop table if exists test; -create table test (s String) engine=MergeTree order by tuple(); -alter table test modify column s JSON; -- { serverError BAD_ARGUMENTS } -drop table test; - -create table test (s Array(String)) engine=MergeTree order by tuple(); -alter table test modify column s Array(JSON); -- { serverError BAD_ARGUMENTS } -drop table test; - -create table test (s Tuple(String, String)) engine=MergeTree order by tuple(); -alter table test modify column s Tuple(JSON, String); -- { serverError BAD_ARGUMENTS } -drop table test; - diff --git a/tests/queries/0_stateless/03228_dynamic_serializations_uninitialized_value.sql b/tests/queries/0_stateless/03228_dynamic_serializations_uninitialized_value.sql index 8a565fe36b9..60e2439d45f 100644 --- a/tests/queries/0_stateless/03228_dynamic_serializations_uninitialized_value.sql +++ b/tests/queries/0_stateless/03228_dynamic_serializations_uninitialized_value.sql @@ -1,4 +1,5 @@ set allow_experimental_dynamic_type=1; +set allow_suspicious_types_in_group_by=1; set cast_keep_nullable=1; SELECT toFixedString('str', 3), 3, CAST(if(1 = 0, toInt8(3), NULL), 'Int32') AS x from numbers(10) GROUP BY GROUPING SETS ((CAST(toInt32(1), 'Int32')), ('str', 3), (CAST(toFixedString('str', 3), 'Dynamic')), (CAST(toFixedString(toFixedString('str', 3), 3), 'Dynamic'))); diff --git a/tests/queries/0_stateless/03231_dynamic_incomplete_type_insert_bug.sql b/tests/queries/0_stateless/03231_dynamic_incomplete_type_insert_bug.sql index a6fc2e66480..4e845a66574 100644 --- a/tests/queries/0_stateless/03231_dynamic_incomplete_type_insert_bug.sql +++ b/tests/queries/0_stateless/03231_dynamic_incomplete_type_insert_bug.sql @@ -1,4 +1,5 @@ SET allow_experimental_dynamic_type = 1; +SET allow_suspicious_types_in_order_by = 1; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c0 Array(Dynamic)) ENGINE = MergeTree() ORDER BY tuple(); INSERT INTO t1 (c0) VALUES ([]); diff --git a/tests/queries/0_stateless/03231_dynamic_not_safe_primary_key.sql b/tests/queries/0_stateless/03231_dynamic_not_safe_primary_key.sql deleted file mode 100644 index f207581f482..00000000000 --- a/tests/queries/0_stateless/03231_dynamic_not_safe_primary_key.sql +++ /dev/null @@ -1,10 +0,0 @@ -SET allow_experimental_dynamic_type = 1; -DROP TABLE IF EXISTS t0; -DROP TABLE IF EXISTS t1; -CREATE TABLE t0 (c0 Int) ENGINE = AggregatingMergeTree() ORDER BY (c0); -CREATE TABLE t1 (c0 Array(Dynamic), c1 Int) ENGINE = MergeTree() ORDER BY (c0); -INSERT INTO t1 (c0, c1) VALUES ([18446717433683171873], 13623876564923702671), ([-4], 6111684076076982207); -SELECT 1 FROM t0 FINAL JOIN t1 ON TRUE; -DROP TABLE t0; -DROP TABLE t1; - diff --git a/tests/queries/0_stateless/03231_dynamic_uniq_group_by.sql b/tests/queries/0_stateless/03231_dynamic_uniq_group_by.sql index fe052027f56..d8869e71405 100644 --- a/tests/queries/0_stateless/03231_dynamic_uniq_group_by.sql +++ b/tests/queries/0_stateless/03231_dynamic_uniq_group_by.sql @@ -1,4 +1,6 @@ set allow_experimental_dynamic_type = 1; +set allow_suspicious_types_in_group_by = 1; +set allow_suspicious_types_in_order_by = 1; drop table if exists test; create table test (d Dynamic(max_types=2)) engine=Memory; insert into test values (42), ('Hello'), ([1,2,3]), ('2020-01-01'); diff --git a/tests/queries/0_stateless/03231_dynamic_variant_in_order_by_group_by.reference b/tests/queries/0_stateless/03231_dynamic_variant_in_order_by_group_by.reference new file mode 100644 index 00000000000..5983dd15f5b --- /dev/null +++ b/tests/queries/0_stateless/03231_dynamic_variant_in_order_by_group_by.reference @@ -0,0 +1,184 @@ +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +[0] +[1] +[2] +[3] +[4] +{'str':0} +{'str':1} +{'str':2} +{'str':3} +{'str':4} +0 +1 +2 +3 +4 +\N +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +[0] +[1] +[2] +[3] +[4] +{'str':0} +{'str':1} +{'str':2} +{'str':3} +{'str':4} +0 +1 +2 +3 +4 +\N +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +[0] +[1] +[2] +[3] +[4] +{'str':0} +{'str':1} +{'str':2} +{'str':3} +{'str':4} +0 +1 +2 +3 +4 +\N +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +0 +1 +2 +3 +4 +[0] +[1] +[2] +[3] +[4] +{'str':0} +{'str':1} +{'str':2} +{'str':3} +{'str':4} +0 +1 +2 +3 +4 +\N diff --git a/tests/queries/0_stateless/03231_dynamic_variant_in_order_by_group_by.sql b/tests/queries/0_stateless/03231_dynamic_variant_in_order_by_group_by.sql new file mode 100644 index 00000000000..a53b02e8e41 --- /dev/null +++ b/tests/queries/0_stateless/03231_dynamic_variant_in_order_by_group_by.sql @@ -0,0 +1,166 @@ +set allow_experimental_variant_type=1; +set allow_experimental_dynamic_type=1; + +drop table if exists test; + +create table test (d Dynamic) engine=MergeTree order by d; -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Dynamic) engine=MergeTree order by tuple(d); -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Dynamic) engine=MergeTree order by array(d); -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Dynamic) engine=MergeTree order by map('str', d); -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Dynamic) engine=MergeTree order by tuple() primary key d; -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Dynamic) engine=MergeTree order by tuple() partition by d; -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Dynamic) engine=MergeTree order by tuple() partition by tuple(d); -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Dynamic) engine=MergeTree order by tuple() partition by array(d); -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Dynamic) engine=MergeTree order by tuple() partition by map('str', d); -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} + +create table test (d Variant(UInt64)) engine=MergeTree order by d; -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Variant(UInt64)) engine=MergeTree order by tuple(d); -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Variant(UInt64)) engine=MergeTree order by array(d); -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Variant(UInt64)) engine=MergeTree order by map('str', d); -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Variant(UInt64)) engine=MergeTree order by tuple() primary key d; -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Variant(UInt64)) engine=MergeTree order by tuple() partition by d; -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Variant(UInt64)) engine=MergeTree order by tuple() partition by tuple(d); -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Variant(UInt64)) engine=MergeTree order by tuple() partition by array(d); -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} +create table test (d Variant(UInt64)) engine=MergeTree order by tuple() partition by map('str', d); -- {serverError DATA_TYPE_CANNOT_BE_USED_IN_KEY} + +create table test (d Dynamic) engine=Memory; +insert into test select * from numbers(5); + +set allow_experimental_analyzer=1; + +set allow_suspicious_types_in_group_by=1; +set allow_suspicious_types_in_order_by=0; + +select * from test order by d; -- {serverError ILLEGAL_COLUMN} +select * from test order by tuple(d); -- {serverError ILLEGAL_COLUMN} +select * from test order by array(d); -- {serverError ILLEGAL_COLUMN} +select * from test order by map('str', d); -- {serverError ILLEGAL_COLUMN} + +set allow_suspicious_types_in_group_by=0; +set allow_suspicious_types_in_order_by=1; + +select * from test group by d; -- {serverError ILLEGAL_COLUMN} +select * from test group by tuple(d); -- {serverError ILLEGAL_COLUMN} +select array(d) from test group by array(d); -- {serverError ILLEGAL_COLUMN} +select map('str', d) from test group by map('str', d); -- {serverError ILLEGAL_COLUMN} +select * from test group by grouping sets ((d), ('str')); -- {serverError ILLEGAL_COLUMN} + +set allow_suspicious_types_in_group_by=1; +set allow_suspicious_types_in_order_by=1; + +select * from test order by d; +select * from test order by tuple(d); +select * from test order by array(d); +select * from test order by map('str', d); + +select * from test group by d order by all; +select * from test group by tuple(d) order by all; +select array(d) from test group by array(d) order by all; +select map('str', d) from test group by map('str', d) order by all; +select * from test group by grouping sets ((d), ('str')) order by all; + +set allow_experimental_analyzer=0; + +set allow_suspicious_types_in_group_by=1; +set allow_suspicious_types_in_order_by=0; + +select * from test order by d; -- {serverError ILLEGAL_COLUMN} +select * from test order by tuple(d); -- {serverError ILLEGAL_COLUMN} +select * from test order by array(d); -- {serverError ILLEGAL_COLUMN} +select * from test order by map('str', d); -- {serverError ILLEGAL_COLUMN} + +set allow_suspicious_types_in_group_by=0; +set allow_suspicious_types_in_order_by=1; + +select * from test group by d; -- {serverError ILLEGAL_COLUMN} +select * from test group by tuple(d); -- {serverError ILLEGAL_COLUMN} +select array(d) from test group by array(d); -- {serverError ILLEGAL_COLUMN} +select map('str', d) from test group by map('str', d); -- {serverError ILLEGAL_COLUMN} +select * from test group by grouping sets ((d), ('str')); -- {serverError ILLEGAL_COLUMN} + +set allow_suspicious_types_in_group_by=1; +set allow_suspicious_types_in_order_by=1; + +select * from test order by d; +select * from test order by tuple(d); +select * from test order by array(d); +select * from test order by map('str', d); + +select * from test group by d order by all; +select * from test group by tuple(d) order by all; +select array(d) from test group by array(d) order by all; +select map('str', d) from test group by map('str', d) order by all; +select * from test group by grouping sets ((d), ('str')) order by all; + +drop table test; + +create table test (d Variant(UInt64)) engine=Memory; +insert into test select * from numbers(5); + +set allow_experimental_analyzer=1; + +set allow_suspicious_types_in_group_by=1; +set allow_suspicious_types_in_order_by=0; + +select * from test order by d; -- {serverError ILLEGAL_COLUMN} +select * from test order by tuple(d); -- {serverError ILLEGAL_COLUMN} +select * from test order by array(d); -- {serverError ILLEGAL_COLUMN} +select * from test order by map('str', d); -- {serverError ILLEGAL_COLUMN} + +set allow_suspicious_types_in_group_by=0; +set allow_suspicious_types_in_order_by=1; + +select * from test group by d; -- {serverError ILLEGAL_COLUMN} +select * from test group by tuple(d); -- {serverError ILLEGAL_COLUMN} +select array(d) from test group by array(d); -- {serverError ILLEGAL_COLUMN} +select map('str', d) from test group by map('str', d); -- {serverError ILLEGAL_COLUMN} +select * from test group by grouping sets ((d), ('str')); -- {serverError ILLEGAL_COLUMN} + +set allow_suspicious_types_in_group_by=1; +set allow_suspicious_types_in_order_by=1; + +select * from test order by d; +select * from test order by tuple(d); +select * from test order by array(d); +select * from test order by map('str', d); + +select * from test group by d order by all; +select * from test group by tuple(d) order by all; +select array(d) from test group by array(d) order by all; +select map('str', d) from test group by map('str', d) order by all; +select * from test group by grouping sets ((d), ('str')) order by all; + +set allow_experimental_analyzer=0; + +set allow_suspicious_types_in_group_by=1; +set allow_suspicious_types_in_order_by=0; + +select * from test order by d; -- {serverError ILLEGAL_COLUMN} +select * from test order by tuple(d); -- {serverError ILLEGAL_COLUMN} +select * from test order by array(d); -- {serverError ILLEGAL_COLUMN} +select * from test order by map('str', d); -- {serverError ILLEGAL_COLUMN} + +set allow_suspicious_types_in_group_by=0; +set allow_suspicious_types_in_order_by=1; + +select * from test group by d; -- {serverError ILLEGAL_COLUMN} +select * from test group by tuple(d); -- {serverError ILLEGAL_COLUMN} +select array(d) from test group by array(d); -- {serverError ILLEGAL_COLUMN} +select map('str', d) from test group by map('str', d); -- {serverError ILLEGAL_COLUMN} +select * from test group by grouping sets ((d), ('str')); -- {serverError ILLEGAL_COLUMN} + +set allow_suspicious_types_in_group_by=1; +set allow_suspicious_types_in_order_by=1; + +select * from test order by d; +select * from test order by tuple(d); +select * from test order by array(d); +select * from test order by map('str', d); + +select * from test group by d order by all; +select * from test group by tuple(d) order by all; +select array(d) from test group by array(d) order by all; +select map('str', d) from test group by map('str', d) order by all; +select * from test group by grouping sets ((d), ('str')) order by all; + +drop table test; diff --git a/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.reference b/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.reference index 23cbe2bfdec..ccb315b8305 100644 --- a/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.reference +++ b/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.reference @@ -1,6 +1,6 @@ quantileExactWeightedInterpolated -0 0 0 Decimal(38, 8) --25.5 -8.49999999 -5.1 Decimal(38, 8) +0 0 0 25 2024-02-20 Decimal(38, 8) +-25.5 -8.49999999 -5.1 12.25 2024-01-25 Decimal(38, 8) 0 0 0 10 3.33333333 2 20 6.66666666 4 @@ -10,11 +10,14 @@ quantileExactWeightedInterpolated [-50,-40,-30,-20,-10,0,10,20,30,40,50] [-16.66666666,-13.33333333,-10,-6.66666666,-3.33333333,0,3.33333333,6.66666666,10,13.33333333,16.66666666] [-10,-8,-6,-4,-2,0,2,4,6,8,10] +[0,5,10,15,20,25,30,35,40,45,50] +['2024-01-01','2024-01-11','2024-01-21','2024-01-31','2024-02-10','2024-02-20','2024-03-01','2024-03-11','2024-03-21','2024-03-31','2024-04-10'] quantileExactWeightedInterpolatedState [10000.6,20000.2,29999.8,39999.4] Test with filter that returns no rows -0 0 0 +0 0 0 nan 1970-01-01 +0 0 0 nan 1970-01-01 Test with dynamic weights -21 7 4.2 +21 7 4.2 35.5 2024-03-12 Test with all weights set to 0 -0 0 0 +0 0 0 nan 1970-01-01 diff --git a/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.sql b/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.sql index dba16eae22a..a64b46e751b 100644 --- a/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.sql +++ b/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.sql @@ -5,16 +5,28 @@ CREATE TABLE decimal a Decimal32(4), b Decimal64(8), c Decimal128(8), + f Float64, + d Date, w UInt64 ) ENGINE = Memory; -INSERT INTO decimal (a, b, c, w) -SELECT toDecimal32(number - 50, 4), toDecimal64(number - 50, 8) / 3, toDecimal128(number - 50, 8) / 5, number +INSERT INTO decimal (a, b, c, f, d, w) +SELECT toDecimal32(number - 50, 4), toDecimal64(number - 50, 8) / 3, toDecimal128(number - 50, 8) / 5, number/2, addDays(toDate('2024-01-01'), number), number FROM system.numbers LIMIT 101; SELECT 'quantileExactWeightedInterpolated'; -SELECT medianExactWeightedInterpolated(a, 1), medianExactWeightedInterpolated(b, 2), medianExactWeightedInterpolated(c, 3) as x, toTypeName(x) FROM decimal; -SELECT quantileExactWeightedInterpolated(a, 1), quantileExactWeightedInterpolated(b, 2), quantileExactWeightedInterpolated(c, 3) as x, toTypeName(x) FROM decimal WHERE a < 0; +SELECT medianExactWeightedInterpolated(a, 1), + medianExactWeightedInterpolated(b, 2), + medianExactWeightedInterpolated(c, 3) as x, + medianExactWeightedInterpolated(f, 4), + medianExactWeightedInterpolated(d, 5), + toTypeName(x) FROM decimal; +SELECT quantileExactWeightedInterpolated(a, 1), + quantileExactWeightedInterpolated(b, 2), + quantileExactWeightedInterpolated(c, 3) as x, + quantileExactWeightedInterpolated(f, 4), + quantileExactWeightedInterpolated(d, 5), + toTypeName(x) FROM decimal WHERE a < 0; SELECT quantileExactWeightedInterpolated(0.0)(a, 1), quantileExactWeightedInterpolated(0.0)(b, 2), quantileExactWeightedInterpolated(0.0)(c, 3) FROM decimal WHERE a >= 0; SELECT quantileExactWeightedInterpolated(0.2)(a, 1), quantileExactWeightedInterpolated(0.2)(b, 2), quantileExactWeightedInterpolated(0.2)(c, 3) FROM decimal WHERE a >= 0; SELECT quantileExactWeightedInterpolated(0.4)(a, 1), quantileExactWeightedInterpolated(0.4)(b, 2), quantileExactWeightedInterpolated(0.4)(c, 3) FROM decimal WHERE a >= 0; @@ -24,6 +36,8 @@ SELECT quantileExactWeightedInterpolated(1.0)(a, 1), quantileExactWeightedInterp SELECT quantilesExactWeightedInterpolated(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(a, 1) FROM decimal; SELECT quantilesExactWeightedInterpolated(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(b, 2) FROM decimal; SELECT quantilesExactWeightedInterpolated(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(c, 3) FROM decimal; +SELECT quantilesExactWeightedInterpolated(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(f, 4) FROM decimal; +SELECT quantilesExactWeightedInterpolated(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(d, 5) FROM decimal; SELECT 'quantileExactWeightedInterpolatedState'; SELECT quantilesExactWeightedInterpolatedMerge(0.2, 0.4, 0.6, 0.8)(x) @@ -34,12 +48,29 @@ FROM ); SELECT 'Test with filter that returns no rows'; -SELECT medianExactWeightedInterpolated(a, 1), medianExactWeightedInterpolated(b, 2), medianExactWeightedInterpolated(c, 3) FROM decimal WHERE a > 1000; +SELECT medianExactWeightedInterpolated(a, 1), + medianExactWeightedInterpolated(b, 2), + medianExactWeightedInterpolated(c, 3), + medianExactWeightedInterpolated(f, 4), + medianExactWeightedInterpolated(d, 5) FROM decimal WHERE a > 1000; +SELECT quantileExactWeightedInterpolated(a, 1), + quantileExactWeightedInterpolated(b, 2), + quantileExactWeightedInterpolated(c, 3), + quantileExactWeightedInterpolated(f, 4), + quantileExactWeightedInterpolated(d, 5) FROM decimal WHERE d < toDate('2024-01-01'); SELECT 'Test with dynamic weights'; -SELECT medianExactWeightedInterpolated(a, w), medianExactWeightedInterpolated(b, w), medianExactWeightedInterpolated(c, w) FROM decimal; +SELECT medianExactWeightedInterpolated(a, w), + medianExactWeightedInterpolated(b, w), + medianExactWeightedInterpolated(c, w), + medianExactWeightedInterpolated(f, w), + medianExactWeightedInterpolated(d, w) FROM decimal; SELECT 'Test with all weights set to 0'; -SELECT medianExactWeightedInterpolated(a, 0), medianExactWeightedInterpolated(b, 0), medianExactWeightedInterpolated(c, 0) FROM decimal; +SELECT medianExactWeightedInterpolated(a, 0), + medianExactWeightedInterpolated(b, 0), + medianExactWeightedInterpolated(c, 0), + medianExactWeightedInterpolated(f, 0), + medianExactWeightedInterpolated(d, 0) FROM decimal; DROP TABLE IF EXISTS decimal; diff --git a/tests/queries/0_stateless/03246_alter_from_string_to_json.reference b/tests/queries/0_stateless/03246_alter_from_string_to_json.reference new file mode 100644 index 00000000000..8253c4fef48 --- /dev/null +++ b/tests/queries/0_stateless/03246_alter_from_string_to_json.reference @@ -0,0 +1,134 @@ +All paths: +['key0','key1','key2','key3','key4','key5'] +Shared data paths: +key2 +key3 +key4 +key5 +{"key0":"value0"} +{"key1":"value1"} +{"key0":"value2"} +{"key1":"value3"} +{"key0":"value4"} +{"key1":"value5"} +{"key0":"value6"} +{"key1":"value7"} +{"key0":"value8"} +{"key1":"value9"} +{"key2":"value60000"} +{"key3":"value60001"} +{"key2":"value60002"} +{"key3":"value60003"} +{"key2":"value60004"} +{"key3":"value60005"} +{"key2":"value60006"} +{"key3":"value60007"} +{"key2":"value60008"} +{"key3":"value60009"} +{"key4":"value120000"} +{"key5":"value120001"} +{"key4":"value120002"} +{"key5":"value120003"} +{"key4":"value120004"} +{"key5":"value120005"} +{"key4":"value120006"} +{"key5":"value120007"} +{"key4":"value120008"} +{"key5":"value120009"} +value0 \N \N \N \N \N +\N value1 \N \N \N \N +value2 \N \N \N \N \N +\N value3 \N \N \N \N +value4 \N \N \N \N \N +\N value5 \N \N \N \N +value6 \N \N \N \N \N +\N value7 \N \N \N \N +value8 \N \N \N \N \N +\N value9 \N \N \N \N +\N \N value60000 \N \N \N +\N \N \N value60001 \N \N +\N \N value60002 \N \N \N +\N \N \N value60003 \N \N +\N \N value60004 \N \N \N +\N \N \N value60005 \N \N +\N \N value60006 \N \N \N +\N \N \N value60007 \N \N +\N \N value60008 \N \N \N +\N \N \N value60009 \N \N +\N \N \N \N value120000 \N +\N \N \N \N \N value120001 +\N \N \N \N value120002 \N +\N \N \N \N \N value120003 +\N \N \N \N value120004 \N +\N \N \N \N \N value120005 +\N \N \N \N value120006 \N +\N \N \N \N \N value120007 +\N \N \N \N value120008 \N +\N \N \N \N \N value120009 +All paths: +['key0','key1','key2','key3','key4','key5'] +Shared data paths: +key2 +key3 +key4 +key5 +{"key0":"value0"} +{"key1":"value1"} +{"key0":"value2"} +{"key1":"value3"} +{"key0":"value4"} +{"key1":"value5"} +{"key0":"value6"} +{"key1":"value7"} +{"key0":"value8"} +{"key1":"value9"} +{"key2":"value60000"} +{"key3":"value60001"} +{"key2":"value60002"} +{"key3":"value60003"} +{"key2":"value60004"} +{"key3":"value60005"} +{"key2":"value60006"} +{"key3":"value60007"} +{"key2":"value60008"} +{"key3":"value60009"} +{"key4":"value120000"} +{"key5":"value120001"} +{"key4":"value120002"} +{"key5":"value120003"} +{"key4":"value120004"} +{"key5":"value120005"} +{"key4":"value120006"} +{"key5":"value120007"} +{"key4":"value120008"} +{"key5":"value120009"} +value0 \N \N \N \N \N +\N value1 \N \N \N \N +value2 \N \N \N \N \N +\N value3 \N \N \N \N +value4 \N \N \N \N \N +\N value5 \N \N \N \N +value6 \N \N \N \N \N +\N value7 \N \N \N \N +value8 \N \N \N \N \N +\N value9 \N \N \N \N +\N \N value60000 \N \N \N +\N \N \N value60001 \N \N +\N \N value60002 \N \N \N +\N \N \N value60003 \N \N +\N \N value60004 \N \N \N +\N \N \N value60005 \N \N +\N \N value60006 \N \N \N +\N \N \N value60007 \N \N +\N \N value60008 \N \N \N +\N \N \N value60009 \N \N +\N \N \N \N value120000 \N +\N \N \N \N \N value120001 +\N \N \N \N value120002 \N +\N \N \N \N \N value120003 +\N \N \N \N value120004 \N +\N \N \N \N \N value120005 +\N \N \N \N value120006 \N +\N \N \N \N \N value120007 +\N \N \N \N value120008 \N +\N \N \N \N \N value120009 diff --git a/tests/queries/0_stateless/03246_alter_from_string_to_json.sql.j2 b/tests/queries/0_stateless/03246_alter_from_string_to_json.sql.j2 new file mode 100644 index 00000000000..2ccf2153699 --- /dev/null +++ b/tests/queries/0_stateless/03246_alter_from_string_to_json.sql.j2 @@ -0,0 +1,36 @@ +-- Random settings limits: index_granularity=(None, 60000) +-- Tags: long + +set allow_experimental_json_type = 1; +set max_block_size = 20000; + +drop table if exists test; + +{% for create_command in ['create table test (x UInt64, json String) engine=MergeTree order by x settings min_rows_for_wide_part=100000000, min_bytes_for_wide_part=1000000000;', + 'create table test (x UInt64, json String) engine=MergeTree order by x settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;'] -%} + +{{ create_command }} + +insert into test select number, toJSONString(map('key' || multiIf(number < 60000, number % 2, number < 120000, number % 2 + 2, number % 2 + 4), 'value' || number)) from numbers(200000); + +alter table test modify column json JSON settings mutations_sync=1; + +select 'All paths:'; +select distinctJSONPaths(json) from test; +select 'Shared data paths:'; +select distinct (arrayJoin(JSONSharedDataPaths(json))) as path from test order by path; +select json from test order by x limit 10; +select json from test order by x limit 10 offset 60000; +select json from test order by x limit 10 offset 120000; +select json.key0, json.key1, json.key2, json.key3, json.key4, json.key5 from test order by x limit 10; +select json.key0, json.key1, json.key2, json.key3, json.key4, json.key5 from test order by x limit 10 offset 60000; +select json.key0, json.key1, json.key2, json.key3, json.key4, json.key5 from test order by x limit 10 offset 120000; + +select json from test format Null; +select json from test order by x format Null; +select json.key0, json.key1, json.key2, json.key3, json.key4, json.key5 from test format Null; +select json.key0, json.key1, json.key2, json.key3, json.key4, json.key5 from test order by x format Null; + +drop table test; + +{% endfor -%} diff --git a/tests/queries/0_stateless/03247_ghdata_string_to_json_alter.reference b/tests/queries/0_stateless/03247_ghdata_string_to_json_alter.reference new file mode 100644 index 00000000000..ca2fb7e8ff9 --- /dev/null +++ b/tests/queries/0_stateless/03247_ghdata_string_to_json_alter.reference @@ -0,0 +1,12 @@ +5000 +leonardomso/33-js-concepts 3 +ytdl-org/youtube-dl 3 +Bogdanp/neko 2 +bminossi/AllVideoPocsFromHackerOne 2 +disclose/diodata 2 +Commit 182 +chipeo345 119 +phanwi346 114 +Nicholas Piggin 95 +direwolf-github 49 +2 diff --git a/tests/queries/0_stateless/03247_ghdata_string_to_json_alter.sh b/tests/queries/0_stateless/03247_ghdata_string_to_json_alter.sh new file mode 100755 index 00000000000..e8368b6702a --- /dev/null +++ b/tests/queries/0_stateless/03247_ghdata_string_to_json_alter.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-s3-storage, long +# ^ no-s3-storage: too memory hungry + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata (data String) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'" + +cat $CUR_DIR/data_json/ghdata_sample.json | ${CLICKHOUSE_CLIENT} \ + --max_memory_usage 10G --query "INSERT INTO ghdata FORMAT JSONAsString" + +${CLICKHOUSE_CLIENT} -q "ALTER TABLE ghdata MODIFY column data JSON SETTINGS mutations_sync=1" --allow_experimental_json_type 1 + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM ghdata WHERE NOT ignore(*)" + +${CLICKHOUSE_CLIENT} -q \ +"SELECT data.repo.name, count() AS stars FROM ghdata \ + WHERE data.type = 'WatchEvent' GROUP BY data.repo.name ORDER BY stars DESC, data.repo.name LIMIT 5" --allow_suspicious_types_in_group_by=1 --allow_suspicious_types_in_order_by=1 + +${CLICKHOUSE_CLIENT} --enable_analyzer=1 -q \ +"SELECT data.payload.commits[].author.name AS name, count() AS c FROM ghdata \ + ARRAY JOIN data.payload.commits[].author.name \ + GROUP BY name ORDER BY c DESC, name LIMIT 5" --allow_suspicious_types_in_group_by=1 --allow_suspicious_types_in_order_by=1 + +${CLICKHOUSE_CLIENT} -q "SELECT max(data.payload.pull_request.assignees[].size0) FROM ghdata" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata" diff --git a/tests/queries/0_stateless/03254_parallel_replicas_join_with_totals.reference b/tests/queries/0_stateless/03254_parallel_replicas_join_with_totals.reference new file mode 100644 index 00000000000..f87bb786c46 --- /dev/null +++ b/tests/queries/0_stateless/03254_parallel_replicas_join_with_totals.reference @@ -0,0 +1,10 @@ +1 1 +1 1 + +0 0 +----- +1 1 +1 1 + +0 0 +----- diff --git a/tests/queries/0_stateless/03254_parallel_replicas_join_with_totals.sh b/tests/queries/0_stateless/03254_parallel_replicas_join_with_totals.sh new file mode 100755 index 00000000000..365d7abed7a --- /dev/null +++ b/tests/queries/0_stateless/03254_parallel_replicas_join_with_totals.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +${CLICKHOUSE_CLIENT} --query=" +CREATE TABLE t +( + item_id UInt64, + price_sold Float32, + date Date +) +ENGINE = MergeTree +ORDER BY item_id; + +INSERT INTO t VALUES (1, 100, '1970-01-01'), (1, 200, '1970-01-02'); +" + +for enable_parallel_replicas in {0..1}; do + ${CLICKHOUSE_CLIENT} --query=" + --- Old analyzer uses different code path and it produces wrong result in this case. + set enable_analyzer=1; + set allow_experimental_parallel_reading_from_replicas=${enable_parallel_replicas}, cluster_for_parallel_replicas='parallel_replicas', max_parallel_replicas=100, parallel_replicas_for_non_replicated_merge_tree=1; + + SELECT * + FROM + ( + SELECT item_id + FROM t + ) AS l + LEFT JOIN + ( + SELECT item_id + FROM t + GROUP BY item_id + WITH TOTALS + ORDER BY item_id ASC + ) AS r ON l.item_id = r.item_id; + + SELECT '-----'; + " +done + +${CLICKHOUSE_CLIENT} --query=" +DROP TABLE t; +" diff --git a/tests/queries/0_stateless/03254_system_prewarm_mark_cache.reference b/tests/queries/0_stateless/03254_system_prewarm_mark_cache.reference new file mode 100644 index 00000000000..86674e7765a --- /dev/null +++ b/tests/queries/0_stateless/03254_system_prewarm_mark_cache.reference @@ -0,0 +1,4 @@ +20000 +20000 +1 +0 diff --git a/tests/queries/0_stateless/03254_system_prewarm_mark_cache.sql b/tests/queries/0_stateless/03254_system_prewarm_mark_cache.sql new file mode 100644 index 00000000000..f9e77365836 --- /dev/null +++ b/tests/queries/0_stateless/03254_system_prewarm_mark_cache.sql @@ -0,0 +1,27 @@ +-- Tags: no-parallel, no-shared-merge-tree + +DROP TABLE IF EXISTS t_prewarm_cache; + +CREATE TABLE t_prewarm_cache (a UInt64, b UInt64, c UInt64) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/03254_prewarm_mark_cache_smt/t_prewarm_cache', '1') +ORDER BY a SETTINGS prewarm_mark_cache = 0; + +SYSTEM DROP MARK CACHE; + +INSERT INTO t_prewarm_cache SELECT number, rand(), rand() FROM numbers(20000); + +SELECT count() FROM t_prewarm_cache WHERE NOT ignore(*); + +SYSTEM DROP MARK CACHE; + +SYSTEM PREWARM MARK CACHE t_prewarm_cache; + +SELECT count() FROM t_prewarm_cache WHERE NOT ignore(*); + +SYSTEM FLUSH LOGS; + +SELECT ProfileEvents['LoadedMarksCount'] > 0 FROM system.query_log +WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND query LIKE 'SELECT count() FROM t_prewarm_cache%' +ORDER BY event_time_microseconds; + +DROP TABLE IF EXISTS t_prewarm_cache; diff --git a/tests/queries/0_stateless/03255_parallel_replicas_join_algo_and_analyzer_4.reference b/tests/queries/0_stateless/03255_parallel_replicas_join_algo_and_analyzer_4.reference new file mode 100644 index 00000000000..d846b26b72b --- /dev/null +++ b/tests/queries/0_stateless/03255_parallel_replicas_join_algo_and_analyzer_4.reference @@ -0,0 +1,116 @@ +4999950000 +4999950000 +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t` AS `__table1` GROUP BY `__table1`.`item_id` +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t1` AS `__table1` +4999950000 +4999950000 +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t` AS `__table1` +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t1` AS `__table1` GROUP BY `__table1`.`item_id` +499950000 +499960000 +499970000 +499980000 +499990000 +500000000 +500010000 +500020000 +500030000 +500040000 +499950000 +499960000 +499970000 +499980000 +499990000 +500000000 +500010000 +500020000 +500030000 +500040000 +SELECT sum(`__table1`.`item_id`) AS `sum(item_id)` FROM (SELECT `__table2`.`item_id` AS `item_id`, `__table2`.`price_sold` AS `price_sold` FROM `default`.`t` AS `__table2`) AS `__table1` ALL LEFT JOIN (SELECT `__table4`.`item_id` AS `item_id` FROM `default`.`t1` AS `__table4`) AS `__table3` ON `__table1`.`item_id` = `__table3`.`item_id` GROUP BY `__table1`.`price_sold` ORDER BY `__table1`.`price_sold` ASC +4999950000 +4999950000 +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t` AS `__table1` GROUP BY `__table1`.`item_id` +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t1` AS `__table1` +4999950000 +4999950000 +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t` AS `__table1` +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t1` AS `__table1` GROUP BY `__table1`.`item_id` +499950000 +499960000 +499970000 +499980000 +499990000 +500000000 +500010000 +500020000 +500030000 +500040000 +499950000 +499960000 +499970000 +499980000 +499990000 +500000000 +500010000 +500020000 +500030000 +500040000 +SELECT sum(`__table1`.`item_id`) AS `sum(item_id)` FROM (SELECT `__table2`.`item_id` AS `item_id`, `__table2`.`price_sold` AS `price_sold` FROM `default`.`t` AS `__table2`) AS `__table1` ALL LEFT JOIN (SELECT `__table4`.`item_id` AS `item_id` FROM `default`.`t1` AS `__table4`) AS `__table3` ON `__table1`.`item_id` = `__table3`.`item_id` GROUP BY `__table1`.`price_sold` ORDER BY `__table1`.`price_sold` ASC +4999950000 +4999950000 +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t` AS `__table1` GROUP BY `__table1`.`item_id` +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t1` AS `__table1` +4999950000 +4999950000 +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t` AS `__table1` +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t1` AS `__table1` GROUP BY `__table1`.`item_id` +499950000 +499960000 +499970000 +499980000 +499990000 +500000000 +500010000 +500020000 +500030000 +500040000 +499950000 +499960000 +499970000 +499980000 +499990000 +500000000 +500010000 +500020000 +500030000 +500040000 +SELECT sum(`__table1`.`item_id`) AS `sum(item_id)` FROM (SELECT `__table2`.`item_id` AS `item_id`, `__table2`.`price_sold` AS `price_sold` FROM `default`.`t` AS `__table2`) AS `__table1` GLOBAL ALL LEFT JOIN `_data_x_y_` AS `__table3` ON `__table1`.`item_id` = `__table3`.`item_id` GROUP BY `__table1`.`price_sold` ORDER BY `__table1`.`price_sold` ASC +4999950000 +4999950000 +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t` AS `__table1` GROUP BY `__table1`.`item_id` +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t1` AS `__table1` +4999950000 +4999950000 +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t` AS `__table1` +SELECT `__table1`.`item_id` AS `item_id` FROM `default`.`t1` AS `__table1` GROUP BY `__table1`.`item_id` +499950000 +499960000 +499970000 +499980000 +499990000 +500000000 +500010000 +500020000 +500030000 +500040000 +499950000 +499960000 +499970000 +499980000 +499990000 +500000000 +500010000 +500020000 +500030000 +500040000 +SELECT sum(`__table1`.`item_id`) AS `sum(item_id)` FROM (SELECT `__table2`.`item_id` AS `item_id`, `__table2`.`price_sold` AS `price_sold` FROM `default`.`t` AS `__table2`) AS `__table1` GLOBAL ALL LEFT JOIN `_data_x_y_` AS `__table3` ON `__table1`.`item_id` = `__table3`.`item_id` GROUP BY `__table1`.`price_sold` ORDER BY `__table1`.`price_sold` ASC diff --git a/tests/queries/0_stateless/03255_parallel_replicas_join_algo_and_analyzer_4.sh b/tests/queries/0_stateless/03255_parallel_replicas_join_algo_and_analyzer_4.sh new file mode 100755 index 00000000000..19866f26949 --- /dev/null +++ b/tests/queries/0_stateless/03255_parallel_replicas_join_algo_and_analyzer_4.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Tags: long, no-random-settings, no-random-merge-tree-settings + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +${CLICKHOUSE_CLIENT} --query=" +CREATE TABLE t +( + item_id UInt64, + price_sold Float32, + date Date +) +ENGINE = MergeTree +ORDER BY item_id; + +CREATE TABLE t1 +( + item_id UInt64, + price_sold Float32, + date Date +) +ENGINE = MergeTree +ORDER BY item_id; + +INSERT INTO t SELECT number, number % 10, toDate(number) FROM numbers(100000); +INSERT INTO t1 SELECT number, number % 10, toDate(number) FROM numbers(100000); +" + +query1=" + SELECT sum(item_id) + FROM + ( + SELECT item_id + FROM t + GROUP BY item_id + ) AS l + LEFT JOIN + ( + SELECT item_id + FROM t1 + ) AS r ON l.item_id = r.item_id +" + +query2=" + SELECT sum(item_id) + FROM + ( + SELECT item_id + FROM t + ) AS l + LEFT JOIN + ( + SELECT item_id + FROM t1 + GROUP BY item_id + ) AS r ON l.item_id = r.item_id +" + +query3=" + SELECT sum(item_id) + FROM + ( + SELECT item_id, price_sold + FROM t + ) AS l + LEFT JOIN + ( + SELECT item_id + FROM t1 + ) AS r ON l.item_id = r.item_id + GROUP BY price_sold + ORDER BY price_sold +" + +for parallel_replicas_prefer_local_join in 1 0; do + for prefer_local_plan in {0..1}; do + for query in "${query1}" "${query2}" "${query3}"; do + for enable_parallel_replicas in {0..1}; do + ${CLICKHOUSE_CLIENT} --query=" + set enable_analyzer=1; + set parallel_replicas_prefer_local_join=${parallel_replicas_prefer_local_join}; + set parallel_replicas_local_plan=${prefer_local_plan}; + set allow_experimental_parallel_reading_from_replicas=${enable_parallel_replicas}, cluster_for_parallel_replicas='parallel_replicas', max_parallel_replicas=100, parallel_replicas_for_non_replicated_merge_tree=1; + + --SELECT '----- enable_parallel_replicas=$enable_parallel_replicas prefer_local_plan=$prefer_local_plan parallel_replicas_prefer_local_join=$parallel_replicas_prefer_local_join -----'; + ${query}; + + SELECT replaceRegexpAll(replaceRegexpAll(explain, '.*Query: (.*) Replicas:.*', '\\1'), '(.*)_data_[\d]+_[\d]+(.*)', '\1_data_x_y_\2') + FROM + ( + EXPLAIN actions=1 ${query} + ) + WHERE explain LIKE '%ParallelReplicas%'; + " + done + done + done +done diff --git a/tests/queries/0_stateless/03258_multiple_array_joins.reference b/tests/queries/0_stateless/03258_multiple_array_joins.reference new file mode 100644 index 00000000000..4d357c8ac80 --- /dev/null +++ b/tests/queries/0_stateless/03258_multiple_array_joins.reference @@ -0,0 +1,8 @@ +1 Michel Foucault alive no +1 Michel Foucault profession philosopher +1 Thomas Aquinas alive no +1 Thomas Aquinas profession philosopher +2 Nicola Tesla alive no +2 Nicola Tesla profession inventor +2 Thomas Edison alive no +2 Thomas Edison profession inventor diff --git a/tests/queries/0_stateless/03258_multiple_array_joins.sql b/tests/queries/0_stateless/03258_multiple_array_joins.sql new file mode 100644 index 00000000000..ddfac1da080 --- /dev/null +++ b/tests/queries/0_stateless/03258_multiple_array_joins.sql @@ -0,0 +1,25 @@ +SET enable_analyzer = 1; +DROP TABLE IF EXISTS test_multiple_array_join; + +CREATE TABLE test_multiple_array_join ( + id UInt64, + person Nested ( + name String, + surname String + ), + properties Nested ( + key String, + value String + ) +) Engine=MergeTree ORDER BY id; + +INSERT INTO test_multiple_array_join VALUES (1, ['Thomas', 'Michel'], ['Aquinas', 'Foucault'], ['profession', 'alive'], ['philosopher', 'no']); +INSERT INTO test_multiple_array_join VALUES (2, ['Thomas', 'Nicola'], ['Edison', 'Tesla'], ['profession', 'alive'], ['inventor', 'no']); + +SELECT * +FROM test_multiple_array_join +ARRAY JOIN person +ARRAY JOIN properties +ORDER BY ALL; + +DROP TABLE test_multiple_array_join; diff --git a/tests/queries/0_stateless/03258_old_analyzer_const_expr_bug.reference b/tests/queries/0_stateless/03258_old_analyzer_const_expr_bug.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03258_old_analyzer_const_expr_bug.sql b/tests/queries/0_stateless/03258_old_analyzer_const_expr_bug.sql new file mode 100644 index 00000000000..913de3b849c --- /dev/null +++ b/tests/queries/0_stateless/03258_old_analyzer_const_expr_bug.sql @@ -0,0 +1,23 @@ +WITH + multiIf('-1' = '-1', 10080, '-1' = '7', 60, '-1' = '1', 5, 1440) AS interval_start, -- noqa + multiIf('-1' = '-1', CEIL((today() - toDate('2017-06-22')) / 7)::UInt16, '-1' = '7', 168, '-1' = '1', 288, 90) AS days_run, -- noqa:L045 + block_time as (SELECT arrayJoin( + arrayMap( + i -> toDateTime(toStartOfInterval(now(), INTERVAL interval_start MINUTE) - interval_start * 60 * i, 'UTC'), + range(days_run) + ) + )), + +sales AS ( + SELECT + toDateTime(toStartOfInterval(now(), INTERVAL interval_start MINUTE), 'UTC') AS block_time + FROM + numbers(1) + GROUP BY + block_time + ORDER BY + block_time) + +SELECT + block_time +FROM sales where block_time >= (SELECT MIN(block_time) FROM sales) format Null; diff --git a/tests/queries/0_stateless/03258_quantile_exact_weighted_issue.reference b/tests/queries/0_stateless/03258_quantile_exact_weighted_issue.reference new file mode 100644 index 00000000000..69afec5d545 --- /dev/null +++ b/tests/queries/0_stateless/03258_quantile_exact_weighted_issue.reference @@ -0,0 +1,2 @@ +AggregateFunction(quantilesExactWeighted(0.2, 0.4, 0.6, 0.8), UInt64, UInt8) +AggregateFunction(quantilesExactWeightedInterpolated(0.2, 0.4, 0.6, 0.8), UInt64, UInt8) diff --git a/tests/queries/0_stateless/03258_quantile_exact_weighted_issue.sql b/tests/queries/0_stateless/03258_quantile_exact_weighted_issue.sql new file mode 100644 index 00000000000..3069389f4e2 --- /dev/null +++ b/tests/queries/0_stateless/03258_quantile_exact_weighted_issue.sql @@ -0,0 +1,2 @@ +SELECT toTypeName(quantilesExactWeightedState(0.2, 0.4, 0.6, 0.8)(number + 1, 1) AS x) FROM numbers(49999); +SELECT toTypeName(quantilesExactWeightedInterpolatedState(0.2, 0.4, 0.6, 0.8)(number + 1, 1) AS x) FROM numbers(49999); diff --git a/tests/queries/0_stateless/03259_orc_date_out_of_range.reference b/tests/queries/0_stateless/03259_orc_date_out_of_range.reference new file mode 100644 index 00000000000..ddac785369f --- /dev/null +++ b/tests/queries/0_stateless/03259_orc_date_out_of_range.reference @@ -0,0 +1,12 @@ +number Nullable(Int64) +date_field Nullable(Date32) +\N +1970-01-02 +\N +1970-01-04 +\N +1970-01-06 +\N +1970-01-08 +\N +1970-01-10 diff --git a/tests/queries/0_stateless/03259_orc_date_out_of_range.sql b/tests/queries/0_stateless/03259_orc_date_out_of_range.sql new file mode 100644 index 00000000000..e73d2faa5dd --- /dev/null +++ b/tests/queries/0_stateless/03259_orc_date_out_of_range.sql @@ -0,0 +1,14 @@ +-- Tags: no-fasttest, no-parallel + +SET session_timezone = 'UTC'; +SET engine_file_truncate_on_insert = 1; + +insert into function file('03259.orc', 'ORC') +select + number, + if (number % 2 = 0, null, toDate32(number)) as date_field +from numbers(10); + +desc file('03259.orc', 'ORC'); + +select date_field from file('03259.orc', 'ORC') order by number; diff --git a/tests/queries/0_stateless/03261_delayed_streams_memory.reference b/tests/queries/0_stateless/03261_delayed_streams_memory.reference new file mode 100644 index 00000000000..7326d960397 --- /dev/null +++ b/tests/queries/0_stateless/03261_delayed_streams_memory.reference @@ -0,0 +1 @@ +Ok diff --git a/tests/queries/0_stateless/03261_delayed_streams_memory.sql b/tests/queries/0_stateless/03261_delayed_streams_memory.sql new file mode 100644 index 00000000000..863644a0dff --- /dev/null +++ b/tests/queries/0_stateless/03261_delayed_streams_memory.sql @@ -0,0 +1,20 @@ +-- Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-random-settings, no-random-merge-tree-settings + +DROP TABLE IF EXISTS t_100_columns; + +CREATE TABLE t_100_columns (id UInt64, c0 String, c1 String, c2 String, c3 String, c4 String, c5 String, c6 String, c7 String, c8 String, c9 String, c10 String, c11 String, c12 String, c13 String, c14 String, c15 String, c16 String, c17 String, c18 String, c19 String, c20 String, c21 String, c22 String, c23 String, c24 String, c25 String, c26 String, c27 String, c28 String, c29 String, c30 String, c31 String, c32 String, c33 String, c34 String, c35 String, c36 String, c37 String, c38 String, c39 String, c40 String, c41 String, c42 String, c43 String, c44 String, c45 String, c46 String, c47 String, c48 String, c49 String, c50 String) +ENGINE = MergeTree +ORDER BY id PARTITION BY id % 50 +SETTINGS min_bytes_for_wide_part = 0, ratio_of_defaults_for_sparse_serialization = 1.0, max_compress_block_size = '1M', storage_policy = 's3_cache'; + +SET max_insert_delayed_streams_for_parallel_write = 55; + +INSERT INTO t_100_columns (id) SELECT number FROM numbers(100); + +SYSTEM FLUSH LOGS; + +SELECT if (memory_usage < 300000000, 'Ok', format('Fail: memory usage {}', formatReadableSize(memory_usage))) +FROM system.query_log +WHERE current_database = currentDatabase() AND query LIKE 'INSERT INTO t_100_columns%' AND type = 'QueryFinish'; + +DROP TABLE t_100_columns; diff --git a/tests/queries/0_stateless/03261_json_hints_types_check.reference b/tests/queries/0_stateless/03261_json_hints_types_check.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03261_json_hints_types_check.sql b/tests/queries/0_stateless/03261_json_hints_types_check.sql new file mode 100644 index 00000000000..a407aa9474b --- /dev/null +++ b/tests/queries/0_stateless/03261_json_hints_types_check.sql @@ -0,0 +1,9 @@ +set allow_experimental_json_type=1; +set allow_experimental_variant_type=0; +set allow_experimental_object_type=0; + +select '{}'::JSON(a LowCardinality(Int128)); -- {serverError SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY} +select '{}'::JSON(a FixedString(100000)); -- {serverError ILLEGAL_COLUMN} +select '{}'::JSON(a Variant(Int32)); -- {serverError ILLEGAL_COLUMN} +select '{}'::JSON(a Object('json')); -- {serverError ILLEGAL_COLUMN} + diff --git a/tests/queries/0_stateless/03261_low_cardinality_nullable_to_dynamic_cast.reference b/tests/queries/0_stateless/03261_low_cardinality_nullable_to_dynamic_cast.reference new file mode 100644 index 00000000000..96e34d5a44c --- /dev/null +++ b/tests/queries/0_stateless/03261_low_cardinality_nullable_to_dynamic_cast.reference @@ -0,0 +1,2 @@ +\N +\N diff --git a/tests/queries/0_stateless/03261_low_cardinality_nullable_to_dynamic_cast.sql b/tests/queries/0_stateless/03261_low_cardinality_nullable_to_dynamic_cast.sql new file mode 100644 index 00000000000..fdb497a62bf --- /dev/null +++ b/tests/queries/0_stateless/03261_low_cardinality_nullable_to_dynamic_cast.sql @@ -0,0 +1,7 @@ +SET allow_suspicious_low_cardinality_types = 1, allow_experimental_dynamic_type = 1; +DROP TABLE IF EXISTS t0; +CREATE TABLE t0 (c0 LowCardinality(Nullable(Int))) ENGINE = Memory(); +INSERT INTO TABLE t0 (c0) VALUES (NULL); +SELECT c0::Dynamic FROM t0; +SELECT c0 FROM t0; +DROP TABLE t0; diff --git a/tests/queries/0_stateless/03261_mongodb_argumetns_crash.reference b/tests/queries/0_stateless/03261_mongodb_argumetns_crash.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03261_mongodb_argumetns_crash.sql b/tests/queries/0_stateless/03261_mongodb_argumetns_crash.sql new file mode 100644 index 00000000000..ca558ac6bc6 --- /dev/null +++ b/tests/queries/0_stateless/03261_mongodb_argumetns_crash.sql @@ -0,0 +1,14 @@ +-- Tags: no-fasttest + +SELECT * FROM mongodb('mongodb://some-cluster:27017/?retryWrites=false', NULL, 'my_collection', 'test_user', 'password', 'x Int32'); -- { serverError BAD_ARGUMENTS } +SELECT * FROM mongodb('mongodb://some-cluster:27017/?retryWrites=false', 'test', NULL, 'test_user', 'password', 'x Int32'); -- { serverError BAD_ARGUMENTS } +SELECT * FROM mongodb('mongodb://some-cluster:27017/?retryWrites=false', 'test', 'my_collection', NULL, 'password', 'x Int32'); -- { serverError BAD_ARGUMENTS } +SELECT * FROM mongodb('mongodb://some-cluster:27017/?retryWrites=false', 'test', 'my_collection', 'test_user', NULL, 'x Int32'); -- { serverError BAD_ARGUMENTS } +SELECT * FROM mongodb('mongodb://some-cluster:27017/?retryWrites=false', 'test', 'my_collection', 'test_user', 'password', NULL); -- { serverError BAD_ARGUMENTS } +SELECT * FROM mongodb('mongodb://some-cluster:27017/?retryWrites=false', 'test', 'my_collection', 'test_user', 'password', materialize(1) + 1); -- { serverError BAD_ARGUMENTS } +SELECT * FROM mongodb('mongodb://some-cluster:27017/?retryWrites=false', 'test', 'my_collection', 'test_user', 'password', 'x Int32', NULL); -- { serverError BAD_ARGUMENTS } +SELECT * FROM mongodb('mongodb://some-cluster:27017/?retryWrites=false', 'test', 'my_collection', 'test_user', 'password', NULL, 'x Int32'); -- { serverError BAD_ARGUMENTS } +SELECT * FROM mongodb('mongodb://some-cluster:27017/?retryWrites=false', 'test', 'my_collection', 'test_user', 'password', NULL, 'x Int32'); -- { serverError BAD_ARGUMENTS } +SELECT * FROM mongodb(NULL, 'test', 'my_collection', 'test_user', 'password', 'x Int32'); -- { serverError BAD_ARGUMENTS } + +CREATE TABLE IF NOT EXISTS store_version ( `_id` String ) ENGINE = MongoDB(`localhost:27017`, mongodb, storeinfo, adminUser, adminUser); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } diff --git a/tests/queries/0_stateless/03261_optimize_rewrite_array_exists_to_has_crash.reference b/tests/queries/0_stateless/03261_optimize_rewrite_array_exists_to_has_crash.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03261_optimize_rewrite_array_exists_to_has_crash.sql b/tests/queries/0_stateless/03261_optimize_rewrite_array_exists_to_has_crash.sql new file mode 100644 index 00000000000..e0018632be4 --- /dev/null +++ b/tests/queries/0_stateless/03261_optimize_rewrite_array_exists_to_has_crash.sql @@ -0,0 +1,10 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/71382 +DROP TABLE IF EXISTS rewrite; +CREATE TABLE rewrite (c0 Int) ENGINE = Memory(); +SELECT 1 +FROM rewrite +INNER JOIN rewrite AS y ON ( + SELECT 1 +) +INNER JOIN rewrite AS z ON 1 +SETTINGS optimize_rewrite_array_exists_to_has=1; diff --git a/tests/queries/0_stateless/03261_sort_cursor_crash.reference b/tests/queries/0_stateless/03261_sort_cursor_crash.reference new file mode 100644 index 00000000000..7299f2f5a5f --- /dev/null +++ b/tests/queries/0_stateless/03261_sort_cursor_crash.reference @@ -0,0 +1,4 @@ +42 +43 +44 +45 diff --git a/tests/queries/0_stateless/03261_sort_cursor_crash.sql b/tests/queries/0_stateless/03261_sort_cursor_crash.sql new file mode 100644 index 00000000000..b659f3d4a92 --- /dev/null +++ b/tests/queries/0_stateless/03261_sort_cursor_crash.sql @@ -0,0 +1,24 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/70779 +-- Crash in SortCursorImpl with the old analyzer, which produces a block with 0 columns and 1 row +DROP TABLE IF EXISTS t0; +DROP TABLE IF EXISTS t1; + +CREATE TABLE t0 (c0 Int) ENGINE = AggregatingMergeTree() ORDER BY tuple(); +INSERT INTO TABLE t0 (c0) VALUES (1); +SELECT 42 FROM t0 FINAL PREWHERE t0.c0 = 1; +DROP TABLE t0; + +CREATE TABLE t0 (c0 Int) ENGINE = SummingMergeTree() ORDER BY tuple(); +INSERT INTO TABLE t0 (c0) VALUES (1); +SELECT 43 FROM t0 FINAL PREWHERE t0.c0 = 1; +DROP TABLE t0; + +CREATE TABLE t0 (c0 Int) ENGINE = ReplacingMergeTree() ORDER BY tuple(); +INSERT INTO TABLE t0 (c0) VALUES (1); +SELECT 44 FROM t0 FINAL PREWHERE t0.c0 = 1; +DROP TABLE t0; + +CREATE TABLE t1 (a0 UInt8, c0 Int32, c1 UInt8) ENGINE = AggregatingMergeTree() ORDER BY tuple(); +INSERT INTO TABLE t1 (a0, c0, c1) VALUES (1, 1, 1); +SELECT 45 FROM t1 FINAL PREWHERE t1.c0 = t1.c1; +DROP TABLE t1; diff --git a/tests/queries/0_stateless/03261_tuple_map_object_to_json_cast.reference b/tests/queries/0_stateless/03261_tuple_map_object_to_json_cast.reference new file mode 100644 index 00000000000..0ae94e68663 --- /dev/null +++ b/tests/queries/0_stateless/03261_tuple_map_object_to_json_cast.reference @@ -0,0 +1,23 @@ +Map to JSON +{"a":"0","b":"1970-01-01","c":[],"d":[{"e":"0"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"1","b":"1970-01-02","c":["0"],"d":[{"e":"1"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"2","b":"1970-01-03","c":["0","1"],"d":[{"e":"2"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"3","b":"1970-01-04","c":["0","1","2"],"d":[{"e":"3"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"4","b":"1970-01-05","c":["0","1","2","3"],"d":[{"e":"4"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a0":"0","b0":"1970-01-01","c0":[],"d0":[{"e0":"0"}]} {'a0':'Int64','b0':'Date','c0':'Array(Nullable(String))','d0':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a1":"1","b1":"1970-01-02","c1":["0"],"d1":[{"e1":"1"}]} {'a1':'Int64','b1':'Date','c1':'Array(Nullable(String))','d1':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a2":"2","b2":"1970-01-03","c2":["0","1"],"d2":[{"e2":"2"}]} {'a2':'Int64','b2':'Date','c2':'Array(Nullable(String))','d2':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a0":"3","b0":"1970-01-04","c0":["0","1","2"],"d0":[{"e0":"3"}]} {'a0':'Int64','b0':'Date','c0':'Array(Nullable(String))','d0':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a1":"4","b1":"1970-01-05","c1":["0","1","2","3"],"d1":[{"e1":"4"}]} {'a1':'Int64','b1':'Date','c1':'Array(Nullable(String))','d1':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +Tuple to JSON +{"a":"0","b":"1970-01-01","c":[],"d":[{"e":"0"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"1","b":"1970-01-02","c":["0"],"d":[{"e":"1"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"2","b":"1970-01-03","c":["0","1"],"d":[{"e":"2"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"3","b":"1970-01-04","c":["0","1","2"],"d":[{"e":"3"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"4","b":"1970-01-05","c":["0","1","2","3"],"d":[{"e":"4"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +Object to JSON +{"a":"0","b":"1970-01-01","c":[],"d":{"e":["0"]}} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d.e':'Array(Nullable(Int64))'} +{"a":"1","b":"1970-01-02","c":["0"],"d":{"e":["1"]}} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d.e':'Array(Nullable(Int64))'} +{"a":"2","b":"1970-01-03","c":["0","1"],"d":{"e":["2"]}} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d.e':'Array(Nullable(Int64))'} +{"a":"3","b":"1970-01-04","c":["0","1","2"],"d":{"e":["3"]}} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d.e':'Array(Nullable(Int64))'} +{"a":"4","b":"1970-01-05","c":["0","1","2","3"],"d":{"e":["4"]}} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d.e':'Array(Nullable(Int64))'} diff --git a/tests/queries/0_stateless/03261_tuple_map_object_to_json_cast.sql b/tests/queries/0_stateless/03261_tuple_map_object_to_json_cast.sql new file mode 100644 index 00000000000..2e5cecaf502 --- /dev/null +++ b/tests/queries/0_stateless/03261_tuple_map_object_to_json_cast.sql @@ -0,0 +1,18 @@ +-- Tags: no-fasttest + +set allow_experimental_json_type = 1; +set allow_experimental_object_type = 1; +set allow_experimental_variant_type = 1; +set use_variant_as_common_type = 1; +set enable_named_columns_in_function_tuple = 1; +set enable_analyzer = 1; + +select 'Map to JSON'; +select map('a', number::UInt32, 'b', toDate(number), 'c', range(number), 'd', [map('e', number::UInt32)])::JSON as json, JSONAllPathsWithTypes(json) from numbers(5); +select map('a' || number % 3, number::UInt32, 'b' || number % 3, toDate(number), 'c' || number % 3, range(number), 'd' || number % 3, [map('e' || number % 3, number::UInt32)])::JSON as json, JSONAllPathsWithTypes(json) from numbers(5); + +select 'Tuple to JSON'; +select tuple(number::UInt32 as a, toDate(number) as b, range(number) as c, [tuple(number::UInt32 as e)] as d)::JSON as json, JSONAllPathsWithTypes(json) from numbers(5); + +select 'Object to JSON'; +select toJSONString(map('a', number::UInt32, 'b', toDate(number), 'c', range(number), 'd', [map('e', number::UInt32)]))::Object('json')::JSON as json, JSONAllPathsWithTypes(json) from numbers(5); diff --git a/tests/queries/0_stateless/03261_variant_permutation_bug.reference b/tests/queries/0_stateless/03261_variant_permutation_bug.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03261_variant_permutation_bug.sql b/tests/queries/0_stateless/03261_variant_permutation_bug.sql new file mode 100644 index 00000000000..373dd9e19fa --- /dev/null +++ b/tests/queries/0_stateless/03261_variant_permutation_bug.sql @@ -0,0 +1,6 @@ +set allow_experimental_variant_type=1; +create table test (x UInt64, d Variant(UInt64)) engine=Memory; +insert into test select number, null from numbers(200000); +select d from test order by d::String limit 32213 format Null; +drop table test; + diff --git a/tests/queries/0_stateless/03262_analyzer_materialized_view_in_with_cte.reference b/tests/queries/0_stateless/03262_analyzer_materialized_view_in_with_cte.reference new file mode 100644 index 00000000000..5ddf8439af5 --- /dev/null +++ b/tests/queries/0_stateless/03262_analyzer_materialized_view_in_with_cte.reference @@ -0,0 +1 @@ +1 2 \N test diff --git a/tests/queries/0_stateless/03262_analyzer_materialized_view_in_with_cte.sql b/tests/queries/0_stateless/03262_analyzer_materialized_view_in_with_cte.sql new file mode 100644 index 00000000000..4543d336d14 --- /dev/null +++ b/tests/queries/0_stateless/03262_analyzer_materialized_view_in_with_cte.sql @@ -0,0 +1,63 @@ +SET allow_experimental_analyzer = 1; + +DROP TABLE IF EXISTS mv_test; +DROP TABLE IF EXISTS mv_test_target; +DROP VIEW IF EXISTS mv_test_mv; + +CREATE TABLE mv_test +( + `id` UInt64, + `ref_id` UInt64, + `final_id` Nullable(UInt64), + `display` String +) +ENGINE = Log; + +CREATE TABLE mv_test_target +( + `id` UInt64, + `ref_id` UInt64, + `final_id` Nullable(UInt64), + `display` String +) +ENGINE = Log; + +CREATE MATERIALIZED VIEW mv_test_mv TO mv_test_target +( + `id` UInt64, + `ref_id` UInt64, + `final_id` Nullable(UInt64), + `display` String +) +AS WITH + tester AS + ( + SELECT + id, + ref_id, + final_id, + display + FROM mv_test + ), + id_set AS + ( + SELECT + display, + max(id) AS max_id + FROM mv_test + GROUP BY display + ) +SELECT * +FROM tester +WHERE id IN ( + SELECT max_id + FROM id_set +); + +INSERT INTO mv_test ( id, ref_id, display) values ( 1, 2, 'test'); + +SELECT * FROM mv_test_target; + +DROP VIEW mv_test_mv; +DROP TABLE mv_test_target; +DROP TABLE mv_test; diff --git a/tests/queries/0_stateless/03262_filter_push_down_view.reference b/tests/queries/0_stateless/03262_filter_push_down_view.reference new file mode 100644 index 00000000000..275ff18f73b --- /dev/null +++ b/tests/queries/0_stateless/03262_filter_push_down_view.reference @@ -0,0 +1,2 @@ +Condition: and((materialize(auid) in [1, 1]), (_CAST(toDate(ts)) in (-Inf, 1703980800])) +Granules: 1/3 diff --git a/tests/queries/0_stateless/03262_filter_push_down_view.sql b/tests/queries/0_stateless/03262_filter_push_down_view.sql new file mode 100644 index 00000000000..8492d8c8ebd --- /dev/null +++ b/tests/queries/0_stateless/03262_filter_push_down_view.sql @@ -0,0 +1,36 @@ +DROP TABLE IF EXISTS alpha; +DROP TABLE IF EXISTS alpha__day; + +SET session_timezone = 'Etc/UTC'; + +CREATE TABLE alpha +( + `ts` DateTime64(6), + `auid` Int64, +) +ENGINE = MergeTree +ORDER BY (auid, ts) +SETTINGS index_granularity = 1; + +CREATE VIEW alpha__day +( + `ts_date` Date, + `auid` Int64, +) +AS SELECT + ts_date, + auid, +FROM +( + SELECT + toDate(ts) AS ts_date, + auid + FROM alpha +) +WHERE ts_date <= toDateTime('2024-01-01 00:00:00') - INTERVAL 1 DAY; + +INSERT INTO alpha VALUES (toDateTime64('2024-01-01 00:00:00.000', 3) - INTERVAL 3 DAY, 1); +INSERT INTO alpha VALUES (toDateTime64('2024-01-01 00:00:00.000', 3) - INTERVAL 3 DAY, 2); +INSERT INTO alpha VALUES (toDateTime64('2024-01-01 00:00:00.000', 3) - INTERVAL 3 DAY, 3); + +select trimLeft(explain) from (EXPLAIN indexes = 1 SELECT auid FROM alpha__day WHERE auid = 1) where explain like '%Condition:%' or explain like '%Granules:%' settings allow_experimental_analyzer = 1; diff --git a/tests/queries/0_stateless/03262_system_functions_should_not_fill_query_log_functions.reference b/tests/queries/0_stateless/03262_system_functions_should_not_fill_query_log_functions.reference new file mode 100644 index 00000000000..021c06382c8 --- /dev/null +++ b/tests/queries/0_stateless/03262_system_functions_should_not_fill_query_log_functions.reference @@ -0,0 +1 @@ +[] ['equals'] [] diff --git a/tests/queries/0_stateless/03262_system_functions_should_not_fill_query_log_functions.sql b/tests/queries/0_stateless/03262_system_functions_should_not_fill_query_log_functions.sql new file mode 100644 index 00000000000..7e6f384c0a8 --- /dev/null +++ b/tests/queries/0_stateless/03262_system_functions_should_not_fill_query_log_functions.sql @@ -0,0 +1,9 @@ +SELECT * FROM system.functions WHERE name = 'bitShiftLeft' format Null; +SYSTEM FLUSH LOGS; +SELECT used_aggregate_functions, used_functions, used_table_functions +FROM system.query_log +WHERE + event_date >= yesterday() + AND type = 'QueryFinish' + AND current_database = currentDatabase() + AND query LIKE '%bitShiftLeft%'; diff --git a/tests/queries/0_stateless/03262_udf_in_constraint.reference b/tests/queries/0_stateless/03262_udf_in_constraint.reference new file mode 100644 index 00000000000..29d403b85a8 --- /dev/null +++ b/tests/queries/0_stateless/03262_udf_in_constraint.reference @@ -0,0 +1,2 @@ +CREATE TABLE default.t0\n(\n `c0` Int32,\n CONSTRAINT c1 CHECK c0 > 5\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +10 diff --git a/tests/queries/0_stateless/03262_udf_in_constraint.sh b/tests/queries/0_stateless/03262_udf_in_constraint.sh new file mode 100755 index 00000000000..3c36e7caeb4 --- /dev/null +++ b/tests/queries/0_stateless/03262_udf_in_constraint.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q " + CREATE FUNCTION ${CLICKHOUSE_DATABASE}_function AS (x) -> x > 5; + CREATE TABLE t0 (c0 Int, CONSTRAINT c1 CHECK ${CLICKHOUSE_DATABASE}_function(c0)) ENGINE = MergeTree() ORDER BY tuple(); + SHOW CREATE TABLE t0; + INSERT INTO t0(c0) VALUES (10); + INSERT INTO t0(c0) VALUES (3); -- {serverError VIOLATED_CONSTRAINT} + SELECT * FROM t0; + + DROP TABLE t0; + DROP FUNCTION ${CLICKHOUSE_DATABASE}_function; +" diff --git a/tests/queries/0_stateless/03263_analyzer_materialized_view_cte_nested.reference b/tests/queries/0_stateless/03263_analyzer_materialized_view_cte_nested.reference new file mode 100644 index 00000000000..0cfbf08886f --- /dev/null +++ b/tests/queries/0_stateless/03263_analyzer_materialized_view_cte_nested.reference @@ -0,0 +1 @@ +2 diff --git a/tests/queries/0_stateless/03263_analyzer_materialized_view_cte_nested.sql b/tests/queries/0_stateless/03263_analyzer_materialized_view_cte_nested.sql new file mode 100644 index 00000000000..4ea853a7c22 --- /dev/null +++ b/tests/queries/0_stateless/03263_analyzer_materialized_view_cte_nested.sql @@ -0,0 +1,19 @@ +SET allow_experimental_analyzer = 1; + +DROP TABLE IF EXISTS test_table; +DROP VIEW IF EXISTS test_mv; + +CREATE TABLE test_table ENGINE = MergeTree ORDER BY tuple() AS SELECT 1 as col1; + +CREATE MATERIALIZED VIEW test_mv ENGINE = MergeTree ORDER BY tuple() AS +WITH + subquery_on_source AS (SELECT col1 AS aliased FROM test_table), + output AS (SELECT * FROM test_table WHERE col1 IN (SELECT aliased FROM subquery_on_source)) +SELECT * FROM output; + +INSERT INTO test_table VALUES (2); + +SELECT * FROM test_mv; + +DROP VIEW test_mv; +DROP TABLE test_table; diff --git a/tests/queries/0_stateless/03266_with_fill_staleness.reference b/tests/queries/0_stateless/03266_with_fill_staleness.reference new file mode 100644 index 00000000000..25d7b7c3f24 --- /dev/null +++ b/tests/queries/0_stateless/03266_with_fill_staleness.reference @@ -0,0 +1,151 @@ +add samples +regular with fill +2016-06-15 23:00:00 0 original +2016-06-15 23:00:01 0 +2016-06-15 23:00:02 0 +2016-06-15 23:00:03 0 +2016-06-15 23:00:04 0 +2016-06-15 23:00:05 5 original +2016-06-15 23:00:06 5 +2016-06-15 23:00:07 5 +2016-06-15 23:00:08 5 +2016-06-15 23:00:09 5 +2016-06-15 23:00:10 10 original +2016-06-15 23:00:11 10 +2016-06-15 23:00:12 10 +2016-06-15 23:00:13 10 +2016-06-15 23:00:14 10 +2016-06-15 23:00:15 15 original +2016-06-15 23:00:16 15 +2016-06-15 23:00:17 15 +2016-06-15 23:00:18 15 +2016-06-15 23:00:19 15 +2016-06-15 23:00:20 20 original +2016-06-15 23:00:21 20 +2016-06-15 23:00:22 20 +2016-06-15 23:00:23 20 +2016-06-15 23:00:24 20 +2016-06-15 23:00:25 25 original +staleness 1 seconds +2016-06-15 23:00:00 0 original +2016-06-15 23:00:05 5 original +2016-06-15 23:00:10 10 original +2016-06-15 23:00:15 15 original +2016-06-15 23:00:20 20 original +2016-06-15 23:00:25 25 original +staleness 3 seconds +2016-06-15 23:00:00 0 original +2016-06-15 23:00:01 0 +2016-06-15 23:00:02 0 +2016-06-15 23:00:05 5 original +2016-06-15 23:00:06 5 +2016-06-15 23:00:07 5 +2016-06-15 23:00:10 10 original +2016-06-15 23:00:11 10 +2016-06-15 23:00:12 10 +2016-06-15 23:00:15 15 original +2016-06-15 23:00:16 15 +2016-06-15 23:00:17 15 +2016-06-15 23:00:20 20 original +2016-06-15 23:00:21 20 +2016-06-15 23:00:22 20 +2016-06-15 23:00:25 25 original +2016-06-15 23:00:26 25 +2016-06-15 23:00:27 25 +descending order +2016-06-15 23:00:25 25 original +2016-06-15 23:00:24 25 +2016-06-15 23:00:20 20 original +2016-06-15 23:00:19 20 +2016-06-15 23:00:15 15 original +2016-06-15 23:00:14 15 +2016-06-15 23:00:10 10 original +2016-06-15 23:00:09 10 +2016-06-15 23:00:05 5 original +2016-06-15 23:00:04 5 +2016-06-15 23:00:00 0 original +2016-06-15 22:59:59 0 +staleness with to and step +2016-06-15 23:00:00 0 original +2016-06-15 23:00:03 0 +2016-06-15 23:00:05 5 original +2016-06-15 23:00:06 5 +2016-06-15 23:00:09 5 +2016-06-15 23:00:10 10 original +2016-06-15 23:00:12 10 +2016-06-15 23:00:15 15 original +2016-06-15 23:00:18 15 +2016-06-15 23:00:20 20 original +2016-06-15 23:00:21 20 +2016-06-15 23:00:24 20 +2016-06-15 23:00:25 25 original +2016-06-15 23:00:27 25 +2016-06-15 23:00:30 25 +staleness with another regular with fill +2016-06-15 23:00:00 1970-01-01 01:00:00 0 +2016-06-15 23:00:00 1970-01-01 01:00:01 0 +2016-06-15 23:00:00 1970-01-01 01:00:02 0 +2016-06-15 23:00:00 2016-06-15 23:00:00 0 original +2016-06-15 23:00:01 1970-01-01 01:00:00 0 +2016-06-15 23:00:01 1970-01-01 01:00:01 0 +2016-06-15 23:00:01 1970-01-01 01:00:02 0 +2016-06-15 23:00:05 1970-01-01 01:00:00 0 +2016-06-15 23:00:05 1970-01-01 01:00:01 0 +2016-06-15 23:00:05 1970-01-01 01:00:02 0 +2016-06-15 23:00:05 2016-06-15 23:00:05 5 original +2016-06-15 23:00:06 1970-01-01 01:00:00 5 +2016-06-15 23:00:06 1970-01-01 01:00:01 5 +2016-06-15 23:00:06 1970-01-01 01:00:02 5 +2016-06-15 23:00:10 1970-01-01 01:00:00 5 +2016-06-15 23:00:10 1970-01-01 01:00:01 5 +2016-06-15 23:00:10 1970-01-01 01:00:02 5 +2016-06-15 23:00:10 2016-06-15 23:00:10 10 original +2016-06-15 23:00:11 1970-01-01 01:00:00 10 +2016-06-15 23:00:11 1970-01-01 01:00:01 10 +2016-06-15 23:00:11 1970-01-01 01:00:02 10 +2016-06-15 23:00:15 1970-01-01 01:00:00 10 +2016-06-15 23:00:15 1970-01-01 01:00:01 10 +2016-06-15 23:00:15 1970-01-01 01:00:02 10 +2016-06-15 23:00:15 2016-06-15 23:00:15 15 original +2016-06-15 23:00:16 1970-01-01 01:00:00 15 +2016-06-15 23:00:16 1970-01-01 01:00:01 15 +2016-06-15 23:00:16 1970-01-01 01:00:02 15 +2016-06-15 23:00:20 1970-01-01 01:00:00 15 +2016-06-15 23:00:20 1970-01-01 01:00:01 15 +2016-06-15 23:00:20 1970-01-01 01:00:02 15 +2016-06-15 23:00:20 2016-06-15 23:00:20 20 original +2016-06-15 23:00:21 1970-01-01 01:00:00 20 +2016-06-15 23:00:21 1970-01-01 01:00:01 20 +2016-06-15 23:00:21 1970-01-01 01:00:02 20 +2016-06-15 23:00:25 1970-01-01 01:00:00 20 +2016-06-15 23:00:25 1970-01-01 01:00:01 20 +2016-06-15 23:00:25 1970-01-01 01:00:02 20 +2016-06-15 23:00:25 2016-06-15 23:00:25 25 original +2016-06-15 23:00:26 1970-01-01 01:00:00 25 +2016-06-15 23:00:26 1970-01-01 01:00:01 25 +2016-06-15 23:00:26 1970-01-01 01:00:02 25 +double staleness +2016-06-15 23:00:00 2016-06-15 23:00:00 0 original +2016-06-15 23:00:00 2016-06-15 23:00:02 0 +2016-06-15 23:00:00 2016-06-15 23:00:04 0 +2016-06-15 23:00:01 1970-01-01 01:00:00 0 +2016-06-15 23:00:05 2016-06-15 23:00:05 5 original +2016-06-15 23:00:05 2016-06-15 23:00:07 5 +2016-06-15 23:00:05 2016-06-15 23:00:09 5 +2016-06-15 23:00:06 1970-01-01 01:00:00 5 +2016-06-15 23:00:10 2016-06-15 23:00:10 10 original +2016-06-15 23:00:10 2016-06-15 23:00:12 10 +2016-06-15 23:00:10 2016-06-15 23:00:14 10 +2016-06-15 23:00:11 1970-01-01 01:00:00 10 +2016-06-15 23:00:15 2016-06-15 23:00:15 15 original +2016-06-15 23:00:15 2016-06-15 23:00:17 15 +2016-06-15 23:00:15 2016-06-15 23:00:19 15 +2016-06-15 23:00:16 1970-01-01 01:00:00 15 +2016-06-15 23:00:20 2016-06-15 23:00:20 20 original +2016-06-15 23:00:20 2016-06-15 23:00:22 20 +2016-06-15 23:00:20 2016-06-15 23:00:24 20 +2016-06-15 23:00:21 1970-01-01 01:00:00 20 +2016-06-15 23:00:25 2016-06-15 23:00:25 25 original +2016-06-15 23:00:25 2016-06-15 23:00:27 25 +2016-06-15 23:00:25 2016-06-15 23:00:29 25 +2016-06-15 23:00:26 1970-01-01 01:00:00 25 diff --git a/tests/queries/0_stateless/03266_with_fill_staleness.sql b/tests/queries/0_stateless/03266_with_fill_staleness.sql new file mode 100644 index 00000000000..de47d8287ad --- /dev/null +++ b/tests/queries/0_stateless/03266_with_fill_staleness.sql @@ -0,0 +1,34 @@ +SET session_timezone='Europe/Amsterdam'; +SET enable_analyzer=1; + +DROP TABLE IF EXISTS with_fill_staleness; +CREATE TABLE with_fill_staleness (a DateTime, b DateTime, c UInt64) ENGINE = MergeTree ORDER BY a; + +SELECT 'add samples'; + +INSERT INTO with_fill_staleness +SELECT + toDateTime('2016-06-15 23:00:00') + number AS a, a as b, number as c +FROM numbers(30) +WHERE (number % 5) == 0; + +SELECT 'regular with fill'; +SELECT a, c, 'original' as original FROM with_fill_staleness ORDER BY a ASC WITH FILL INTERPOLATE (c); + +SELECT 'staleness 1 seconds'; +SELECT a, c, 'original' as original FROM with_fill_staleness ORDER BY a ASC WITH FILL STALENESS INTERVAL 1 SECOND INTERPOLATE (c); + +SELECT 'staleness 3 seconds'; +SELECT a, c, 'original' as original FROM with_fill_staleness ORDER BY a ASC WITH FILL STALENESS INTERVAL 3 SECOND INTERPOLATE (c); + +SELECT 'descending order'; +SELECT a, c, 'original' as original FROM with_fill_staleness ORDER BY a DESC WITH FILL STALENESS INTERVAL -2 SECOND INTERPOLATE (c); + +SELECT 'staleness with to and step'; +SELECT a, c, 'original' as original FROM with_fill_staleness ORDER BY a ASC WITH FILL TO toDateTime('2016-06-15 23:00:40') STEP 3 STALENESS INTERVAL 7 SECOND INTERPOLATE (c); + +SELECT 'staleness with another regular with fill'; +SELECT a, b, c, 'original' as original FROM with_fill_staleness ORDER BY a ASC WITH FILL STALENESS INTERVAL 2 SECOND, b ASC WITH FILL FROM 0 TO 3 INTERPOLATE (c); + +SELECT 'double staleness'; +SELECT a, b, c, 'original' as original FROM with_fill_staleness ORDER BY a ASC WITH FILL STALENESS INTERVAL 2 SECOND, b ASC WITH FILL TO toDateTime('2016-06-15 23:01:00') STEP 2 STALENESS 5 INTERPOLATE (c); diff --git a/tests/queries/0_stateless/03266_with_fill_staleness_cases.reference b/tests/queries/0_stateless/03266_with_fill_staleness_cases.reference new file mode 100644 index 00000000000..bf8e5bbe331 --- /dev/null +++ b/tests/queries/0_stateless/03266_with_fill_staleness_cases.reference @@ -0,0 +1,67 @@ +test-1 +0 5 10 original +0 5 13 +0 5 16 +0 5 19 +0 5 22 +0 7 0 +7 8 15 original +7 8 18 +7 8 21 +7 8 24 +7 10 0 +14 10 20 original +14 10 23 +14 12 0 +test-2-1 +1 0 original +1 1 +1 2 +1 3 +1 4 original +1 5 +1 6 +1 7 +1 8 original +1 9 +1 10 +1 11 +1 12 original +test-2-2 +1 0 original +1 1 +1 2 +1 3 +1 4 original +1 5 +1 6 +1 7 +1 8 original +1 9 +1 10 +1 11 +1 12 original +1 13 +1 14 +2 0 +3 0 +4 0 +test-3-1 +25 -10 +25 -8 +25 -6 +25 -4 +25 -2 +25 0 +25 2 +25 4 +25 6 +25 8 +25 10 +25 12 +25 14 +25 16 +25 17 original +28 -10 +30 18 original +31 -10 diff --git a/tests/queries/0_stateless/03266_with_fill_staleness_cases.sql b/tests/queries/0_stateless/03266_with_fill_staleness_cases.sql new file mode 100644 index 00000000000..9e28041c9a1 --- /dev/null +++ b/tests/queries/0_stateless/03266_with_fill_staleness_cases.sql @@ -0,0 +1,25 @@ +SET enable_analyzer=1; + +DROP TABLE IF EXISTS test; +CREATE TABLE test (a Int64, b Int64, c Int64) Engine=MergeTree ORDER BY a; +INSERT INTO test(a, b, c) VALUES (0, 5, 10), (7, 8, 15), (14, 10, 20); + +SELECT 'test-1'; +SELECT *, 'original' AS orig FROM test ORDER BY a, b WITH FILL TO 20 STEP 2 STALENESS 3, c WITH FILL TO 25 step 3; + +DROP TABLE IF EXISTS test2; +CREATE TABLE test2 (a Int64, b Int64) Engine=MergeTree ORDER BY a; +INSERT INTO test2(a, b) values (1, 0), (1, 4), (1, 8), (1, 12); + +SELECT 'test-2-1'; +SELECT *, 'original' AS orig FROM test2 ORDER BY a, b WITH FILL; + +SELECT 'test-2-2'; +SELECT *, 'original' AS orig FROM test2 ORDER BY a WITH FILL to 20 STALENESS 4, b WITH FILL TO 15 STALENESS 7; + +DROP TABLE IF EXISTS test2; +CREATE TABLE test3 (a Int64, b Int64) Engine=MergeTree ORDER BY a; +INSERT INTO test3(a, b) VALUES (25, 17), (30, 18); + +SELECT 'test-3-1'; +SELECT a, b, 'original' AS orig FROM test3 ORDER BY a WITH FILL TO 33 STEP 3, b WITH FILL FROM -10 STEP 2; diff --git a/tests/queries/0_stateless/03266_with_fill_staleness_errors.reference b/tests/queries/0_stateless/03266_with_fill_staleness_errors.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03266_with_fill_staleness_errors.sql b/tests/queries/0_stateless/03266_with_fill_staleness_errors.sql new file mode 100644 index 00000000000..fbfaf3743ca --- /dev/null +++ b/tests/queries/0_stateless/03266_with_fill_staleness_errors.sql @@ -0,0 +1,5 @@ +SET enable_analyzer=1; + +SELECT 1 AS a, 2 AS b ORDER BY a, b WITH FILL FROM 0 TO 10 STALENESS 3; -- { serverError INVALID_WITH_FILL_EXPRESSION } +SELECT 1 AS a, 2 AS b ORDER BY a, b DESC WITH FILL TO 10 STALENESS 3; -- { serverError INVALID_WITH_FILL_EXPRESSION } +SELECT 1 AS a, 2 AS b ORDER BY a, b ASC WITH FILL TO 10 STALENESS -3; -- { serverError INVALID_WITH_FILL_EXPRESSION } diff --git a/tests/queries/0_stateless/03267_implicit_select.reference b/tests/queries/0_stateless/03267_implicit_select.reference new file mode 100644 index 00000000000..97c1fd4333b --- /dev/null +++ b/tests/queries/0_stateless/03267_implicit_select.reference @@ -0,0 +1,5 @@ +3 +3 +3 +Syntax error +3 diff --git a/tests/queries/0_stateless/03267_implicit_select.sh b/tests/queries/0_stateless/03267_implicit_select.sh new file mode 100755 index 00000000000..068fb457bb1 --- /dev/null +++ b/tests/queries/0_stateless/03267_implicit_select.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_LOCAL "1 + 2" +$CLICKHOUSE_LOCAL -q "1 + 2" +$CLICKHOUSE_LOCAL --query "1 + 2" +$CLICKHOUSE_LOCAL --implicit_select 0 --query "1 + 2" 2>&1 | grep -oF 'Syntax error' +$CLICKHOUSE_LOCAL --implicit_select 0 --query "SELECT 1 + 2" diff --git a/tests/queries/0_stateless/03267_materialized_view_keeps_security_context.reference b/tests/queries/0_stateless/03267_materialized_view_keeps_security_context.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/03267_materialized_view_keeps_security_context.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/03267_materialized_view_keeps_security_context.sql b/tests/queries/0_stateless/03267_materialized_view_keeps_security_context.sql new file mode 100644 index 00000000000..bb44e4920af --- /dev/null +++ b/tests/queries/0_stateless/03267_materialized_view_keeps_security_context.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS {CLICKHOUSE_DATABASE:Identifier}.rview; +DROP TABLE IF EXISTS {CLICKHOUSE_DATABASE:Identifier}.wview; + +-- Read from view +CREATE MATERIALIZED VIEW rview ENGINE = File(CSV) POPULATE AS SELECT 1 AS c0; +SELECT 1 FROM rview; + +-- Write through view populate +CREATE MATERIALIZED VIEW wview ENGINE = Join(ALL, INNER, c0) POPULATE AS SELECT 1 AS c0; diff --git a/tests/queries/0_stateless/03267_min_parts_to_merge_at_once.reference b/tests/queries/0_stateless/03267_min_parts_to_merge_at_once.reference new file mode 100644 index 00000000000..966a0980e59 --- /dev/null +++ b/tests/queries/0_stateless/03267_min_parts_to_merge_at_once.reference @@ -0,0 +1,4 @@ +2 +3 +4 +1 diff --git a/tests/queries/0_stateless/03267_min_parts_to_merge_at_once.sh b/tests/queries/0_stateless/03267_min_parts_to_merge_at_once.sh new file mode 100755 index 00000000000..90b9d0339cf --- /dev/null +++ b/tests/queries/0_stateless/03267_min_parts_to_merge_at_once.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS t;" + +$CLICKHOUSE_CLIENT --query "CREATE TABLE t (key UInt64) ENGINE = MergeTree() ORDER BY tuple() SETTINGS min_parts_to_merge_at_once=5, merge_selector_base=1" + +$CLICKHOUSE_CLIENT --query "INSERT INTO t VALUES (1)" +$CLICKHOUSE_CLIENT --query "INSERT INTO t VALUES (2);" + +# doesn't make test flaky +sleep 1 + +$CLICKHOUSE_CLIENT --query "SELECT count() FROM system.parts WHERE active and database = currentDatabase() and table = 't'" + +$CLICKHOUSE_CLIENT --query "INSERT INTO t VALUES (3)" + +$CLICKHOUSE_CLIENT --query "SELECT count() FROM system.parts WHERE active and database = currentDatabase() and table = 't'" + +$CLICKHOUSE_CLIENT --query "INSERT INTO t VALUES (4)" + +$CLICKHOUSE_CLIENT --query "SELECT count() FROM system.parts WHERE active and database = currentDatabase() and table = 't'" + +$CLICKHOUSE_CLIENT --query "INSERT INTO t VALUES (5)" + +counter=0 retries=60 + +while [[ $counter -lt $retries ]]; do + result=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.parts WHERE active and database = currentDatabase() and table = 't'") + if [ "$result" -eq "1" ];then + break; + fi + sleep 0.5 + counter=$((counter + 1)) +done + +$CLICKHOUSE_CLIENT --query "SELECT count() FROM system.parts WHERE active and database = currentDatabase() and table = 't'" + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS t" diff --git a/tests/queries/0_stateless/03267_positional_arguments_implicit_query_file.reference b/tests/queries/0_stateless/03267_positional_arguments_implicit_query_file.reference new file mode 100644 index 00000000000..fe2432a063f --- /dev/null +++ b/tests/queries/0_stateless/03267_positional_arguments_implicit_query_file.reference @@ -0,0 +1,11 @@ +Hello from a file +Hello from a file +Hello from a file +Hello from a file +Hello from a file +Hello from a file +Hello from a file +Hello from a file +Hello from a file +max_local_read_bandwidth 1 100 +max_local_read_bandwidth 1 200 diff --git a/tests/queries/0_stateless/03267_positional_arguments_implicit_query_file.sh b/tests/queries/0_stateless/03267_positional_arguments_implicit_query_file.sh new file mode 100755 index 00000000000..791aa3af0db --- /dev/null +++ b/tests/queries/0_stateless/03267_positional_arguments_implicit_query_file.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Tags: no-random-settings + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +FILE=${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_without_extension +echo "SELECT 'Hello from a file'" > ${FILE} + +# Queries can be read from a file. +${CLICKHOUSE_BINARY} --queries-file ${FILE} + +# Or from stdin. +${CLICKHOUSE_BINARY} < ${FILE} + +# Also the positional argument can be interpreted as a file. +${CLICKHOUSE_BINARY} ${FILE} + +${CLICKHOUSE_LOCAL} --queries-file ${FILE} +${CLICKHOUSE_LOCAL} < ${FILE} +${CLICKHOUSE_LOCAL} ${FILE} + +${CLICKHOUSE_CLIENT} --queries-file ${FILE} +${CLICKHOUSE_CLIENT} < ${FILE} +${CLICKHOUSE_CLIENT} ${FILE} + +# Check that positional arguments work in any place +echo "Select name, changed, value FROM system.settings where name = 'max_local_read_bandwidth'" > ${FILE} +${CLICKHOUSE_BINARY} ${FILE} --max-local-read-bandwidth 100 +${CLICKHOUSE_BINARY} --max-local-read-bandwidth 200 ${FILE} + +rm ${FILE} diff --git a/tests/queries/0_stateless/03268_nested_analyzer.reference b/tests/queries/0_stateless/03268_nested_analyzer.reference new file mode 100644 index 00000000000..01dabfe4ba7 --- /dev/null +++ b/tests/queries/0_stateless/03268_nested_analyzer.reference @@ -0,0 +1,3 @@ +[(1,3),(2,4)] +0 0 +0 0 1 diff --git a/tests/queries/0_stateless/03268_nested_analyzer.sql b/tests/queries/0_stateless/03268_nested_analyzer.sql new file mode 100644 index 00000000000..920cf2b3174 --- /dev/null +++ b/tests/queries/0_stateless/03268_nested_analyzer.sql @@ -0,0 +1,16 @@ +SELECT nested(['a', 'b'], [1, 2], materialize([3, 4])); + +DROP TABLE IF EXISTS test; +CREATE TABLE test +( + x UInt8, + “struct.x” DEFAULT [0], + “struct.y” ALIAS [1], +) +ENGINE = Memory; + +insert into test (x) values (0); +select * from test array join struct; +select x, struct.x, struct.y from test array join struct; + +DROP TABLE test; diff --git a/tests/queries/0_stateless/03268_system_parts_index_granularity.reference b/tests/queries/0_stateless/03268_system_parts_index_granularity.reference new file mode 100644 index 00000000000..f301cd54ad2 --- /dev/null +++ b/tests/queries/0_stateless/03268_system_parts_index_granularity.reference @@ -0,0 +1 @@ +88 88 diff --git a/tests/queries/0_stateless/03268_system_parts_index_granularity.sql b/tests/queries/0_stateless/03268_system_parts_index_granularity.sql new file mode 100644 index 00000000000..1bab7840856 --- /dev/null +++ b/tests/queries/0_stateless/03268_system_parts_index_granularity.sql @@ -0,0 +1,15 @@ +-- Tags: no-random-settings, no-random-merge-tree-settings +DROP TABLE IF EXISTS t; + +CREATE TABLE t ( + key UInt64, + value String +) +ENGINE MergeTree() +ORDER by key SETTINGS index_granularity = 10, index_granularity_bytes = '1024K'; + +INSERT INTO t SELECT number, toString(number) FROM numbers(100); + +SELECT index_granularity_bytes_in_memory, index_granularity_bytes_in_memory_allocated FROM system.parts where table = 't' and database = currentDatabase(); + +DROP TABLE IF EXISTS t; diff --git a/tests/queries/0_stateless/03268_vertical_pretty_numbers.reference b/tests/queries/0_stateless/03268_vertical_pretty_numbers.reference new file mode 100644 index 00000000000..397e9145798 --- /dev/null +++ b/tests/queries/0_stateless/03268_vertical_pretty_numbers.reference @@ -0,0 +1,1532 @@ +Row 1: +────── +exp2(number): 1 +exp10(number): 1 +concat('test', number): test0 + +Row 2: +────── +exp2(number): 2 -- 2.00 +exp10(number): 10 -- 10.00 +concat('test', number): test1 + +Row 3: +────── +exp2(number): 4 -- 4.00 +exp10(number): 100 -- 100.00 +concat('test', number): test2 + +Row 4: +────── +exp2(number): 8 -- 8.00 +exp10(number): 1000 -- 1.00 thousand +concat('test', number): test3 + +Row 5: +────── +exp2(number): 16 -- 16.00 +exp10(number): 10000 -- 10.00 thousand +concat('test', number): test4 + +Row 6: +────── +exp2(number): 32 -- 32.00 +exp10(number): 100000 -- 100.00 thousand +concat('test', number): test5 + +Row 7: +────── +exp2(number): 64 -- 64.00 +exp10(number): 1000000 -- 1.00 million +concat('test', number): test6 + +Row 8: +────── +exp2(number): 128 -- 128.00 +exp10(number): 10000000 -- 10.00 million +concat('test', number): test7 + +Row 9: +─────── +exp2(number): 256 -- 256.00 +exp10(number): 100000000 -- 100.00 million +concat('test', number): test8 + +Row 10: +─────── +exp2(number): 512 -- 512.00 +exp10(number): 1000000000 -- 1.00 billion +concat('test', number): test9 + +Row 11: +─────── +exp2(number): 1024 -- 1.02 thousand +exp10(number): 10000000000 -- 10.00 billion +concat('test', number): test10 + +Row 12: +─────── +exp2(number): 2048 -- 2.05 thousand +exp10(number): 100000000000 -- 100.00 billion +concat('test', number): test11 + +Row 13: +─────── +exp2(number): 4096 -- 4.10 thousand +exp10(number): 1000000000000 -- 1.00 trillion +concat('test', number): test12 + +Row 14: +─────── +exp2(number): 8192 -- 8.19 thousand +exp10(number): 10000000000000 -- 10.00 trillion +concat('test', number): test13 + +Row 15: +─────── +exp2(number): 16384 -- 16.38 thousand +exp10(number): 100000000000000 -- 100.00 trillion +concat('test', number): test14 + +Row 16: +─────── +exp2(number): 32768 -- 32.77 thousand +exp10(number): 1000000000000000 -- 1.00 quadrillion +concat('test', number): test15 + +Row 17: +─────── +exp2(number): 65536 -- 65.54 thousand +exp10(number): 10000000000000000 -- 10.00 quadrillion +concat('test', number): test16 + +Row 18: +─────── +exp2(number): 131072 -- 131.07 thousand +exp10(number): 100000000000000000 -- 100.00 quadrillion +concat('test', number): test17 + +Row 19: +─────── +exp2(number): 262144 -- 262.14 thousand +exp10(number): 1000000000000000000 -- 1.00 quintillion +concat('test', number): test18 + +Row 20: +─────── +exp2(number): 524288 -- 524.29 thousand +exp10(number): 10000000000000000000 -- 10.00 quintillion +concat('test', number): test19 + +Row 21: +─────── +exp2(number): 1048576 -- 1.05 million +exp10(number): 100000000000000000000 -- 100.00 quintillion +concat('test', number): test20 + +Row 22: +─────── +exp2(number): 2097152 -- 2.10 million +exp10(number): 1e21 -- 1.00 sextillion +concat('test', number): test21 + +Row 23: +─────── +exp2(number): 4194304 -- 4.19 million +exp10(number): 1e22 -- 10.00 sextillion +concat('test', number): test22 + +Row 24: +─────── +exp2(number): 8388608 -- 8.39 million +exp10(number): 1e23 -- 100.00 sextillion +concat('test', number): test23 + +Row 25: +─────── +exp2(number): 16777216 -- 16.78 million +exp10(number): 1e24 -- 1.00 septillion +concat('test', number): test24 + +Row 26: +─────── +exp2(number): 33554432 -- 33.55 million +exp10(number): 1e25 -- 10.00 septillion +concat('test', number): test25 + +Row 27: +─────── +exp2(number): 67108864 -- 67.11 million +exp10(number): 1e26 -- 100.00 septillion +concat('test', number): test26 + +Row 28: +─────── +exp2(number): 134217728 -- 134.22 million +exp10(number): 1e27 -- 1.00 octillion +concat('test', number): test27 + +Row 29: +─────── +exp2(number): 268435456 -- 268.44 million +exp10(number): 1e28 -- 10.00 octillion +concat('test', number): test28 + +Row 30: +─────── +exp2(number): 536870912 -- 536.87 million +exp10(number): 1e29 -- 100.00 octillion +concat('test', number): test29 + +Row 31: +─────── +exp2(number): 1073741824 -- 1.07 billion +exp10(number): 1e30 -- 1.00 nonillion +concat('test', number): test30 + +Row 32: +─────── +exp2(number): 2147483648 -- 2.15 billion +exp10(number): 1e31 -- 10.00 nonillion +concat('test', number): test31 + +Row 33: +─────── +exp2(number): 4294967296 -- 4.29 billion +exp10(number): 1e32 -- 100.00 nonillion +concat('test', number): test32 + +Row 34: +─────── +exp2(number): 8589934592 -- 8.59 billion +exp10(number): 1e33 -- 1000.00 nonillion +concat('test', number): test33 + +Row 35: +─────── +exp2(number): 17179869184 -- 17.18 billion +exp10(number): 1e34 -- 10.00 decillion +concat('test', number): test34 + +Row 36: +─────── +exp2(number): 34359738368 -- 34.36 billion +exp10(number): 1e35 -- 100.00 decillion +concat('test', number): test35 + +Row 37: +─────── +exp2(number): 68719476736 -- 68.72 billion +exp10(number): 1e36 -- 1.00 undecillion +concat('test', number): test36 + +Row 38: +─────── +exp2(number): 137438953472 -- 137.44 billion +exp10(number): 1e37 -- 10.00 undecillion +concat('test', number): test37 + +Row 39: +─────── +exp2(number): 274877906944 -- 274.88 billion +exp10(number): 1e38 -- 100.00 undecillion +concat('test', number): test38 + +Row 40: +─────── +exp2(number): 549755813888 -- 549.76 billion +exp10(number): 1e39 -- 1000.00 undecillion +concat('test', number): test39 + +Row 41: +─────── +exp2(number): 1099511627776 -- 1.10 trillion +exp10(number): 1e40 -- 10.00 duodecillion +concat('test', number): test40 + +Row 42: +─────── +exp2(number): 2199023255552 -- 2.20 trillion +exp10(number): 1e41 -- 100.00 duodecillion +concat('test', number): test41 + +Row 43: +─────── +exp2(number): 4398046511104 -- 4.40 trillion +exp10(number): 1e42 -- 1.00 tredecillion +concat('test', number): test42 + +Row 44: +─────── +exp2(number): 8796093022208 -- 8.80 trillion +exp10(number): 1e43 -- 10.00 tredecillion +concat('test', number): test43 + +Row 45: +─────── +exp2(number): 17592186044416 -- 17.59 trillion +exp10(number): 1e44 -- 100.00 tredecillion +concat('test', number): test44 + +Row 46: +─────── +exp2(number): 35184372088832 -- 35.18 trillion +exp10(number): 1e45 -- 1000.00 tredecillion +concat('test', number): test45 + +Row 47: +─────── +exp2(number): 70368744177664 -- 70.37 trillion +exp10(number): 1e46 -- 10.00 quattuordecillion +concat('test', number): test46 + +Row 48: +─────── +exp2(number): 140737488355328 -- 140.74 trillion +exp10(number): 1e47 -- 100.00 quattuordecillion +concat('test', number): test47 + +Row 49: +─────── +exp2(number): 281474976710656 -- 281.47 trillion +exp10(number): 1e48 -- 1.00 quindecillion +concat('test', number): test48 + +Row 50: +─────── +exp2(number): 562949953421312 -- 562.95 trillion +exp10(number): 1e49 -- 10.00 quindecillion +concat('test', number): test49 + +Row 51: +─────── +exp2(number): 1125899906842624 -- 1.13 quadrillion +exp10(number): 1e50 -- 100.00 quindecillion +concat('test', number): test50 + +Row 52: +─────── +exp2(number): 2251799813685248 -- 2.25 quadrillion +exp10(number): 1e51 -- 1.00 sexdecillion +concat('test', number): test51 + +Row 53: +─────── +exp2(number): 4503599627370496 -- 4.50 quadrillion +exp10(number): 1e52 -- 10.00 sexdecillion +concat('test', number): test52 + +Row 54: +─────── +exp2(number): 9007199254740992 -- 9.01 quadrillion +exp10(number): 1e53 -- 100.00 sexdecillion +concat('test', number): test53 + +Row 55: +─────── +exp2(number): 18014398509481984 -- 18.01 quadrillion +exp10(number): 1e54 -- 1.00 septendecillion +concat('test', number): test54 + +Row 56: +─────── +exp2(number): 36028797018963970 -- 36.03 quadrillion +exp10(number): 1e55 -- 10.00 septendecillion +concat('test', number): test55 + +Row 57: +─────── +exp2(number): 72057594037927940 -- 72.06 quadrillion +exp10(number): 1e56 -- 100.00 septendecillion +concat('test', number): test56 + +Row 58: +─────── +exp2(number): 144115188075855870 -- 144.12 quadrillion +exp10(number): 1e57 -- 1.00 octodecillion +concat('test', number): test57 + +Row 59: +─────── +exp2(number): 288230376151711740 -- 288.23 quadrillion +exp10(number): 1e58 -- 10.00 octodecillion +concat('test', number): test58 + +Row 60: +─────── +exp2(number): 576460752303423500 -- 576.46 quadrillion +exp10(number): 1e59 -- 100.00 octodecillion +concat('test', number): test59 + +Row 61: +─────── +exp2(number): 1152921504606847000 -- 1.15 quintillion +exp10(number): 1e60 -- 1000.00 octodecillion +concat('test', number): test60 + +Row 62: +─────── +exp2(number): 2305843009213694000 -- 2.31 quintillion +exp10(number): 1e61 -- 10.00 novemdecillion +concat('test', number): test61 + +Row 63: +─────── +exp2(number): 4611686018427388000 -- 4.61 quintillion +exp10(number): 1e62 -- 100.00 novemdecillion +concat('test', number): test62 + +Row 64: +─────── +exp2(number): 9223372036854776000 -- 9.22 quintillion +exp10(number): 1e63 -- 1.00 vigintillion +concat('test', number): test63 +Row 1: +────── +exp2(number): 1 +exp10(number): 1 +concat('test', number): test0 + +Row 2: +────── +exp2(number): 2 -- 2.00 +exp10(number): 10 -- 10.00 +concat('test', number): test1 + +Row 3: +────── +exp2(number): 4 -- 4.00 +exp10(number): 100 -- 100.00 +concat('test', number): test2 + +Row 4: +────── +exp2(number): 8 -- 8.00 +exp10(number): 1000 -- 1.00 thousand +concat('test', number): test3 + +Row 5: +────── +exp2(number): 16 -- 16.00 +exp10(number): 10000 -- 10.00 thousand +concat('test', number): test4 + +Row 6: +────── +exp2(number): 32 -- 32.00 +exp10(number): 100000 -- 100.00 thousand +concat('test', number): test5 + +Row 7: +────── +exp2(number): 64 -- 64.00 +exp10(number): 1000000 -- 1.00 million +concat('test', number): test6 + +Row 8: +────── +exp2(number): 128 -- 128.00 +exp10(number): 10000000 -- 10.00 million +concat('test', number): test7 + +Row 9: +─────── +exp2(number): 256 -- 256.00 +exp10(number): 100000000 -- 100.00 million +concat('test', number): test8 + +Row 10: +─────── +exp2(number): 512 -- 512.00 +exp10(number): 1000000000 -- 1.00 billion +concat('test', number): test9 + +Row 11: +─────── +exp2(number): 1024 -- 1.02 thousand +exp10(number): 10000000000 -- 10.00 billion +concat('test', number): test10 + +Row 12: +─────── +exp2(number): 2048 -- 2.05 thousand +exp10(number): 100000000000 -- 100.00 billion +concat('test', number): test11 + +Row 13: +─────── +exp2(number): 4096 -- 4.10 thousand +exp10(number): 1000000000000 -- 1.00 trillion +concat('test', number): test12 + +Row 14: +─────── +exp2(number): 8192 -- 8.19 thousand +exp10(number): 10000000000000 -- 10.00 trillion +concat('test', number): test13 + +Row 15: +─────── +exp2(number): 16384 -- 16.38 thousand +exp10(number): 100000000000000 -- 100.00 trillion +concat('test', number): test14 + +Row 16: +─────── +exp2(number): 32768 -- 32.77 thousand +exp10(number): 1000000000000000 -- 1.00 quadrillion +concat('test', number): test15 + +Row 17: +─────── +exp2(number): 65536 -- 65.54 thousand +exp10(number): 10000000000000000 -- 10.00 quadrillion +concat('test', number): test16 + +Row 18: +─────── +exp2(number): 131072 -- 131.07 thousand +exp10(number): 100000000000000000 -- 100.00 quadrillion +concat('test', number): test17 + +Row 19: +─────── +exp2(number): 262144 -- 262.14 thousand +exp10(number): 1000000000000000000 -- 1.00 quintillion +concat('test', number): test18 + +Row 20: +─────── +exp2(number): 524288 -- 524.29 thousand +exp10(number): 10000000000000000000 -- 10.00 quintillion +concat('test', number): test19 + +Row 21: +─────── +exp2(number): 1048576 -- 1.05 million +exp10(number): 100000000000000000000 -- 100.00 quintillion +concat('test', number): test20 + +Row 22: +─────── +exp2(number): 2097152 -- 2.10 million +exp10(number): 1e21 -- 1.00 sextillion +concat('test', number): test21 + +Row 23: +─────── +exp2(number): 4194304 -- 4.19 million +exp10(number): 1e22 -- 10.00 sextillion +concat('test', number): test22 + +Row 24: +─────── +exp2(number): 8388608 -- 8.39 million +exp10(number): 1e23 -- 100.00 sextillion +concat('test', number): test23 + +Row 25: +─────── +exp2(number): 16777216 -- 16.78 million +exp10(number): 1e24 -- 1.00 septillion +concat('test', number): test24 + +Row 26: +─────── +exp2(number): 33554432 -- 33.55 million +exp10(number): 1e25 -- 10.00 septillion +concat('test', number): test25 + +Row 27: +─────── +exp2(number): 67108864 -- 67.11 million +exp10(number): 1e26 -- 100.00 septillion +concat('test', number): test26 + +Row 28: +─────── +exp2(number): 134217728 -- 134.22 million +exp10(number): 1e27 -- 1.00 octillion +concat('test', number): test27 + +Row 29: +─────── +exp2(number): 268435456 -- 268.44 million +exp10(number): 1e28 -- 10.00 octillion +concat('test', number): test28 + +Row 30: +─────── +exp2(number): 536870912 -- 536.87 million +exp10(number): 1e29 -- 100.00 octillion +concat('test', number): test29 + +Row 31: +─────── +exp2(number): 1073741824 -- 1.07 billion +exp10(number): 1e30 -- 1.00 nonillion +concat('test', number): test30 + +Row 32: +─────── +exp2(number): 2147483648 -- 2.15 billion +exp10(number): 1e31 -- 10.00 nonillion +concat('test', number): test31 + +Row 33: +─────── +exp2(number): 4294967296 -- 4.29 billion +exp10(number): 1e32 -- 100.00 nonillion +concat('test', number): test32 + +Row 34: +─────── +exp2(number): 8589934592 -- 8.59 billion +exp10(number): 1e33 -- 1000.00 nonillion +concat('test', number): test33 + +Row 35: +─────── +exp2(number): 17179869184 -- 17.18 billion +exp10(number): 1e34 -- 10.00 decillion +concat('test', number): test34 + +Row 36: +─────── +exp2(number): 34359738368 -- 34.36 billion +exp10(number): 1e35 -- 100.00 decillion +concat('test', number): test35 + +Row 37: +─────── +exp2(number): 68719476736 -- 68.72 billion +exp10(number): 1e36 -- 1.00 undecillion +concat('test', number): test36 + +Row 38: +─────── +exp2(number): 137438953472 -- 137.44 billion +exp10(number): 1e37 -- 10.00 undecillion +concat('test', number): test37 + +Row 39: +─────── +exp2(number): 274877906944 -- 274.88 billion +exp10(number): 1e38 -- 100.00 undecillion +concat('test', number): test38 + +Row 40: +─────── +exp2(number): 549755813888 -- 549.76 billion +exp10(number): 1e39 -- 1000.00 undecillion +concat('test', number): test39 + +Row 41: +─────── +exp2(number): 1099511627776 -- 1.10 trillion +exp10(number): 1e40 -- 10.00 duodecillion +concat('test', number): test40 + +Row 42: +─────── +exp2(number): 2199023255552 -- 2.20 trillion +exp10(number): 1e41 -- 100.00 duodecillion +concat('test', number): test41 + +Row 43: +─────── +exp2(number): 4398046511104 -- 4.40 trillion +exp10(number): 1e42 -- 1.00 tredecillion +concat('test', number): test42 + +Row 44: +─────── +exp2(number): 8796093022208 -- 8.80 trillion +exp10(number): 1e43 -- 10.00 tredecillion +concat('test', number): test43 + +Row 45: +─────── +exp2(number): 17592186044416 -- 17.59 trillion +exp10(number): 1e44 -- 100.00 tredecillion +concat('test', number): test44 + +Row 46: +─────── +exp2(number): 35184372088832 -- 35.18 trillion +exp10(number): 1e45 -- 1000.00 tredecillion +concat('test', number): test45 + +Row 47: +─────── +exp2(number): 70368744177664 -- 70.37 trillion +exp10(number): 1e46 -- 10.00 quattuordecillion +concat('test', number): test46 + +Row 48: +─────── +exp2(number): 140737488355328 -- 140.74 trillion +exp10(number): 1e47 -- 100.00 quattuordecillion +concat('test', number): test47 + +Row 49: +─────── +exp2(number): 281474976710656 -- 281.47 trillion +exp10(number): 1e48 -- 1.00 quindecillion +concat('test', number): test48 + +Row 50: +─────── +exp2(number): 562949953421312 -- 562.95 trillion +exp10(number): 1e49 -- 10.00 quindecillion +concat('test', number): test49 + +Row 51: +─────── +exp2(number): 1125899906842624 -- 1.13 quadrillion +exp10(number): 1e50 -- 100.00 quindecillion +concat('test', number): test50 + +Row 52: +─────── +exp2(number): 2251799813685248 -- 2.25 quadrillion +exp10(number): 1e51 -- 1.00 sexdecillion +concat('test', number): test51 + +Row 53: +─────── +exp2(number): 4503599627370496 -- 4.50 quadrillion +exp10(number): 1e52 -- 10.00 sexdecillion +concat('test', number): test52 + +Row 54: +─────── +exp2(number): 9007199254740992 -- 9.01 quadrillion +exp10(number): 1e53 -- 100.00 sexdecillion +concat('test', number): test53 + +Row 55: +─────── +exp2(number): 18014398509481984 -- 18.01 quadrillion +exp10(number): 1e54 -- 1.00 septendecillion +concat('test', number): test54 + +Row 56: +─────── +exp2(number): 36028797018963970 -- 36.03 quadrillion +exp10(number): 1e55 -- 10.00 septendecillion +concat('test', number): test55 + +Row 57: +─────── +exp2(number): 72057594037927940 -- 72.06 quadrillion +exp10(number): 1e56 -- 100.00 septendecillion +concat('test', number): test56 + +Row 58: +─────── +exp2(number): 144115188075855870 -- 144.12 quadrillion +exp10(number): 1e57 -- 1.00 octodecillion +concat('test', number): test57 + +Row 59: +─────── +exp2(number): 288230376151711740 -- 288.23 quadrillion +exp10(number): 1e58 -- 10.00 octodecillion +concat('test', number): test58 + +Row 60: +─────── +exp2(number): 576460752303423500 -- 576.46 quadrillion +exp10(number): 1e59 -- 100.00 octodecillion +concat('test', number): test59 + +Row 61: +─────── +exp2(number): 1152921504606847000 -- 1.15 quintillion +exp10(number): 1e60 -- 1000.00 octodecillion +concat('test', number): test60 + +Row 62: +─────── +exp2(number): 2305843009213694000 -- 2.31 quintillion +exp10(number): 1e61 -- 10.00 novemdecillion +concat('test', number): test61 + +Row 63: +─────── +exp2(number): 4611686018427388000 -- 4.61 quintillion +exp10(number): 1e62 -- 100.00 novemdecillion +concat('test', number): test62 + +Row 64: +─────── +exp2(number): 9223372036854776000 -- 9.22 quintillion +exp10(number): 1e63 -- 1.00 vigintillion +concat('test', number): test63 +Row 1: +────── +exp2(number): 1 +exp10(number): 1 +concat('test', number): test0 + +Row 2: +────── +exp2(number): 2 -- 2.00 +exp10(number): 10 -- 10.00 +concat('test', number): test1 + +Row 3: +────── +exp2(number): 4 -- 4.00 +exp10(number): 100 -- 100.00 +concat('test', number): test2 + +Row 4: +────── +exp2(number): 8 -- 8.00 +exp10(number): 1000 -- 1.00 thousand +concat('test', number): test3 + +Row 5: +────── +exp2(number): 16 -- 16.00 +exp10(number): 10000 -- 10.00 thousand +concat('test', number): test4 + +Row 6: +────── +exp2(number): 32 -- 32.00 +exp10(number): 100000 -- 100.00 thousand +concat('test', number): test5 + +Row 7: +────── +exp2(number): 64 -- 64.00 +exp10(number): 1000000 -- 1.00 million +concat('test', number): test6 + +Row 8: +────── +exp2(number): 128 -- 128.00 +exp10(number): 10000000 -- 10.00 million +concat('test', number): test7 + +Row 9: +─────── +exp2(number): 256 -- 256.00 +exp10(number): 100000000 -- 100.00 million +concat('test', number): test8 + +Row 10: +─────── +exp2(number): 512 -- 512.00 +exp10(number): 1000000000 -- 1.00 billion +concat('test', number): test9 + +Row 11: +─────── +exp2(number): 1024 -- 1.02 thousand +exp10(number): 10000000000 -- 10.00 billion +concat('test', number): test10 + +Row 12: +─────── +exp2(number): 2048 -- 2.05 thousand +exp10(number): 100000000000 -- 100.00 billion +concat('test', number): test11 + +Row 13: +─────── +exp2(number): 4096 -- 4.10 thousand +exp10(number): 1000000000000 -- 1.00 trillion +concat('test', number): test12 + +Row 14: +─────── +exp2(number): 8192 -- 8.19 thousand +exp10(number): 10000000000000 -- 10.00 trillion +concat('test', number): test13 + +Row 15: +─────── +exp2(number): 16384 -- 16.38 thousand +exp10(number): 100000000000000 -- 100.00 trillion +concat('test', number): test14 + +Row 16: +─────── +exp2(number): 32768 -- 32.77 thousand +exp10(number): 1000000000000000 -- 1.00 quadrillion +concat('test', number): test15 + +Row 17: +─────── +exp2(number): 65536 -- 65.54 thousand +exp10(number): 10000000000000000 -- 10.00 quadrillion +concat('test', number): test16 + +Row 18: +─────── +exp2(number): 131072 -- 131.07 thousand +exp10(number): 100000000000000000 -- 100.00 quadrillion +concat('test', number): test17 + +Row 19: +─────── +exp2(number): 262144 -- 262.14 thousand +exp10(number): 1000000000000000000 -- 1.00 quintillion +concat('test', number): test18 + +Row 20: +─────── +exp2(number): 524288 -- 524.29 thousand +exp10(number): 10000000000000000000 -- 10.00 quintillion +concat('test', number): test19 + +Row 21: +─────── +exp2(number): 1048576 -- 1.05 million +exp10(number): 100000000000000000000 -- 100.00 quintillion +concat('test', number): test20 + +Row 22: +─────── +exp2(number): 2097152 -- 2.10 million +exp10(number): 1e21 -- 1.00 sextillion +concat('test', number): test21 + +Row 23: +─────── +exp2(number): 4194304 -- 4.19 million +exp10(number): 1e22 -- 10.00 sextillion +concat('test', number): test22 + +Row 24: +─────── +exp2(number): 8388608 -- 8.39 million +exp10(number): 1e23 -- 100.00 sextillion +concat('test', number): test23 + +Row 25: +─────── +exp2(number): 16777216 -- 16.78 million +exp10(number): 1e24 -- 1.00 septillion +concat('test', number): test24 + +Row 26: +─────── +exp2(number): 33554432 -- 33.55 million +exp10(number): 1e25 -- 10.00 septillion +concat('test', number): test25 + +Row 27: +─────── +exp2(number): 67108864 -- 67.11 million +exp10(number): 1e26 -- 100.00 septillion +concat('test', number): test26 + +Row 28: +─────── +exp2(number): 134217728 -- 134.22 million +exp10(number): 1e27 -- 1.00 octillion +concat('test', number): test27 + +Row 29: +─────── +exp2(number): 268435456 -- 268.44 million +exp10(number): 1e28 -- 10.00 octillion +concat('test', number): test28 + +Row 30: +─────── +exp2(number): 536870912 -- 536.87 million +exp10(number): 1e29 -- 100.00 octillion +concat('test', number): test29 + +Row 31: +─────── +exp2(number): 1073741824 -- 1.07 billion +exp10(number): 1e30 -- 1.00 nonillion +concat('test', number): test30 + +Row 32: +─────── +exp2(number): 2147483648 -- 2.15 billion +exp10(number): 1e31 -- 10.00 nonillion +concat('test', number): test31 + +Row 33: +─────── +exp2(number): 4294967296 -- 4.29 billion +exp10(number): 1e32 -- 100.00 nonillion +concat('test', number): test32 + +Row 34: +─────── +exp2(number): 8589934592 -- 8.59 billion +exp10(number): 1e33 -- 1000.00 nonillion +concat('test', number): test33 + +Row 35: +─────── +exp2(number): 17179869184 -- 17.18 billion +exp10(number): 1e34 -- 10.00 decillion +concat('test', number): test34 + +Row 36: +─────── +exp2(number): 34359738368 -- 34.36 billion +exp10(number): 1e35 -- 100.00 decillion +concat('test', number): test35 + +Row 37: +─────── +exp2(number): 68719476736 -- 68.72 billion +exp10(number): 1e36 -- 1.00 undecillion +concat('test', number): test36 + +Row 38: +─────── +exp2(number): 137438953472 -- 137.44 billion +exp10(number): 1e37 -- 10.00 undecillion +concat('test', number): test37 + +Row 39: +─────── +exp2(number): 274877906944 -- 274.88 billion +exp10(number): 1e38 -- 100.00 undecillion +concat('test', number): test38 + +Row 40: +─────── +exp2(number): 549755813888 -- 549.76 billion +exp10(number): 1e39 -- 1000.00 undecillion +concat('test', number): test39 + +Row 41: +─────── +exp2(number): 1099511627776 -- 1.10 trillion +exp10(number): 1e40 -- 10.00 duodecillion +concat('test', number): test40 + +Row 42: +─────── +exp2(number): 2199023255552 -- 2.20 trillion +exp10(number): 1e41 -- 100.00 duodecillion +concat('test', number): test41 + +Row 43: +─────── +exp2(number): 4398046511104 -- 4.40 trillion +exp10(number): 1e42 -- 1.00 tredecillion +concat('test', number): test42 + +Row 44: +─────── +exp2(number): 8796093022208 -- 8.80 trillion +exp10(number): 1e43 -- 10.00 tredecillion +concat('test', number): test43 + +Row 45: +─────── +exp2(number): 17592186044416 -- 17.59 trillion +exp10(number): 1e44 -- 100.00 tredecillion +concat('test', number): test44 + +Row 46: +─────── +exp2(number): 35184372088832 -- 35.18 trillion +exp10(number): 1e45 -- 1000.00 tredecillion +concat('test', number): test45 + +Row 47: +─────── +exp2(number): 70368744177664 -- 70.37 trillion +exp10(number): 1e46 -- 10.00 quattuordecillion +concat('test', number): test46 + +Row 48: +─────── +exp2(number): 140737488355328 -- 140.74 trillion +exp10(number): 1e47 -- 100.00 quattuordecillion +concat('test', number): test47 + +Row 49: +─────── +exp2(number): 281474976710656 -- 281.47 trillion +exp10(number): 1e48 -- 1.00 quindecillion +concat('test', number): test48 + +Row 50: +─────── +exp2(number): 562949953421312 -- 562.95 trillion +exp10(number): 1e49 -- 10.00 quindecillion +concat('test', number): test49 + +Row 51: +─────── +exp2(number): 1125899906842624 -- 1.13 quadrillion +exp10(number): 1e50 -- 100.00 quindecillion +concat('test', number): test50 + +Row 52: +─────── +exp2(number): 2251799813685248 -- 2.25 quadrillion +exp10(number): 1e51 -- 1.00 sexdecillion +concat('test', number): test51 + +Row 53: +─────── +exp2(number): 4503599627370496 -- 4.50 quadrillion +exp10(number): 1e52 -- 10.00 sexdecillion +concat('test', number): test52 + +Row 54: +─────── +exp2(number): 9007199254740992 -- 9.01 quadrillion +exp10(number): 1e53 -- 100.00 sexdecillion +concat('test', number): test53 + +Row 55: +─────── +exp2(number): 18014398509481984 -- 18.01 quadrillion +exp10(number): 1e54 -- 1.00 septendecillion +concat('test', number): test54 + +Row 56: +─────── +exp2(number): 36028797018963970 -- 36.03 quadrillion +exp10(number): 1e55 -- 10.00 septendecillion +concat('test', number): test55 + +Row 57: +─────── +exp2(number): 72057594037927940 -- 72.06 quadrillion +exp10(number): 1e56 -- 100.00 septendecillion +concat('test', number): test56 + +Row 58: +─────── +exp2(number): 144115188075855870 -- 144.12 quadrillion +exp10(number): 1e57 -- 1.00 octodecillion +concat('test', number): test57 + +Row 59: +─────── +exp2(number): 288230376151711740 -- 288.23 quadrillion +exp10(number): 1e58 -- 10.00 octodecillion +concat('test', number): test58 + +Row 60: +─────── +exp2(number): 576460752303423500 -- 576.46 quadrillion +exp10(number): 1e59 -- 100.00 octodecillion +concat('test', number): test59 + +Row 61: +─────── +exp2(number): 1152921504606847000 -- 1.15 quintillion +exp10(number): 1e60 -- 1000.00 octodecillion +concat('test', number): test60 + +Row 62: +─────── +exp2(number): 2305843009213694000 -- 2.31 quintillion +exp10(number): 1e61 -- 10.00 novemdecillion +concat('test', number): test61 + +Row 63: +─────── +exp2(number): 4611686018427388000 -- 4.61 quintillion +exp10(number): 1e62 -- 100.00 novemdecillion +concat('test', number): test62 + +Row 64: +─────── +exp2(number): 9223372036854776000 -- 9.22 quintillion +exp10(number): 1e63 -- 1.00 vigintillion +concat('test', number): test63 +Row 1: +────── +exp2(number): 1 +exp10(number): 1 +concat('test', number): test0 + +Row 2: +────── +exp2(number): 2 +exp10(number): 10 +concat('test', number): test1 + +Row 3: +────── +exp2(number): 4 +exp10(number): 100 +concat('test', number): test2 + +Row 4: +────── +exp2(number): 8 +exp10(number): 1000 +concat('test', number): test3 + +Row 5: +────── +exp2(number): 16 +exp10(number): 10000 +concat('test', number): test4 + +Row 6: +────── +exp2(number): 32 +exp10(number): 100000 +concat('test', number): test5 + +Row 7: +────── +exp2(number): 64 +exp10(number): 1000000 +concat('test', number): test6 + +Row 8: +────── +exp2(number): 128 +exp10(number): 10000000 +concat('test', number): test7 + +Row 9: +─────── +exp2(number): 256 +exp10(number): 100000000 +concat('test', number): test8 + +Row 10: +─────── +exp2(number): 512 +exp10(number): 1000000000 +concat('test', number): test9 + +Row 11: +─────── +exp2(number): 1024 +exp10(number): 10000000000 +concat('test', number): test10 + +Row 12: +─────── +exp2(number): 2048 +exp10(number): 100000000000 +concat('test', number): test11 + +Row 13: +─────── +exp2(number): 4096 +exp10(number): 1000000000000 +concat('test', number): test12 + +Row 14: +─────── +exp2(number): 8192 +exp10(number): 10000000000000 +concat('test', number): test13 + +Row 15: +─────── +exp2(number): 16384 +exp10(number): 100000000000000 +concat('test', number): test14 + +Row 16: +─────── +exp2(number): 32768 +exp10(number): 1000000000000000 +concat('test', number): test15 + +Row 17: +─────── +exp2(number): 65536 +exp10(number): 10000000000000000 +concat('test', number): test16 + +Row 18: +─────── +exp2(number): 131072 +exp10(number): 100000000000000000 +concat('test', number): test17 + +Row 19: +─────── +exp2(number): 262144 +exp10(number): 1000000000000000000 +concat('test', number): test18 + +Row 20: +─────── +exp2(number): 524288 +exp10(number): 10000000000000000000 +concat('test', number): test19 + +Row 21: +─────── +exp2(number): 1048576 +exp10(number): 100000000000000000000 +concat('test', number): test20 + +Row 22: +─────── +exp2(number): 2097152 +exp10(number): 1e21 +concat('test', number): test21 + +Row 23: +─────── +exp2(number): 4194304 +exp10(number): 1e22 +concat('test', number): test22 + +Row 24: +─────── +exp2(number): 8388608 +exp10(number): 1e23 +concat('test', number): test23 + +Row 25: +─────── +exp2(number): 16777216 +exp10(number): 1e24 +concat('test', number): test24 + +Row 26: +─────── +exp2(number): 33554432 +exp10(number): 1e25 +concat('test', number): test25 + +Row 27: +─────── +exp2(number): 67108864 +exp10(number): 1e26 +concat('test', number): test26 + +Row 28: +─────── +exp2(number): 134217728 +exp10(number): 1e27 +concat('test', number): test27 + +Row 29: +─────── +exp2(number): 268435456 +exp10(number): 1e28 +concat('test', number): test28 + +Row 30: +─────── +exp2(number): 536870912 +exp10(number): 1e29 +concat('test', number): test29 + +Row 31: +─────── +exp2(number): 1073741824 +exp10(number): 1e30 +concat('test', number): test30 + +Row 32: +─────── +exp2(number): 2147483648 +exp10(number): 1e31 +concat('test', number): test31 + +Row 33: +─────── +exp2(number): 4294967296 +exp10(number): 1e32 +concat('test', number): test32 + +Row 34: +─────── +exp2(number): 8589934592 +exp10(number): 1e33 +concat('test', number): test33 + +Row 35: +─────── +exp2(number): 17179869184 +exp10(number): 1e34 +concat('test', number): test34 + +Row 36: +─────── +exp2(number): 34359738368 +exp10(number): 1e35 +concat('test', number): test35 + +Row 37: +─────── +exp2(number): 68719476736 +exp10(number): 1e36 +concat('test', number): test36 + +Row 38: +─────── +exp2(number): 137438953472 +exp10(number): 1e37 +concat('test', number): test37 + +Row 39: +─────── +exp2(number): 274877906944 +exp10(number): 1e38 +concat('test', number): test38 + +Row 40: +─────── +exp2(number): 549755813888 +exp10(number): 1e39 +concat('test', number): test39 + +Row 41: +─────── +exp2(number): 1099511627776 +exp10(number): 1e40 +concat('test', number): test40 + +Row 42: +─────── +exp2(number): 2199023255552 +exp10(number): 1e41 +concat('test', number): test41 + +Row 43: +─────── +exp2(number): 4398046511104 +exp10(number): 1e42 +concat('test', number): test42 + +Row 44: +─────── +exp2(number): 8796093022208 +exp10(number): 1e43 +concat('test', number): test43 + +Row 45: +─────── +exp2(number): 17592186044416 +exp10(number): 1e44 +concat('test', number): test44 + +Row 46: +─────── +exp2(number): 35184372088832 +exp10(number): 1e45 +concat('test', number): test45 + +Row 47: +─────── +exp2(number): 70368744177664 +exp10(number): 1e46 +concat('test', number): test46 + +Row 48: +─────── +exp2(number): 140737488355328 +exp10(number): 1e47 +concat('test', number): test47 + +Row 49: +─────── +exp2(number): 281474976710656 +exp10(number): 1e48 +concat('test', number): test48 + +Row 50: +─────── +exp2(number): 562949953421312 +exp10(number): 1e49 +concat('test', number): test49 + +Row 51: +─────── +exp2(number): 1125899906842624 +exp10(number): 1e50 +concat('test', number): test50 + +Row 52: +─────── +exp2(number): 2251799813685248 +exp10(number): 1e51 +concat('test', number): test51 + +Row 53: +─────── +exp2(number): 4503599627370496 +exp10(number): 1e52 +concat('test', number): test52 + +Row 54: +─────── +exp2(number): 9007199254740992 +exp10(number): 1e53 +concat('test', number): test53 + +Row 55: +─────── +exp2(number): 18014398509481984 +exp10(number): 1e54 +concat('test', number): test54 + +Row 56: +─────── +exp2(number): 36028797018963970 +exp10(number): 1e55 +concat('test', number): test55 + +Row 57: +─────── +exp2(number): 72057594037927940 +exp10(number): 1e56 +concat('test', number): test56 + +Row 58: +─────── +exp2(number): 144115188075855870 +exp10(number): 1e57 +concat('test', number): test57 + +Row 59: +─────── +exp2(number): 288230376151711740 +exp10(number): 1e58 +concat('test', number): test58 + +Row 60: +─────── +exp2(number): 576460752303423500 +exp10(number): 1e59 +concat('test', number): test59 + +Row 61: +─────── +exp2(number): 1152921504606847000 +exp10(number): 1e60 +concat('test', number): test60 + +Row 62: +─────── +exp2(number): 2305843009213694000 +exp10(number): 1e61 +concat('test', number): test61 + +Row 63: +─────── +exp2(number): 4611686018427388000 +exp10(number): 1e62 +concat('test', number): test62 + +Row 64: +─────── +exp2(number): 9223372036854776000 +exp10(number): 1e63 +concat('test', number): test63 diff --git a/tests/queries/0_stateless/03268_vertical_pretty_numbers.sql b/tests/queries/0_stateless/03268_vertical_pretty_numbers.sql new file mode 100644 index 00000000000..0462134ed63 --- /dev/null +++ b/tests/queries/0_stateless/03268_vertical_pretty_numbers.sql @@ -0,0 +1,11 @@ +SET output_format_pretty_color = 1, output_format_pretty_highlight_digit_groups = 1, output_format_pretty_single_large_number_tip_threshold = 1; +SELECT exp2(number), exp10(number), 'test'||number FROM numbers(64) FORMAT Vertical; + +SET output_format_pretty_color = 0, output_format_pretty_highlight_digit_groups = 1, output_format_pretty_single_large_number_tip_threshold = 1; +SELECT exp2(number), exp10(number), 'test'||number FROM numbers(64) FORMAT Vertical; + +SET output_format_pretty_color = 1, output_format_pretty_highlight_digit_groups = 0, output_format_pretty_single_large_number_tip_threshold = 1; +SELECT exp2(number), exp10(number), 'test'||number FROM numbers(64) FORMAT Vertical; + +SET output_format_pretty_color = 0, output_format_pretty_highlight_digit_groups = 0, output_format_pretty_single_large_number_tip_threshold = 0; +SELECT exp2(number), exp10(number), 'test'||number FROM numbers(64) FORMAT Vertical; diff --git a/tests/queries/0_stateless/03269_partition_key_not_in_set.reference b/tests/queries/0_stateless/03269_partition_key_not_in_set.reference new file mode 100644 index 00000000000..1e34df0c77e --- /dev/null +++ b/tests/queries/0_stateless/03269_partition_key_not_in_set.reference @@ -0,0 +1,13 @@ +-- Monotonic function in partition key +48 +48 +-- Non-monotonic function in partition key +48 +48 +-- Multiple partition columns +50 +50 +96 +96 +98 +98 diff --git a/tests/queries/0_stateless/03269_partition_key_not_in_set.sql b/tests/queries/0_stateless/03269_partition_key_not_in_set.sql new file mode 100644 index 00000000000..562521fb7ee --- /dev/null +++ b/tests/queries/0_stateless/03269_partition_key_not_in_set.sql @@ -0,0 +1,81 @@ +-- Related to https://github.com/ClickHouse/ClickHouse/issues/69829 +-- +-- The main goal of the test is to assert that constant transformation +-- for set constant while partition pruning won't be performed +-- if it's not allowed (NOT IN operator case) + +DROP TABLE IF EXISTS 03269_filters; +CREATE TABLE 03269_filters ( + id Int32, + dt Date +) +engine = MergeTree +order by id; + +INSERT INTO 03269_filters +SELECT 6, '2020-01-01' +UNION ALL +SELECT 38, '2021-01-01'; + +SELECT '-- Monotonic function in partition key'; + +DROP TABLE IF EXISTS 03269_single_monotonic; +CREATE TABLE 03269_single_monotonic( + id Int32 +) +ENGINE = MergeTree +PARTITION BY intDiv(id, 10) +ORDER BY id; + +INSERT INTO 03269_single_monotonic SELECT number FROM numbers(50); + +SELECT count() FROM 03269_single_monotonic WHERE id NOT IN (6, 38); +SELECT count() FROM 03269_single_monotonic WHERE id NOT IN ( + SELECT id FROM 03269_filters +); + +DROP TABLE 03269_single_monotonic; + +SELECT '-- Non-monotonic function in partition key'; + +DROP TABLE IF EXISTS 03269_single_non_monotonic; +CREATE TABLE 03269_single_non_monotonic ( + id Int32 +) +ENGINE = MergeTree +PARTITION BY id % 10 +ORDER BY id; + +INSERT INTO 03269_single_non_monotonic SELECT number FROM numbers(50); + +SELECT count() FROM 03269_single_non_monotonic WHERE id NOT IN (6, 38); +SELECT count() FROM 03269_single_non_monotonic WHERE id NOT IN (SELECT id FROM 03269_filters); + +DROP TABLE 03269_single_non_monotonic; + +SELECT '-- Multiple partition columns'; + +DROP TABLE IF EXISTS 03269_multiple_part_cols; +CREATE TABLE 03269_multiple_part_cols ( + id Int32, + dt Date, +) +ENGINE = MergeTree +PARTITION BY (dt, intDiv(id, 10)) +ORDER BY id; + +INSERT INTO 03269_multiple_part_cols +SELECT number, '2020-01-01' FROM numbers(50) +UNION ALL +SELECT number, '2021-01-01' FROM numbers(50); + +SELECT count() FROM 03269_multiple_part_cols WHERE dt NOT IN ('2020-01-01'); +SELECT count() FROM 03269_multiple_part_cols WHERE dt NOT IN (SELECT dt FROM 03269_filters WHERE dt < '2021-01-01'); + +SELECT count() FROM 03269_multiple_part_cols WHERE id NOT IN (6, 38); +SELECT count() FROM 03269_multiple_part_cols WHERE id NOT IN (SELECT id FROM 03269_filters); + +SELECT count() FROM 03269_multiple_part_cols WHERE (id, dt) NOT IN ((6, '2020-01-01'), (38, '2021-01-01')); +SELECT count() FROM 03269_multiple_part_cols WHERE (id, dt) NOT IN (SELECT id, dt FROM 03269_filters); + +DROP TABLE 03269_multiple_part_cols; diff --git a/tests/queries/0_stateless/03270_processors_profile_log_3.reference b/tests/queries/0_stateless/03270_processors_profile_log_3.reference new file mode 100644 index 00000000000..6ed281c757a --- /dev/null +++ b/tests/queries/0_stateless/03270_processors_profile_log_3.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/03270_processors_profile_log_3.sh b/tests/queries/0_stateless/03270_processors_profile_log_3.sh new file mode 100755 index 00000000000..eb86a9f6352 --- /dev/null +++ b/tests/queries/0_stateless/03270_processors_profile_log_3.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +set -e + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE t + ( + a UInt32, + b UInt32 + ) + ENGINE = MergeTree + ORDER BY (a, b); + + INSERT INTO t SELECT number, number FROM numbers(1000); +" + +query_id="03270_processors_profile_log_3_$RANDOM" + +$CLICKHOUSE_CLIENT --query_id="$query_id" -q " + SET log_processors_profiles = 1; + + WITH + t0 AS + ( + SELECT * + FROM numbers(1000) + ), + t1 AS + ( + SELECT number * 3 AS b + FROM t0 + ) + SELECT b * 3 + FROM t + WHERE a IN (t1) + FORMAT Null; +" + +$CLICKHOUSE_CLIENT --query_id="$query_id" -q " + SYSTEM FLUSH LOGS; + + SELECT sum(elapsed_us) > 0 + FROM system.processors_profile_log + WHERE event_date >= yesterday() AND query_id = '$query_id' AND name = 'CreatingSetsTransform'; +" + +##################################################################### + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE t1 + ( + st FixedString(54) + ) + ENGINE = MergeTree + ORDER BY tuple(); + + INSERT INTO t1 VALUES + ('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRTUVWXYZ'), + ('\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0'), + ('IIIIIIIIII\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0'); +" + +query_id="03270_processors_profile_log_3_$RANDOM" + +$CLICKHOUSE_CLIENT --query_id="$query_id" -q " + SET log_processors_profiles = 1; + SET max_threads=2; -- no merging when max_threads=1 + + WITH + ( + SELECT groupConcat(',')(st) + FROM t1 + ORDER BY ALL + ) AS a, + ( + SELECT groupConcat(',')(CAST(st, 'String')) + FROM t1 + ORDER BY ALL + ) AS b + SELECT a = b + FORMAT Null; +" + +$CLICKHOUSE_CLIENT --query_id="$query_id" -q " + SYSTEM FLUSH LOGS; + + SELECT sum(elapsed_us) > 0 + FROM system.processors_profile_log + WHERE event_date >= yesterday() AND query_id = '$query_id' AND name = 'MergingSortedTransform'; +" + diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index ec44a1e1de9..2373a98239a 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -23,3 +23,7 @@ if (ENABLE_UTILS) add_subdirectory (keeper-data-dumper) add_subdirectory (memcpy-bench) endif () + +if (ENABLE_FUZZING AND ENABLE_FUZZER_TEST) + add_subdirectory (libfuzzer-test) +endif () diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index a08143467cd..a0d4d1d349e 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -186,7 +186,6 @@ ComplexKeyCache ComplexKeyDirect ComplexKeyHashed Composable -composable ConcurrencyControlAcquired ConcurrencyControlSoftLimit Config @@ -405,12 +404,12 @@ ITION Identifiant IdentifierQuotingRule IdentifierQuotingStyle -Incrementing -IndexesAreNeighbors -InfluxDB InJodaSyntax InJodaSyntaxOrNull InJodaSyntaxOrZero +Incrementing +IndexesAreNeighbors +InfluxDB Instana IntN Integrations @@ -1475,6 +1474,7 @@ combinator combinators comparising composable +composable compressability concat concatAssumeInjective @@ -1811,6 +1811,7 @@ geocode geohash geohashDecode geohashEncode +geohashes geohashesInBox geoip geospatial @@ -2355,6 +2356,7 @@ parsedatetime parsers partitionID partitionId +partsupp pathFull pclmulqdq pcre diff --git a/utils/libfuzzer-test/CMakeLists.txt b/utils/libfuzzer-test/CMakeLists.txt new file mode 100644 index 00000000000..8765787ff8a --- /dev/null +++ b/utils/libfuzzer-test/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory (test_basic_fuzzer) diff --git a/utils/libfuzzer-test/README.md b/utils/libfuzzer-test/README.md new file mode 100644 index 00000000000..5598cbdb961 --- /dev/null +++ b/utils/libfuzzer-test/README.md @@ -0,0 +1 @@ +This folder contains various stuff intended to test libfuzzer functionality. diff --git a/utils/libfuzzer-test/test_basic_fuzzer/CMakeLists.txt b/utils/libfuzzer-test/test_basic_fuzzer/CMakeLists.txt new file mode 100644 index 00000000000..dc927f35a4b --- /dev/null +++ b/utils/libfuzzer-test/test_basic_fuzzer/CMakeLists.txt @@ -0,0 +1 @@ +add_executable (test_basic_fuzzer main.cpp) diff --git a/utils/libfuzzer-test/test_basic_fuzzer/main.cpp b/utils/libfuzzer-test/test_basic_fuzzer/main.cpp new file mode 100644 index 00000000000..7ccad63273d --- /dev/null +++ b/utils/libfuzzer-test/test_basic_fuzzer/main.cpp @@ -0,0 +1,11 @@ +#include +#include + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + if (size > 0 && data[0] == 'H') + if (size > 1 && data[1] == 'I') + if (size > 2 && data[2] == '!') + __builtin_trap(); + return 0; +} diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 10c55aa4bf5..fab562a8cbb 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,5 +1,7 @@ +v24.10.1.2812-stable 2024-11-01 v24.9.2.42-stable 2024-10-03 v24.9.1.3278-stable 2024-09-26 +v24.8.6.70-lts 2024-11-04 v24.8.5.115-lts 2024-10-08 v24.8.4.13-lts 2024-09-06 v24.8.3.59-lts 2024-09-03 @@ -29,6 +31,7 @@ v24.4.4.113-stable 2024-08-02 v24.4.3.25-stable 2024-06-14 v24.4.2.141-stable 2024-06-07 v24.4.1.2088-stable 2024-05-01 +v24.3.13.40-lts 2024-11-07 v24.3.12.75-lts 2024-10-08 v24.3.11.7-lts 2024-09-06 v24.3.10.33-lts 2024-09-03