diff --git a/CMakeLists.txt b/CMakeLists.txt
index be804a14765..abbc48ab23a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -135,23 +135,21 @@ endif ()
include (cmake/check_flags.cmake)
include (cmake/add_warning.cmake)
-if (COMPILER_CLANG)
- # generate ranges for fast "addr2line" search
- if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
- # NOTE: that clang has a bug because of it does not emit .debug_aranges
- # with ThinLTO, so custom ld.lld wrapper is shipped in docker images.
- set(COMPILER_FLAGS "${COMPILER_FLAGS} -gdwarf-aranges")
- endif ()
-
- # See https://blog.llvm.org/posts/2021-04-05-constructor-homing-for-debug-info/
- if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO")
- set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing")
- set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing")
- endif()
-
- no_warning(enum-constexpr-conversion) # breaks Protobuf in clang-16
+# generate ranges for fast "addr2line" search
+if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
+ # NOTE: that clang has a bug because of it does not emit .debug_aranges
+ # with ThinLTO, so custom ld.lld wrapper is shipped in docker images.
+ set(COMPILER_FLAGS "${COMPILER_FLAGS} -gdwarf-aranges")
endif ()
+# See https://blog.llvm.org/posts/2021-04-05-constructor-homing-for-debug-info/
+if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO")
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing")
+endif()
+
+no_warning(enum-constexpr-conversion) # breaks Protobuf in clang-16
+
option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF)
option(ENABLE_BENCHMARKS "Build all benchmark programs in 'benchmarks' subdirectories" OFF)
@@ -284,16 +282,12 @@ endif ()
option (ENABLE_BUILD_PROFILING "Enable profiling of build time" OFF)
if (ENABLE_BUILD_PROFILING)
- if (COMPILER_CLANG)
- set (COMPILER_FLAGS "${COMPILER_FLAGS} -ftime-trace")
+ set (COMPILER_FLAGS "${COMPILER_FLAGS} -ftime-trace")
- if (LINKER_NAME MATCHES "lld")
- set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--time-trace")
- set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,--time-trace")
- endif ()
- else ()
- message (${RECONFIGURE_MESSAGE_LEVEL} "Build profiling is only available with CLang")
- endif ()
+ if (LINKER_NAME MATCHES "lld")
+ set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--time-trace")
+ set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,--time-trace")
+ endif ()
endif ()
set (CMAKE_CXX_STANDARD 23)
@@ -304,22 +298,20 @@ set (CMAKE_C_STANDARD 11)
set (CMAKE_C_EXTENSIONS ON) # required by most contribs written in C
set (CMAKE_C_STANDARD_REQUIRED ON)
-if (COMPILER_CLANG)
- # Enable C++14 sized global deallocation functions. It should be enabled by setting -std=c++14 but I'm not sure.
- # See https://reviews.llvm.org/D112921
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation")
+# Enable C++14 sized global deallocation functions. It should be enabled by setting -std=c++14 but I'm not sure.
+# See https://reviews.llvm.org/D112921
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation")
- # falign-functions=32 prevents from random performance regressions with the code change. Thus, providing more stable
- # benchmarks.
- set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32")
+# falign-functions=32 prevents from random performance regressions with the code change. Thus, providing more stable
+# benchmarks.
+set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32")
- if (ARCH_AMD64)
- # align branches within a 32-Byte boundary to avoid the potential performance loss when code layout change,
- # which makes benchmark results more stable.
- set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries")
- set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}")
- endif()
-endif ()
+if (ARCH_AMD64)
+ # align branches within a 32-Byte boundary to avoid the potential performance loss when code layout change,
+ # which makes benchmark results more stable.
+ set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries")
+ set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}")
+endif()
# Disable floating-point expression contraction in order to get consistent floating point calculation results across platforms
set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffp-contract=off")
@@ -348,39 +340,34 @@ set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} ${COMPILER_FLAGS} $
set (CMAKE_ASM_FLAGS_RELWITHDEBINFO "${CMAKE_ASM_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
set (CMAKE_ASM_FLAGS_DEBUG "${CMAKE_ASM_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
-if (COMPILER_CLANG)
- if (OS_DARWIN)
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
- set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-U,_inside_main")
- endif()
+if (OS_DARWIN)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
+ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-U,_inside_main")
+endif()
- # Display absolute paths in error messages. Otherwise KDevelop fails to navigate to correct file and opens a new file instead.
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-absolute-paths")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-absolute-paths")
+# Display absolute paths in error messages. Otherwise KDevelop fails to navigate to correct file and opens a new file instead.
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-absolute-paths")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-absolute-paths")
- if (NOT ENABLE_TESTS AND NOT SANITIZE AND NOT SANITIZE_COVERAGE AND OS_LINUX)
- # https://clang.llvm.org/docs/ThinLTO.html
- # Applies to clang and linux only.
- # Disabled when building with tests or sanitizers.
- option(ENABLE_THINLTO "Clang-specific link time optimization" ON)
- endif()
+if (NOT ENABLE_TESTS AND NOT SANITIZE AND NOT SANITIZE_COVERAGE AND OS_LINUX)
+ # https://clang.llvm.org/docs/ThinLTO.html
+ # Applies to clang and linux only.
+ # Disabled when building with tests or sanitizers.
+ option(ENABLE_THINLTO "Clang-specific link time optimization" ON)
+endif()
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstrict-vtable-pointers")
-
- # We cannot afford to use LTO when compiling unit tests, and it's not enough
- # to only supply -fno-lto at the final linking stage. So we disable it
- # completely.
- if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE)
- # Link time optimization
- set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables")
- set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables")
- set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables")
- elseif (ENABLE_THINLTO)
- message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable ThinLTO")
- endif ()
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstrict-vtable-pointers")
+# We cannot afford to use LTO when compiling unit tests, and it's not enough
+# to only supply -fno-lto at the final linking stage. So we disable it
+# completely.
+if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE)
+ # Link time optimization
+ set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables")
+ set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables")
+ set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables")
elseif (ENABLE_THINLTO)
- message (${RECONFIGURE_MESSAGE_LEVEL} "ThinLTO is only available with Clang")
+ message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable ThinLTO")
endif ()
# Turns on all external libs like s3, kafka, ODBC, ...
diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake
index e5ca8e296fc..4a06243243e 100644
--- a/cmake/linux/default_libs.cmake
+++ b/cmake/linux/default_libs.cmake
@@ -5,17 +5,15 @@ set (DEFAULT_LIBS "-nodefaultlibs")
# We need builtins from Clang's RT even without libcxx - for ubsan+int128.
# See https://bugs.llvm.org/show_bug.cgi?id=16404
-if (COMPILER_CLANG)
- execute_process (COMMAND ${CMAKE_CXX_COMPILER} --target=${CMAKE_CXX_COMPILER_TARGET} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process (COMMAND ${CMAKE_CXX_COMPILER} --target=${CMAKE_CXX_COMPILER_TARGET} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
- # Apparently, in clang-19, the UBSan support library for C++ was moved out into ubsan_standalone_cxx.a, so we have to include both.
- if (SANITIZE STREQUAL undefined)
- string(REPLACE "builtins.a" "ubsan_standalone_cxx.a" EXTRA_BUILTINS_LIBRARY "${BUILTINS_LIBRARY}")
- endif ()
+# Apparently, in clang-19, the UBSan support library for C++ was moved out into ubsan_standalone_cxx.a, so we have to include both.
+if (SANITIZE STREQUAL undefined)
+ string(REPLACE "builtins.a" "ubsan_standalone_cxx.a" EXTRA_BUILTINS_LIBRARY "${BUILTINS_LIBRARY}")
+endif ()
- if (NOT EXISTS "${BUILTINS_LIBRARY}")
- set (BUILTINS_LIBRARY "-lgcc")
- endif ()
+if (NOT EXISTS "${BUILTINS_LIBRARY}")
+ set (BUILTINS_LIBRARY "-lgcc")
endif ()
if (OS_ANDROID)
diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake
index a3523203912..08716c1196b 100644
--- a/cmake/sanitize.cmake
+++ b/cmake/sanitize.cmake
@@ -26,9 +26,7 @@ if (SANITIZE)
elseif (SANITIZE STREQUAL "thread")
set (TSAN_FLAGS "-fsanitize=thread")
- if (COMPILER_CLANG)
- set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-ignorelist=${PROJECT_SOURCE_DIR}/tests/tsan_ignorelist.txt")
- endif()
+ set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-ignorelist=${PROJECT_SOURCE_DIR}/tests/tsan_ignorelist.txt")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}")
@@ -44,9 +42,7 @@ if (SANITIZE)
# that's why we often receive reports about UIO. The simplest way to avoid this is just set this flag here.
set(UBSAN_FLAGS "${UBSAN_FLAGS} -fno-sanitize=unsigned-integer-overflow")
endif()
- if (COMPILER_CLANG)
- set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-ignorelist=${PROJECT_SOURCE_DIR}/tests/ubsan_ignorelist.txt")
- endif()
+ set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-ignorelist=${PROJECT_SOURCE_DIR}/tests/ubsan_ignorelist.txt")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${UBSAN_FLAGS}")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${UBSAN_FLAGS}")
diff --git a/cmake/tools.cmake b/cmake/tools.cmake
index 789dc76e6fa..7aa5d4c51ce 100644
--- a/cmake/tools.cmake
+++ b/cmake/tools.cmake
@@ -1,10 +1,6 @@
# Compiler
-if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
- set (COMPILER_CLANG 1) # Safe to treat AppleClang as a regular Clang, in general.
-elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
- set (COMPILER_CLANG 1)
-else ()
+if (NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang")
message (FATAL_ERROR "Compiler ${CMAKE_CXX_COMPILER_ID} is not supported")
endif ()
@@ -17,30 +13,26 @@ set (CLANG_MINIMUM_VERSION 17)
set (XCODE_MINIMUM_VERSION 12.0)
set (APPLE_CLANG_MINIMUM_VERSION 12.0.0)
-if (COMPILER_CLANG)
- if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
- # (Experimental!) Specify "-DALLOW_APPLECLANG=ON" when running CMake configuration step, if you want to experiment with using it.
- if (NOT ALLOW_APPLECLANG AND NOT DEFINED ENV{ALLOW_APPLECLANG})
- message (FATAL_ERROR "Compilation with AppleClang is unsupported. Please use vanilla Clang, e.g. from Homebrew.")
- endif ()
+if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
+ # (Experimental!) Specify "-DALLOW_APPLECLANG=ON" when running CMake configuration step, if you want to experiment with using it.
+ if (NOT ALLOW_APPLECLANG AND NOT DEFINED ENV{ALLOW_APPLECLANG})
+ message (FATAL_ERROR "Compilation with AppleClang is unsupported. Please use vanilla Clang, e.g. from Homebrew.")
+ endif ()
- # For a mapping between XCode / AppleClang / vanilla Clang versions, see https://en.wikipedia.org/wiki/Xcode
- if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${APPLE_CLANG_MINIMUM_VERSION})
- message (FATAL_ERROR "Compilation with AppleClang version ${CMAKE_CXX_COMPILER_VERSION} is unsupported, the minimum required version is ${APPLE_CLANG_MINIMUM_VERSION} (Xcode ${XCODE_MINIMUM_VERSION}).")
- endif ()
- else ()
- if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
- message (FATAL_ERROR "Compilation with Clang version ${CMAKE_CXX_COMPILER_VERSION} is unsupported, the minimum required version is ${CLANG_MINIMUM_VERSION}.")
- endif ()
+ # For a mapping between XCode / AppleClang / vanilla Clang versions, see https://en.wikipedia.org/wiki/Xcode
+ if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${APPLE_CLANG_MINIMUM_VERSION})
+ message (FATAL_ERROR "Compilation with AppleClang version ${CMAKE_CXX_COMPILER_VERSION} is unsupported, the minimum required version is ${APPLE_CLANG_MINIMUM_VERSION} (Xcode ${XCODE_MINIMUM_VERSION}).")
+ endif ()
+else ()
+ if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
+ message (FATAL_ERROR "Compilation with Clang version ${CMAKE_CXX_COMPILER_VERSION} is unsupported, the minimum required version is ${CLANG_MINIMUM_VERSION}.")
endif ()
endif ()
-# Linker
-
string (REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSION})
list (GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR)
-# Example values: `lld-10`
+# Linker
option (LINKER_NAME "Linker name or full path")
if (LINKER_NAME MATCHES "gold")
@@ -48,19 +40,15 @@ if (LINKER_NAME MATCHES "gold")
endif ()
if (NOT LINKER_NAME)
- if (COMPILER_CLANG)
- if (OS_LINUX AND NOT ARCH_S390X)
- find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "ld.lld")
- elseif (OS_DARWIN)
- find_program (LLD_PATH NAMES "ld")
- endif ()
+ if (OS_LINUX AND NOT ARCH_S390X)
+ find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "ld.lld")
+ elseif (OS_DARWIN)
+ find_program (LLD_PATH NAMES "ld")
endif ()
if (LLD_PATH)
if (OS_LINUX OR OS_DARWIN)
- if (COMPILER_CLANG)
- # Clang driver simply allows full linker path.
- set (LINKER_NAME ${LLD_PATH})
- endif ()
+ # Clang driver simply allows full linker path.
+ set (LINKER_NAME ${LLD_PATH})
endif ()
endif()
endif()
@@ -82,47 +70,28 @@ else ()
endif ()
# Archiver
-
-if (COMPILER_CLANG)
- find_program (LLVM_AR_PATH NAMES "llvm-ar-${COMPILER_VERSION_MAJOR}" "llvm-ar")
-endif ()
-
+find_program (LLVM_AR_PATH NAMES "llvm-ar-${COMPILER_VERSION_MAJOR}" "llvm-ar")
if (LLVM_AR_PATH)
set (CMAKE_AR "${LLVM_AR_PATH}")
endif ()
-
message(STATUS "Using archiver: ${CMAKE_AR}")
# Ranlib
-
-if (COMPILER_CLANG)
- find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib-${COMPILER_VERSION_MAJOR}" "llvm-ranlib")
-endif ()
-
+find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib-${COMPILER_VERSION_MAJOR}" "llvm-ranlib")
if (LLVM_RANLIB_PATH)
set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}")
endif ()
-
message(STATUS "Using ranlib: ${CMAKE_RANLIB}")
# Install Name Tool
-
-if (COMPILER_CLANG)
- find_program (LLVM_INSTALL_NAME_TOOL_PATH NAMES "llvm-install-name-tool-${COMPILER_VERSION_MAJOR}" "llvm-install-name-tool")
-endif ()
-
+find_program (LLVM_INSTALL_NAME_TOOL_PATH NAMES "llvm-install-name-tool-${COMPILER_VERSION_MAJOR}" "llvm-install-name-tool")
if (LLVM_INSTALL_NAME_TOOL_PATH)
set (CMAKE_INSTALL_NAME_TOOL "${LLVM_INSTALL_NAME_TOOL_PATH}")
endif ()
-
message(STATUS "Using install-name-tool: ${CMAKE_INSTALL_NAME_TOOL}")
# Objcopy
-
-if (COMPILER_CLANG)
- find_program (OBJCOPY_PATH NAMES "llvm-objcopy-${COMPILER_VERSION_MAJOR}" "llvm-objcopy" "objcopy")
-endif ()
-
+find_program (OBJCOPY_PATH NAMES "llvm-objcopy-${COMPILER_VERSION_MAJOR}" "llvm-objcopy" "objcopy")
if (OBJCOPY_PATH)
message (STATUS "Using objcopy: ${OBJCOPY_PATH}")
else ()
@@ -130,11 +99,7 @@ else ()
endif ()
# Strip
-
-if (COMPILER_CLANG)
- find_program (STRIP_PATH NAMES "llvm-strip-${COMPILER_VERSION_MAJOR}" "llvm-strip" "strip")
-endif ()
-
+find_program (STRIP_PATH NAMES "llvm-strip-${COMPILER_VERSION_MAJOR}" "llvm-strip" "strip")
if (STRIP_PATH)
message (STATUS "Using strip: ${STRIP_PATH}")
else ()
diff --git a/cmake/warnings.cmake b/cmake/warnings.cmake
index 455e4f09939..807d92d9077 100644
--- a/cmake/warnings.cmake
+++ b/cmake/warnings.cmake
@@ -15,37 +15,35 @@ if ((NOT CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") AND (NOT SANITIZE) AND (NOT CMAKE
add_warning(frame-larger-than=65536)
endif ()
-if (COMPILER_CLANG)
- # Add some warnings that are not available even with -Wall -Wextra -Wpedantic.
- # We want to get everything out of the compiler for code quality.
- add_warning(everything)
- add_warning(pedantic)
- no_warning(zero-length-array)
- no_warning(c++98-compat-pedantic)
- no_warning(c++98-compat)
- no_warning(c++20-compat) # Use constinit in C++20 without warnings
- no_warning(sign-conversion)
- no_warning(implicit-int-conversion)
- no_warning(implicit-int-float-conversion)
- no_warning(ctad-maybe-unsupported) # clang 9+, linux-only
- no_warning(disabled-macro-expansion)
- no_warning(documentation-unknown-command)
- no_warning(double-promotion)
- no_warning(exit-time-destructors)
- no_warning(float-equal)
- no_warning(global-constructors)
- no_warning(missing-prototypes)
- no_warning(missing-variable-declarations)
- no_warning(padded)
- no_warning(switch-enum)
- no_warning(undefined-func-template)
- no_warning(unused-template)
- no_warning(vla)
- no_warning(weak-template-vtables)
- no_warning(weak-vtables)
- no_warning(thread-safety-negative) # experimental flag, too many false positives
- no_warning(enum-constexpr-conversion) # breaks magic-enum library in clang-16
- no_warning(unsafe-buffer-usage) # too aggressive
- no_warning(switch-default) # conflicts with "defaults in a switch covering all enum values"
- # TODO Enable conversion, sign-conversion, double-promotion warnings.
-endif ()
+# Add some warnings that are not available even with -Wall -Wextra -Wpedantic.
+# We want to get everything out of the compiler for code quality.
+add_warning(everything)
+add_warning(pedantic)
+no_warning(zero-length-array)
+no_warning(c++98-compat-pedantic)
+no_warning(c++98-compat)
+no_warning(c++20-compat) # Use constinit in C++20 without warnings
+no_warning(sign-conversion)
+no_warning(implicit-int-conversion)
+no_warning(implicit-int-float-conversion)
+no_warning(ctad-maybe-unsupported) # clang 9+, linux-only
+no_warning(disabled-macro-expansion)
+no_warning(documentation-unknown-command)
+no_warning(double-promotion)
+no_warning(exit-time-destructors)
+no_warning(float-equal)
+no_warning(global-constructors)
+no_warning(missing-prototypes)
+no_warning(missing-variable-declarations)
+no_warning(padded)
+no_warning(switch-enum)
+no_warning(undefined-func-template)
+no_warning(unused-template)
+no_warning(vla)
+no_warning(weak-template-vtables)
+no_warning(weak-vtables)
+no_warning(thread-safety-negative) # experimental flag, too many false positives
+no_warning(enum-constexpr-conversion) # breaks magic-enum library in clang-16
+no_warning(unsafe-buffer-usage) # too aggressive
+no_warning(switch-default) # conflicts with "defaults in a switch covering all enum values"
+# TODO Enable conversion, sign-conversion, double-promotion warnings.
diff --git a/contrib/capnproto-cmake/CMakeLists.txt b/contrib/capnproto-cmake/CMakeLists.txt
index e76268592ee..c07e9e6925b 100644
--- a/contrib/capnproto-cmake/CMakeLists.txt
+++ b/contrib/capnproto-cmake/CMakeLists.txt
@@ -81,9 +81,7 @@ set (CAPNPC_SRCS
add_library(_capnpc ${CAPNPC_SRCS})
target_link_libraries(_capnpc PUBLIC _capnp)
-if (COMPILER_CLANG)
- set (CAPNP_PRIVATE_CXX_FLAGS -fno-char8_t)
-endif ()
+set (CAPNP_PRIVATE_CXX_FLAGS -fno-char8_t)
target_compile_options(_kj PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS})
target_compile_options(_capnp PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS})
diff --git a/contrib/openssl-cmake/CMakeLists.txt b/contrib/openssl-cmake/CMakeLists.txt
index 021c88bcb04..72846143b9e 100644
--- a/contrib/openssl-cmake/CMakeLists.txt
+++ b/contrib/openssl-cmake/CMakeLists.txt
@@ -91,12 +91,10 @@ set(LIB_SOVERSION ${VERSION_MAJOR})
enable_language(ASM)
-if(COMPILER_CLANG)
- add_definitions(-Wno-unused-command-line-argument)
- # Note that s390x build uses mold linker
- if(NOT ARCH_S390X)
- set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") # only relevant for -DENABLE_OPENSSL_DYNAMIC=1
- endif()
+add_definitions(-Wno-unused-command-line-argument)
+# Note that s390x build uses mold linker
+if(NOT ARCH_S390X)
+ set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") # only relevant for -DENABLE_OPENSSL_DYNAMIC=1
endif()
if(ARCH_AMD64)
diff --git a/contrib/sentry-native-cmake/CMakeLists.txt b/contrib/sentry-native-cmake/CMakeLists.txt
index 6364e75db28..6e4c8c36081 100644
--- a/contrib/sentry-native-cmake/CMakeLists.txt
+++ b/contrib/sentry-native-cmake/CMakeLists.txt
@@ -1,4 +1,4 @@
-if (NOT OS_FREEBSD AND NOT (OS_DARWIN AND COMPILER_CLANG))
+if (NOT OS_FREEBSD AND NOT OS_DARWIN)
option (ENABLE_SENTRY "Enable Sentry" ${ENABLE_LIBRARIES})
else()
option (ENABLE_SENTRY "Enable Sentry" OFF)
diff --git a/docker/test/base/setup_export_logs.sh b/docker/test/base/setup_export_logs.sh
index 8858e12c50e..0c869a95db2 100755
--- a/docker/test/base/setup_export_logs.sh
+++ b/docker/test/base/setup_export_logs.sh
@@ -127,9 +127,6 @@ function setup_logs_replication
echo 'Create all configured system logs'
clickhouse-client --query "SYSTEM FLUSH LOGS"
- # It's doesn't make sense to try creating tables if SYNC fails
- echo "SYSTEM SYNC DATABASE REPLICA default" | clickhouse-client "${CONNECTION_ARGS[@]}" || return 0
-
debug_or_sanitizer_build=$(clickhouse-client -q "WITH ((SELECT value FROM system.build_options WHERE name='BUILD_TYPE') AS build, (SELECT value FROM system.build_options WHERE name='CXX_FLAGS') as flags) SELECT build='Debug' OR flags LIKE '%fsanitize%'")
echo "Build is debug or sanitizer: $debug_or_sanitizer_build"
@@ -143,7 +140,7 @@ function setup_logs_replication
time DateTime COMMENT 'The time of test run',
test_name String COMMENT 'The name of the test',
coverage Array(UInt64) COMMENT 'An array of addresses of the code (a subset of addresses instrumented for coverage) that were encountered during the test run'
- ) ENGINE = Null COMMENT 'Contains information about per-test coverage from the CI, but used only for exporting to the CI cluster'
+ ) ENGINE = MergeTree ORDER BY test_name COMMENT 'Contains information about per-test coverage from the CI, but used only for exporting to the CI cluster'
"
# For each system log table:
diff --git a/docker/test/fuzzer/query-fuzzer-tweaks-users.xml b/docker/test/fuzzer/query-fuzzer-tweaks-users.xml
index c31d2fd7f39..ad261be1abe 100644
--- a/docker/test/fuzzer/query-fuzzer-tweaks-users.xml
+++ b/docker/test/fuzzer/query-fuzzer-tweaks-users.xml
@@ -31,6 +31,11 @@
+
+
+
+
+
diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile
index f02a628d13e..c3d80a7334b 100644
--- a/docker/test/stateless/Dockerfile
+++ b/docker/test/stateless/Dockerfile
@@ -89,8 +89,8 @@ ENV MINIO_ROOT_USER="clickhouse"
ENV MINIO_ROOT_PASSWORD="clickhouse"
ENV EXPORT_S3_STORAGE_POLICIES=1
-RUN npm install -g azurite \
- && npm install -g tslib
+RUN npm install -g azurite@3.30.0 \
+ && npm install -g tslib && npm install -g node
COPY run.sh /
COPY setup_minio.sh /
diff --git a/docker/test/stateless/stress_tests.lib b/docker/test/stateless/stress_tests.lib
index a0865f638a1..6aaddbfe590 100644
--- a/docker/test/stateless/stress_tests.lib
+++ b/docker/test/stateless/stress_tests.lib
@@ -120,13 +120,41 @@ EOL
local max_users_mem
max_users_mem=$((total_mem*30/100)) # 30%
+ # Similar to docker/test/fuzzer/query-fuzzer-tweaks-users.xml
echo "Setting max_memory_usage_for_user=$max_users_mem and max_memory_usage for queries to 10G"
- cat > /etc/clickhouse-server/users.d/max_memory_usage_for_user.xml < /etc/clickhouse-server/users.d/stress_test_tweaks-users.xml <
+ 6010G${max_users_mem}
+
+ 200
+
+
+
+ 60
+
+
+
+ 10G
+
+
+
+ 200
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/docs/en/development/build-cross-s390x.md b/docs/en/development/build-cross-s390x.md
index 3c1667350e1..4c111a76d82 100644
--- a/docs/en/development/build-cross-s390x.md
+++ b/docs/en/development/build-cross-s390x.md
@@ -5,22 +5,13 @@ title: How to Build, Run and Debug ClickHouse on Linux for s390x (zLinux)
sidebar_label: Build on Linux for s390x (zLinux)
---
-As of writing (2023/3/10) building for s390x considered to be experimental. Not all features can be enabled, has broken features and is currently under active development.
+At the time of writing (2024 May), support for the s390x platform is considered experimental, i.e. some features are disabled or broken on s390x.
+## Building ClickHouse for s390x
-## Building
-
-s390x has two OpenSSL-related build options.
-- By default, the s390x build will dynamically link to OpenSSL libraries. It will build OpenSSL shared objects, so it's not necessary to install OpenSSL beforehand. (This option is recommended in all cases.)
-- Another option is to build OpenSSL in-tree. In this case two build flags need to be supplied to cmake
-```bash
--DENABLE_OPENSSL_DYNAMIC=0
-```
-
-:::note
-s390x builds are temporarily disabled in CI.
-:::
-
+s390x has two OpenSSL-related build options:
+- By default, OpenSSL is build on s390x as a shared library. This is different from all other platforms, where OpenSSL is build as static library.
+- To build OpenSSL as a static library regardless, pass `-DENABLE_OPENSSL_DYNAMIC=0` to CMake.
These instructions assume that the host machine is x86_64 and has all the tooling required to build natively based on the [build instructions](../development/build.md). It also assumes that the host is Ubuntu 22.04 but the following instructions should also work on Ubuntu 20.04.
@@ -31,11 +22,16 @@ apt-get install binutils-s390x-linux-gnu libc6-dev-s390x-cross gcc-s390x-linux-g
```
If you wish to cross compile rust code install the rust cross compile target for s390x:
+
```bash
rustup target add s390x-unknown-linux-gnu
```
+The s390x build uses the mold linker, download it from https://github.com/rui314/mold/releases/download/v2.0.0/mold-2.0.0-x86_64-linux.tar.gz
+and place it into your `$PATH`.
+
To build for s390x:
+
```bash
cmake -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-s390x.cmake ..
ninja
diff --git a/docs/en/engines/table-engines/integrations/rabbitmq.md b/docs/en/engines/table-engines/integrations/rabbitmq.md
index a4d0cf78066..a8315a5ad9e 100644
--- a/docs/en/engines/table-engines/integrations/rabbitmq.md
+++ b/docs/en/engines/table-engines/integrations/rabbitmq.md
@@ -73,6 +73,7 @@ Optional parameters:
- `rabbitmq_queue_consume` - Use user-defined queues and do not make any RabbitMQ setup: declaring exchanges, queues, bindings. Default: `false`.
- `rabbitmq_username` - RabbitMQ username.
- `rabbitmq_password` - RabbitMQ password.
+- `reject_unhandled_messages` - Reject messages (send RabbitMQ negative acknowledgement) in case of errors. This setting is automatically enabled if there is a `x-dead-letter-exchange` defined in `rabbitmq_queue_settings_list`.
- `rabbitmq_commit_on_select` - Commit messages when select query is made. Default: `false`.
- `rabbitmq_max_rows_per_message` — The maximum number of rows written in one RabbitMQ message for row-based formats. Default : `1`.
- `rabbitmq_empty_queue_backoff_start` — A start backoff point to reschedule read if the rabbitmq queue is empty.
diff --git a/docs/en/engines/table-engines/mergetree-family/annindexes.md b/docs/en/engines/table-engines/mergetree-family/annindexes.md
index 78a27d3ff86..5a81313f62e 100644
--- a/docs/en/engines/table-engines/mergetree-family/annindexes.md
+++ b/docs/en/engines/table-engines/mergetree-family/annindexes.md
@@ -22,9 +22,8 @@ ORDER BY Distance(vectors, Point)
LIMIT N
```
-`vectors` contains N-dimensional values of type [Array](../../../sql-reference/data-types/array.md) or
-[Tuple](../../../sql-reference/data-types/tuple.md), for example embeddings. Function `Distance` computes the distance between two vectors.
-Often, the Euclidean (L2) distance is chosen as distance function but [other
+`vectors` contains N-dimensional values of type [Array(Float32)](../../../sql-reference/data-types/array.md), for example embeddings.
+Function `Distance` computes the distance between two vectors. Often, the Euclidean (L2) distance is chosen as distance function but [other
distance functions](/docs/en/sql-reference/functions/distance-functions.md) are also possible. `Point` is the reference point, e.g. `(0.17,
0.33, ...)`, and `N` limits the number of search results.
@@ -47,7 +46,7 @@ of the search space (using clustering, search trees, etc.) which allows to compu
# Creating and Using ANN Indexes {#creating_using_ann_indexes}
-Syntax to create an ANN index over an [Array](../../../sql-reference/data-types/array.md) column:
+Syntax to create an ANN index over an [Array(Float32)](../../../sql-reference/data-types/array.md) column:
```sql
CREATE TABLE table_with_ann_index
@@ -60,19 +59,6 @@ ENGINE = MergeTree
ORDER BY id;
```
-Syntax to create an ANN index over a [Tuple](../../../sql-reference/data-types/tuple.md) column:
-
-```sql
-CREATE TABLE table_with_ann_index
-(
- `id` Int64,
- `vectors` Tuple(Float32[, Float32[, ...]]),
- INDEX [ann_index_name] vectors TYPE [ann_index_type]([ann_index_parameters]) [GRANULARITY [N]]
-)
-ENGINE = MergeTree
-ORDER BY id;
-```
-
ANN indexes are built during column insertion and merge. As a result, `INSERT` and `OPTIMIZE` statements will be slower than for ordinary
tables. ANNIndexes are ideally used only with immutable or rarely changed data, respectively when are far more read requests than write
requests.
@@ -164,7 +150,7 @@ linear surfaces (lines in 2D, planes in 3D etc.).
-Syntax to create an Annoy index over an [Array](../../../sql-reference/data-types/array.md) column:
+Syntax to create an Annoy index over an [Array(Float32)](../../../sql-reference/data-types/array.md) column:
```sql
CREATE TABLE table_with_annoy_index
@@ -177,19 +163,6 @@ ENGINE = MergeTree
ORDER BY id;
```
-Syntax to create an ANN index over a [Tuple](../../../sql-reference/data-types/tuple.md) column:
-
-```sql
-CREATE TABLE table_with_annoy_index
-(
- id Int64,
- vectors Tuple(Float32[, Float32[, ...]]),
- INDEX [ann_index_name] vectors TYPE annoy([Distance[, NumTrees]]) [GRANULARITY N]
-)
-ENGINE = MergeTree
-ORDER BY id;
-```
-
Annoy currently supports two distance functions:
- `L2Distance`, also called Euclidean distance, is the length of a line segment between two points in Euclidean space
([Wikipedia](https://en.wikipedia.org/wiki/Euclidean_distance)).
@@ -203,10 +176,9 @@ Parameter `NumTrees` is the number of trees which the algorithm creates (default
more accurate search results but slower index creation / query times (approximately linearly) as well as larger index sizes.
:::note
-Indexes over columns of type `Array` will generally work faster than indexes on `Tuple` columns. All arrays must have same length. To avoid
-errors, you can use a [CONSTRAINT](/docs/en/sql-reference/statements/create/table.md#constraints), for example, `CONSTRAINT
-constraint_name_1 CHECK length(vectors) = 256`. Also, empty `Arrays` and unspecified `Array` values in INSERT statements (i.e. default
-values) are not supported.
+All arrays must have same length. To avoid errors, you can use a
+[CONSTRAINT](/docs/en/sql-reference/statements/create/table.md#constraints), for example, `CONSTRAINT constraint_name_1 CHECK
+length(vectors) = 256`. Also, empty `Arrays` and unspecified `Array` values in INSERT statements (i.e. default values) are not supported.
:::
The creation of Annoy indexes (whenever a new part is build, e.g. at the end of a merge) is a relatively slow process. You can increase
@@ -264,19 +236,6 @@ ENGINE = MergeTree
ORDER BY id;
```
-Syntax to create an ANN index over a [Tuple](../../../sql-reference/data-types/tuple.md) column:
-
-```sql
-CREATE TABLE table_with_usearch_index
-(
- id Int64,
- vectors Tuple(Float32[, Float32[, ...]]),
- INDEX [ann_index_name] vectors TYPE usearch([Distance[, ScalarKind]]) [GRANULARITY N]
-)
-ENGINE = MergeTree
-ORDER BY id;
-```
-
USearch currently supports two distance functions:
- `L2Distance`, also called Euclidean distance, is the length of a line segment between two points in Euclidean space
([Wikipedia](https://en.wikipedia.org/wiki/Euclidean_distance)).
diff --git a/docs/en/engines/table-engines/mergetree-family/invertedindexes.md b/docs/en/engines/table-engines/mergetree-family/invertedindexes.md
index 9374f6a3ac1..f58a06464b2 100644
--- a/docs/en/engines/table-engines/mergetree-family/invertedindexes.md
+++ b/docs/en/engines/table-engines/mergetree-family/invertedindexes.md
@@ -2,7 +2,7 @@
slug: /en/engines/table-engines/mergetree-family/invertedindexes
sidebar_label: Full-text Indexes
description: Quickly find search terms in text.
-keywords: [full-text search, text search, inverted, index, indices]
+keywords: [full-text search, text search, index, indices]
---
# Full-text Search using Full-text Indexes [experimental]
@@ -53,6 +53,10 @@ ENGINE = MergeTree
ORDER BY key
```
+:::note
+In earlier versions of ClickHouse, the corresponding index type name was `inverted`.
+:::
+
where `N` specifies the tokenizer:
- `full_text(0)` (or shorter: `full_text()`) set the tokenizer to "tokens", i.e. split strings along spaces,
diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md
index 886c29e755e..7862eef69f8 100644
--- a/docs/en/engines/table-engines/mergetree-family/mergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md
@@ -494,7 +494,7 @@ Syntax: `tokenbf_v1(size_of_bloom_filter_in_bytes, number_of_hash_functions, ran
#### Special-purpose
- Experimental indexes to support approximate nearest neighbor (ANN) search. See [here](annindexes.md) for details.
-- An experimental inverted index to support full-text search. See [here](invertedindexes.md) for details.
+- An experimental full-text index to support full-text search. See [here](invertedindexes.md) for details.
### Functions Support {#functions-support}
@@ -502,31 +502,31 @@ Conditions in the `WHERE` clause contains calls of the functions that operate wi
Indexes of type `set` can be utilized by all functions. The other index types are supported as follows:
-| Function (operator) / Index | primary key | minmax | ngrambf_v1 | tokenbf_v1 | bloom_filter | inverted |
-|------------------------------------------------------------------------------------------------------------|-------------|--------|------------|------------|--------------|----------|
-| [equals (=, ==)](/docs/en/sql-reference/functions/comparison-functions.md/#equals) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ |
-| [notEquals(!=, <>)](/docs/en/sql-reference/functions/comparison-functions.md/#notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ |
-| [like](/docs/en/sql-reference/functions/string-search-functions.md/#like) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ |
-| [notLike](/docs/en/sql-reference/functions/string-search-functions.md/#notlike) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ |
-| [match](/docs/en/sql-reference/functions/string-search-functions.md/#match) | ✗ | ✗ | ✔ | ✔ | ✗ | ✔ |
-| [startsWith](/docs/en/sql-reference/functions/string-functions.md/#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ |
-| [endsWith](/docs/en/sql-reference/functions/string-functions.md/#endswith) | ✗ | ✗ | ✔ | ✔ | ✗ | ✔ |
-| [multiSearchAny](/docs/en/sql-reference/functions/string-search-functions.md/#multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | ✔ |
-| [in](/docs/en/sql-reference/functions/in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ |
-| [notIn](/docs/en/sql-reference/functions/in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ |
-| [less (<)](/docs/en/sql-reference/functions/comparison-functions.md/#less) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
-| [greater (>)](/docs/en/sql-reference/functions/comparison-functions.md/#greater) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
-| [lessOrEquals (<=)](/docs/en/sql-reference/functions/comparison-functions.md/#lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
-| [greaterOrEquals (>=)](/docs/en/sql-reference/functions/comparison-functions.md/#greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
-| [empty](/docs/en/sql-reference/functions/array-functions/#empty) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
-| [notEmpty](/docs/en/sql-reference/functions/array-functions/#notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
-| [has](/docs/en/sql-reference/functions/array-functions/#has) | ✗ | ✗ | ✔ | ✔ | ✔ | ✔ |
-| [hasAny](/docs/en/sql-reference/functions/array-functions/#hasany) | ✗ | ✗ | ✔ | ✔ | ✔ | ✗ |
-| [hasAll](/docs/en/sql-reference/functions/array-functions/#hasall) | ✗ | ✗ | ✗ | ✗ | ✔ | ✗ |
-| hasToken | ✗ | ✗ | ✗ | ✔ | ✗ | ✔ |
-| hasTokenOrNull | ✗ | ✗ | ✗ | ✔ | ✗ | ✔ |
-| hasTokenCaseInsensitive (*) | ✗ | ✗ | ✗ | ✔ | ✗ | ✗ |
-| hasTokenCaseInsensitiveOrNull (*) | ✗ | ✗ | ✗ | ✔ | ✗ | ✗ |
+| Function (operator) / Index | primary key | minmax | ngrambf_v1 | tokenbf_v1 | bloom_filter | full_text |
+|------------------------------------------------------------------------------------------------------------|-------------|--------|------------|------------|--------------|-----------|
+| [equals (=, ==)](/docs/en/sql-reference/functions/comparison-functions.md/#equals) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ |
+| [notEquals(!=, <>)](/docs/en/sql-reference/functions/comparison-functions.md/#notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ |
+| [like](/docs/en/sql-reference/functions/string-search-functions.md/#like) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ |
+| [notLike](/docs/en/sql-reference/functions/string-search-functions.md/#notlike) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ |
+| [match](/docs/en/sql-reference/functions/string-search-functions.md/#match) | ✗ | ✗ | ✔ | ✔ | ✗ | ✔ |
+| [startsWith](/docs/en/sql-reference/functions/string-functions.md/#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ |
+| [endsWith](/docs/en/sql-reference/functions/string-functions.md/#endswith) | ✗ | ✗ | ✔ | ✔ | ✗ | ✔ |
+| [multiSearchAny](/docs/en/sql-reference/functions/string-search-functions.md/#multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | ✔ |
+| [in](/docs/en/sql-reference/functions/in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ |
+| [notIn](/docs/en/sql-reference/functions/in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ |
+| [less (<)](/docs/en/sql-reference/functions/comparison-functions.md/#less) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
+| [greater (>)](/docs/en/sql-reference/functions/comparison-functions.md/#greater) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
+| [lessOrEquals (<=)](/docs/en/sql-reference/functions/comparison-functions.md/#lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
+| [greaterOrEquals (>=)](/docs/en/sql-reference/functions/comparison-functions.md/#greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
+| [empty](/docs/en/sql-reference/functions/array-functions/#empty) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
+| [notEmpty](/docs/en/sql-reference/functions/array-functions/#notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
+| [has](/docs/en/sql-reference/functions/array-functions/#has) | ✗ | ✗ | ✔ | ✔ | ✔ | ✔ |
+| [hasAny](/docs/en/sql-reference/functions/array-functions/#hasany) | ✗ | ✗ | ✔ | ✔ | ✔ | ✗ |
+| [hasAll](/docs/en/sql-reference/functions/array-functions/#hasall) | ✗ | ✗ | ✗ | ✗ | ✔ | ✗ |
+| hasToken | ✗ | ✗ | ✗ | ✔ | ✗ | ✔ |
+| hasTokenOrNull | ✗ | ✗ | ✗ | ✔ | ✗ | ✔ |
+| hasTokenCaseInsensitive (*) | ✗ | ✗ | ✗ | ✔ | ✗ | ✗ |
+| hasTokenCaseInsensitiveOrNull (*) | ✗ | ✗ | ✗ | ✔ | ✗ | ✗ |
Functions with a constant argument that is less than ngram size can’t be used by `ngrambf_v1` for query optimization.
diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md
index e66dfac4db7..d25e18bd397 100644
--- a/docs/en/interfaces/formats.md
+++ b/docs/en/interfaces/formats.md
@@ -207,7 +207,7 @@ SELECT * FROM nestedt FORMAT TSV
Differs from `TabSeparated` format in that the rows are written without escaping.
When parsing with this format, tabs or linefeeds are not allowed in each field.
-This format is also available under the name `TSVRaw`.
+This format is also available under the names `TSVRaw`, `Raw`.
## TabSeparatedWithNames {#tabseparatedwithnames}
@@ -242,14 +242,14 @@ This format is also available under the name `TSVWithNamesAndTypes`.
Differs from `TabSeparatedWithNames` format in that the rows are written without escaping.
When parsing with this format, tabs or linefeeds are not allowed in each field.
-This format is also available under the name `TSVRawWithNames`.
+This format is also available under the names `TSVRawWithNames`, `RawWithNames`.
## TabSeparatedRawWithNamesAndTypes {#tabseparatedrawwithnamesandtypes}
Differs from `TabSeparatedWithNamesAndTypes` format in that the rows are written without escaping.
When parsing with this format, tabs or linefeeds are not allowed in each field.
-This format is also available under the name `TSVRawWithNamesAndNames`.
+This format is also available under the names `TSVRawWithNamesAndNames`, `RawWithNamesAndNames`.
## Template {#format-template}
diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md
index be2d028e87f..bb326dc5242 100644
--- a/docs/en/interfaces/third-party/gui.md
+++ b/docs/en/interfaces/third-party/gui.md
@@ -210,6 +210,18 @@ Features:
- Pre-built metrics dashboards.
- Multiple users/projects via YAML config.
+### clickhouse-monitoring {#clickhouse-monitoring}
+
+[clickhouse-monitoring](https://github.com/duyet/clickhouse-monitoring) is a simple Next.js dashboard that relies on `system.*` tables to help monitor and provide an overview of your ClickHouse cluster.
+
+Features:
+
+- Query monitor: current queries, query history, query resources (memory, parts read, file_open, ...), most expensive queries, most used tables or columns, etc.
+- Cluster monitor: total memory/CPU usage, distributed queue, global settings, mergetree settings, metrics, etc.
+- Tables and parts information: size, row count, compression, part size, etc., at the column level detail.
+- Useful tools: Zookeeper data exploration, query EXPLAIN, kill queries, etc.
+- Visualization metric charts: queries and resource usage, number of merges/mutation, merge performance, query performance, etc.
+
## Commercial {#commercial}
### DataGrip {#datagrip}
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 4f158a4fd6e..91b544c6a82 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -1925,7 +1925,9 @@ Default value: `16`.
### wait_for_async_insert {#wait-for-async-insert}
-Enables or disables waiting for processing of asynchronous insertion. If enabled, server will return `OK` only after the data is inserted. Otherwise, it will return `OK` even if the data wasn't inserted.
+Enables or disables waiting for processing of asynchronous insertion. If enabled, server will return `OK` only after the data is inserted. Otherwise, it will return `OK` as soon it has received the data, but it might still fail to parse or insert it later (You can check in system.asynchronous_insert_log)
+
+If you want to use asynchronous inserts, we need to also enable [`async_insert`](#async-insert).
Possible values:
diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md
index 389c917d427..7005783dd60 100644
--- a/docs/en/operations/storing-data.md
+++ b/docs/en/operations/storing-data.md
@@ -371,6 +371,8 @@ is equal to
```
+Starting from `24.5` it is possible configure any object storage disk (`s3`, `azure`, `local`) using `plain_rewritable` metadata type.
+
### Using Azure Blob Storage {#azure-blob-storage}
`MergeTree` family table engines can store data to [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/) using a disk with type `azure_blob_storage`.
diff --git a/docs/en/operations/system-tables/part_log.md b/docs/en/operations/system-tables/part_log.md
index af582646653..c0caea1ce5e 100644
--- a/docs/en/operations/system-tables/part_log.md
+++ b/docs/en/operations/system-tables/part_log.md
@@ -14,7 +14,7 @@ The `system.part_log` table contains the following columns:
- `event_type` ([Enum8](../../sql-reference/data-types/enum.md)) — Type of the event that occurred with the data part. Can have one of the following values:
- `NewPart` — Inserting of a new data part.
- `MergeParts` — Merging of data parts.
- - `DownloadParts` — Downloading a data part.
+ - `DownloadPart` — Downloading a data part.
- `RemovePart` — Removing or detaching a data part using [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition).
- `MutatePart` — Mutating of a data part.
- `MovePart` — Moving the data part from the one disk to another one.
diff --git a/docs/en/sql-reference/data-types/map.md b/docs/en/sql-reference/data-types/map.md
index 2c734969afc..9d495126d28 100644
--- a/docs/en/sql-reference/data-types/map.md
+++ b/docs/en/sql-reference/data-types/map.md
@@ -10,7 +10,7 @@ sidebar_label: Map(K, V)
**Parameters**
-- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md), [LowCardinality](../../sql-reference/data-types/lowcardinality.md), [FixedString](../../sql-reference/data-types/fixedstring.md), [UUID](../../sql-reference/data-types/uuid.md), [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md), [Date32](../../sql-reference/data-types/date32.md), [Enum](../../sql-reference/data-types/enum.md).
+- `key` — The key part of the pair. Arbitrary type, except [Nullable](../../sql-reference/data-types/nullable.md) and [LowCardinality](../../sql-reference/data-types/lowcardinality.md) nested with [Nullable](../../sql-reference/data-types/nullable.md) types.
- `value` — The value part of the pair. Arbitrary type, including [Map](../../sql-reference/data-types/map.md) and [Array](../../sql-reference/data-types/array.md).
To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. This lookup works now with a linear complexity.
diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md
index ed98f986139..2fc64195eb7 100644
--- a/docs/en/sql-reference/functions/date-time-functions.md
+++ b/docs/en/sql-reference/functions/date-time-functions.md
@@ -1417,31 +1417,31 @@ toStartOfFifteenMinutes(toDateTime('2023-04-21 10:23:00')): 2023-04-21 10:15:00
This function generalizes other `toStartOf*()` functions with `toStartOfInterval(date_or_date_with_time, INTERVAL x unit [, time_zone])` syntax.
For example,
-- `toStartOfInterval(t, INTERVAL 1 year)` returns the same as `toStartOfYear(t)`,
-- `toStartOfInterval(t, INTERVAL 1 month)` returns the same as `toStartOfMonth(t)`,
-- `toStartOfInterval(t, INTERVAL 1 day)` returns the same as `toStartOfDay(t)`,
-- `toStartOfInterval(t, INTERVAL 15 minute)` returns the same as `toStartOfFifteenMinutes(t)`.
+- `toStartOfInterval(t, INTERVAL 1 YEAR)` returns the same as `toStartOfYear(t)`,
+- `toStartOfInterval(t, INTERVAL 1 MONTH)` returns the same as `toStartOfMonth(t)`,
+- `toStartOfInterval(t, INTERVAL 1 DAY)` returns the same as `toStartOfDay(t)`,
+- `toStartOfInterval(t, INTERVAL 15 MINUTE)` returns the same as `toStartOfFifteenMinutes(t)`.
The calculation is performed relative to specific points in time:
| Interval | Start |
|-------------|------------------------|
-| year | year 0 |
-| quarter | 1900 Q1 |
-| month | 1900 January |
-| week | 1970, 1st week (01-05) |
-| day | 1970-01-01 |
-| hour | (*) |
-| minute | 1970-01-01 00:00:00 |
-| second | 1970-01-01 00:00:00 |
-| millisecond | 1970-01-01 00:00:00 |
-| microsecond | 1970-01-01 00:00:00 |
-| nanosecond | 1970-01-01 00:00:00 |
+| YEAR | year 0 |
+| QUARTER | 1900 Q1 |
+| MONTH | 1900 January |
+| WEEK | 1970, 1st week (01-05) |
+| DAY | 1970-01-01 |
+| HOUR | (*) |
+| MINUTE | 1970-01-01 00:00:00 |
+| SECOND | 1970-01-01 00:00:00 |
+| MILLISECOND | 1970-01-01 00:00:00 |
+| MICROSECOND | 1970-01-01 00:00:00 |
+| NANOSECOND | 1970-01-01 00:00:00 |
(*) hour intervals are special: the calculation is always performed relative to 00:00:00 (midnight) of the current day. As a result, only
hour values between 1 and 23 are useful.
-If unit `week` was specified, `toStartOfInterval` assumes that weeks start on Monday. Note that this behavior is different from that of function `toStartOfWeek` in which weeks start by default on Sunday.
+If unit `WEEK` was specified, `toStartOfInterval` assumes that weeks start on Monday. Note that this behavior is different from that of function `toStartOfWeek` in which weeks start by default on Sunday.
**See Also**
@@ -2581,13 +2581,27 @@ SELECT
└─────────────────────┴──────────────────────────┴─────────────────────────────────┘
```
-## subtractYears, subtractQuarters, subtractMonths, subtractWeeks, subtractDays, subtractHours, subtractMinutes, subtractSeconds, subtractMilliseconds, subtractMicroseconds, subtractNanoseconds
+## subtractYears
-These functions subtract units of the interval specified by the function name from a date, a date with time or a string-encoded date / date with time. A date or date with time is returned.
+Subtracts a specified number of years from a date, a date with time or a string-encoded date / date with time.
-Example:
+**Syntax**
-``` sql
+```sql
+subtractYears(date, num)
+```
+
+**Parameters**
+
+- `date`: Date / date with time to subtract specified number of years from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md).
+- `num`: Number of years to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md).
+
+**Returned value**
+- Returns `date` minus `num` years. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md).
+
+**Example**
+
+```sql
WITH
toDate('2024-01-01') AS date,
toDateTime('2024-01-01 00:00:00') AS date_time,
@@ -2598,12 +2612,456 @@ SELECT
subtractYears(date_time_string, 1) AS subtract_years_with_date_time_string
```
-``` text
+```response
┌─subtract_years_with_date─┬─subtract_years_with_date_time─┬─subtract_years_with_date_time_string─┐
│ 2023-01-01 │ 2023-01-01 00:00:00 │ 2023-01-01 00:00:00.000 │
└──────────────────────────┴───────────────────────────────┴──────────────────────────────────────┘
```
+## subtractQuarters
+
+Subtracts a specified number of quarters from a date, a date with time or a string-encoded date / date with time.
+
+**Syntax**
+
+```sql
+subtractQuarters(date, num)
+```
+
+**Parameters**
+
+- `date`: Date / date with time to subtract specified number of quarters from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md).
+- `num`: Number of quarters to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md).
+
+**Returned value**
+- Returns `date` minus `num` quarters. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md).
+
+**Example**
+
+```sql
+WITH
+ toDate('2024-01-01') AS date,
+ toDateTime('2024-01-01 00:00:00') AS date_time,
+ '2024-01-01 00:00:00' AS date_time_string
+SELECT
+ subtractQuarters(date, 1) AS subtract_quarters_with_date,
+ subtractQuarters(date_time, 1) AS subtract_quarters_with_date_time,
+ subtractQuarters(date_time_string, 1) AS subtract_quarters_with_date_time_string
+```
+
+```response
+┌─subtract_quarters_with_date─┬─subtract_quarters_with_date_time─┬─subtract_quarters_with_date_time_string─┐
+│ 2023-10-01 │ 2023-10-01 00:00:00 │ 2023-10-01 00:00:00.000 │
+└─────────────────────────────┴──────────────────────────────────┴─────────────────────────────────────────┘
+```
+
+## subtractMonths
+
+Subtracts a specified number of months from a date, a date with time or a string-encoded date / date with time.
+
+**Syntax**
+
+```sql
+subtractMonths(date, num)
+```
+
+**Parameters**
+
+- `date`: Date / date with time to subtract specified number of months from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md).
+- `num`: Number of months to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md).
+
+**Returned value**
+- Returns `date` minus `num` months. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md).
+
+**Example**
+
+```sql
+WITH
+ toDate('2024-01-01') AS date,
+ toDateTime('2024-01-01 00:00:00') AS date_time,
+ '2024-01-01 00:00:00' AS date_time_string
+SELECT
+ subtractMonths(date, 1) AS subtract_months_with_date,
+ subtractMonths(date_time, 1) AS subtract_months_with_date_time,
+ subtractMonths(date_time_string, 1) AS subtract_months_with_date_time_string
+```
+
+```response
+┌─subtract_months_with_date─┬─subtract_months_with_date_time─┬─subtract_months_with_date_time_string─┐
+│ 2023-12-01 │ 2023-12-01 00:00:00 │ 2023-12-01 00:00:00.000 │
+└───────────────────────────┴────────────────────────────────┴───────────────────────────────────────┘
+```
+
+## subtractWeeks
+
+Subtracts a specified number of weeks from a date, a date with time or a string-encoded date / date with time.
+
+**Syntax**
+
+```sql
+subtractWeeks(date, num)
+```
+
+**Parameters**
+
+- `date`: Date / date with time to subtract specified number of weeks from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md).
+- `num`: Number of weeks to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md).
+
+**Returned value**
+- Returns `date` minus `num` weeks. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md).
+
+**Example**
+
+```sql
+WITH
+ toDate('2024-01-01') AS date,
+ toDateTime('2024-01-01 00:00:00') AS date_time,
+ '2024-01-01 00:00:00' AS date_time_string
+SELECT
+ subtractWeeks(date, 1) AS subtract_weeks_with_date,
+ subtractWeeks(date_time, 1) AS subtract_weeks_with_date_time,
+ subtractWeeks(date_time_string, 1) AS subtract_weeks_with_date_time_string
+```
+
+```response
+ ┌─subtract_weeks_with_date─┬─subtract_weeks_with_date_time─┬─subtract_weeks_with_date_time_string─┐
+ │ 2023-12-25 │ 2023-12-25 00:00:00 │ 2023-12-25 00:00:00.000 │
+ └──────────────────────────┴───────────────────────────────┴──────────────────────────────────────┘
+```
+
+## subtractDays
+
+Subtracts a specified number of days from a date, a date with time or a string-encoded date / date with time.
+
+**Syntax**
+
+```sql
+subtractDays(date, num)
+```
+
+**Parameters**
+
+- `date`: Date / date with time to subtract specified number of days from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md).
+- `num`: Number of days to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md).
+
+**Returned value**
+- Returns `date` minus `num` days. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md).
+
+**Example**
+
+```sql
+WITH
+ toDate('2024-01-01') AS date,
+ toDateTime('2024-01-01 00:00:00') AS date_time,
+ '2024-01-01 00:00:00' AS date_time_string
+SELECT
+ subtractDays(date, 31) AS subtract_days_with_date,
+ subtractDays(date_time, 31) AS subtract_days_with_date_time,
+ subtractDays(date_time_string, 31) AS subtract_days_with_date_time_string
+```
+
+```response
+┌─subtract_days_with_date─┬─subtract_days_with_date_time─┬─subtract_days_with_date_time_string─┐
+│ 2023-12-01 │ 2023-12-01 00:00:00 │ 2023-12-01 00:00:00.000 │
+└─────────────────────────┴──────────────────────────────┴─────────────────────────────────────┘
+```
+
+## subtractHours
+
+Subtracts a specified number of hours from a date, a date with time or a string-encoded date / date with time.
+
+**Syntax**
+
+```sql
+subtractHours(date, num)
+```
+
+**Parameters**
+
+- `date`: Date / date with time to subtract specified number of hours from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[Datetime](../data-types/datetime.md)/[Datetime64](../data-types/datetime64.md), [String](../data-types/string.md).
+- `num`: Number of hours to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md).
+
+**Returned value**
+- Returns `date` minus `num` hours. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[Datetime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md).
+
+**Example**
+
+```sql
+WITH
+ toDate('2024-01-01') AS date,
+ toDateTime('2024-01-01 00:00:00') AS date_time,
+ '2024-01-01 00:00:00' AS date_time_string
+SELECT
+ subtractHours(date, 12) AS subtract_hours_with_date,
+ subtractHours(date_time, 12) AS subtract_hours_with_date_time,
+ subtractHours(date_time_string, 12) AS subtract_hours_with_date_time_string
+```
+
+```response
+┌─subtract_hours_with_date─┬─subtract_hours_with_date_time─┬─subtract_hours_with_date_time_string─┐
+│ 2023-12-31 12:00:00 │ 2023-12-31 12:00:00 │ 2023-12-31 12:00:00.000 │
+└──────────────────────────┴───────────────────────────────┴──────────────────────────────────────┘
+```
+
+## subtractMinutes
+
+Subtracts a specified number of minutes from a date, a date with time or a string-encoded date / date with time.
+
+**Syntax**
+
+```sql
+subtractMinutes(date, num)
+```
+
+**Parameters**
+
+- `date`: Date / date with time to subtract specified number of minutes from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md).
+- `num`: Number of minutes to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md).
+
+**Returned value**
+- Returns `date` minus `num` minutes. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md).
+
+**Example**
+
+```sql
+WITH
+ toDate('2024-01-01') AS date,
+ toDateTime('2024-01-01 00:00:00') AS date_time,
+ '2024-01-01 00:00:00' AS date_time_string
+SELECT
+ subtractMinutes(date, 30) AS subtract_minutes_with_date,
+ subtractMinutes(date_time, 30) AS subtract_minutes_with_date_time,
+ subtractMinutes(date_time_string, 30) AS subtract_minutes_with_date_time_string
+```
+
+```response
+┌─subtract_minutes_with_date─┬─subtract_minutes_with_date_time─┬─subtract_minutes_with_date_time_string─┐
+│ 2023-12-31 23:30:00 │ 2023-12-31 23:30:00 │ 2023-12-31 23:30:00.000 │
+└────────────────────────────┴─────────────────────────────────┴────────────────────────────────────────┘
+```
+
+## subtractSeconds
+
+Subtracts a specified number of seconds from a date, a date with time or a string-encoded date / date with time.
+
+**Syntax**
+
+```sql
+subtractSeconds(date, num)
+```
+
+**Parameters**
+
+- `date`: Date / date with time to subtract specified number of seconds from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md).
+- `num`: Number of seconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md).
+
+**Returned value**
+- Returns `date` minus `num` seconds. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md).
+
+**Example**
+
+```sql
+WITH
+ toDate('2024-01-01') AS date,
+ toDateTime('2024-01-01 00:00:00') AS date_time,
+ '2024-01-01 00:00:00' AS date_time_string
+SELECT
+ subtractSeconds(date, 60) AS subtract_seconds_with_date,
+ subtractSeconds(date_time, 60) AS subtract_seconds_with_date_time,
+ subtractSeconds(date_time_string, 60) AS subtract_seconds_with_date_time_string
+```
+
+```response
+┌─subtract_seconds_with_date─┬─subtract_seconds_with_date_time─┬─subtract_seconds_with_date_time_string─┐
+│ 2023-12-31 23:59:00 │ 2023-12-31 23:59:00 │ 2023-12-31 23:59:00.000 │
+└────────────────────────────┴─────────────────────────────────┴────────────────────────────────────────┘
+```
+
+## subtractMilliseconds
+
+Subtracts a specified number of milliseconds from a date with time or a string-encoded date with time.
+
+**Syntax**
+
+```sql
+subtractMilliseconds(date_time, num)
+```
+
+**Parameters**
+
+- `date_time`: Date with time to subtract specified number of milliseconds from. [DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md).
+- `num`: Number of milliseconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md).
+
+**Returned value**
+- Returns `date_time` minus `num` milliseconds. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md).
+
+**Example**
+
+```sql
+WITH
+ toDateTime('2024-01-01 00:00:00') AS date_time,
+ '2024-01-01 00:00:00' AS date_time_string
+SELECT
+ subtractMilliseconds(date_time, 1000) AS subtract_milliseconds_with_date_time,
+ subtractMilliseconds(date_time_string, 1000) AS subtract_milliseconds_with_date_time_string
+```
+
+```response
+┌─subtract_milliseconds_with_date_time─┬─subtract_milliseconds_with_date_time_string─┐
+│ 2023-12-31 23:59:59.000 │ 2023-12-31 23:59:59.000 │
+└──────────────────────────────────────┴─────────────────────────────────────────────┘
+```
+
+## subtractMicroseconds
+
+Subtracts a specified number of microseconds from a date with time or a string-encoded date with time.
+
+**Syntax**
+
+```sql
+subtractMicroseconds(date_time, num)
+```
+
+**Parameters**
+
+- `date_time`: Date with time to subtract specified number of microseconds from. [DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md).
+- `num`: Number of microseconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md).
+
+**Returned value**
+- Returns `date_time` minus `num` microseconds. [DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md).
+
+**Example**
+
+```sql
+WITH
+ toDateTime('2024-01-01 00:00:00') AS date_time,
+ '2024-01-01 00:00:00' AS date_time_string
+SELECT
+ subtractMicroseconds(date_time, 1000000) AS subtract_microseconds_with_date_time,
+ subtractMicroseconds(date_time_string, 1000000) AS subtract_microseconds_with_date_time_string
+```
+
+```response
+┌─subtract_microseconds_with_date_time─┬─subtract_microseconds_with_date_time_string─┐
+│ 2023-12-31 23:59:59.000000 │ 2023-12-31 23:59:59.000000 │
+└──────────────────────────────────────┴─────────────────────────────────────────────┘
+```
+
+## subtractNanoseconds
+
+Subtracts a specified number of nanoseconds from a date with time or a string-encoded date with time.
+
+**Syntax**
+
+```sql
+subtractNanoseconds(date_time, num)
+```
+
+**Parameters**
+
+- `date_time`: Date with time to subtract specified number of nanoseconds from. [DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md).
+- `num`: Number of nanoseconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md).
+
+**Returned value**
+- Returns `date_time` minus `num` nanoseconds. [DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md).
+
+**Example**
+
+```sql
+WITH
+ toDateTime('2024-01-01 00:00:00') AS date_time,
+ '2024-01-01 00:00:00' AS date_time_string
+SELECT
+ subtractNanoseconds(date_time, 1000) AS subtract_nanoseconds_with_date_time,
+ subtractNanoseconds(date_time_string, 1000) AS subtract_nanoseconds_with_date_time_string
+```
+
+```response
+┌─subtract_nanoseconds_with_date_time─┬─subtract_nanoseconds_with_date_time_string─┐
+│ 2023-12-31 23:59:59.999999000 │ 2023-12-31 23:59:59.999999000 │
+└─────────────────────────────────────┴────────────────────────────────────────────┘
+```
+
+## subtractInterval
+
+Adds a negated interval to another interval or tuple of intervals.
+
+**Syntax**
+
+```sql
+subtractInterval(interval_1, interval_2)
+```
+
+**Parameters**
+
+- `interval_1`: First interval or interval of tuples. [interval](../data-types/special-data-types/interval.md), [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)).
+- `interval_2`: Second interval to be negated. [interval](../data-types/special-data-types/interval.md).
+
+**Returned value**
+- Returns a tuple of intervals. [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)).
+
+:::note
+If the types of the first interval (or the interval in the tuple) and the second interval are the same they will be merged into one interval.
+:::
+
+**Example**
+
+Query:
+
+```sql
+SELECT subtractInterval(INTERVAL 1 DAY, INTERVAL 1 MONTH);
+SELECT subtractInterval((INTERVAL 1 DAY, INTERVAL 1 YEAR), INTERVAL 1 MONTH);
+SELECT subtractInterval(INTERVAL 2 DAY, INTERVAL 1 DAY);
+```
+
+Result:
+
+```response
+┌─subtractInterval(toIntervalDay(1), toIntervalMonth(1))─┐
+│ (1,-1) │
+└────────────────────────────────────────────────────────┘
+┌─subtractInterval((toIntervalDay(1), toIntervalYear(1)), toIntervalMonth(1))─┐
+│ (1,1,-1) │
+└─────────────────────────────────────────────────────────────────────────────┘
+┌─subtractInterval(toIntervalDay(2), toIntervalDay(1))─┐
+│ (1) │
+└──────────────────────────────────────────────────────┘
+```
+
+## subtractTupleOfIntervals
+
+Consecutively subtracts a tuple of intervals from a Date or a DateTime.
+
+**Syntax**
+
+```sql
+subtractTupleOfIntervals(interval_1, interval_2)
+```
+
+**Parameters**
+
+- `date`: First interval or interval of tuples. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md).
+- `intervals`: Tuple of intervals to subtract from `date`. [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)).
+
+**Returned value**
+- Returns `date` with subtracted `intervals`. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md).
+
+**Example**
+
+Query:
+
+```sql
+WITH toDate('2018-01-01') AS date SELECT subtractTupleOfIntervals(date, (INTERVAL 1 DAY, INTERVAL 1 YEAR))
+```
+
+Result:
+
+```response
+┌─subtractTupleOfIntervals(date, (toIntervalDay(1), toIntervalYear(1)))─┐
+│ 2016-12-31 │
+└───────────────────────────────────────────────────────────────────────┘
+```
+
## timeSlots(StartTime, Duration,\[, Size\])
For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default.
diff --git a/docs/en/sql-reference/functions/math-functions.md b/docs/en/sql-reference/functions/math-functions.md
index 2feb91cfce4..945166056af 100644
--- a/docs/en/sql-reference/functions/math-functions.md
+++ b/docs/en/sql-reference/functions/math-functions.md
@@ -792,6 +792,39 @@ Result:
│ -1 │
└──────────┘
```
+## sigmoid
+
+Returns the [sigmoid function](https://en.wikipedia.org/wiki/Sigmoid_function).
+
+**Syntax**
+
+```sql
+sigmoid(x)
+```
+
+**Parameters**
+
+- `x` — input value. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
+
+**Returned value**
+
+- Corresponding value along the sigmoid curve between 0 and 1. [Float64](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT round(sigmoid(x), 5) FROM (SELECT arrayJoin([-1, 0, 1]) AS x);
+```
+
+Result:
+
+```result
+0.26894
+0.5
+0.73106
+```
## degrees
@@ -914,37 +947,3 @@ Result:
│ 11 │
└──────────────────────────────────┘
```
-
-## sigmoid
-
-Returns the [sigmoid function](https://en.wikipedia.org/wiki/Sigmoid_function).
-
-**Syntax**
-
-```sql
-sigmoid(x)
-```
-
-**Parameters**
-
-- `x` — input value. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
-
-**Returned value**
-
-- Corresponding value along the sigmoid curve between 0 and 1. [Float64](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-``` sql
-SELECT round(sigmoid(x), 5) FROM (SELECT arrayJoin([-1, 0, 1]) AS x);
-```
-
-Result:
-
-```result
-0.26894
-0.5
-0.73106
-```
diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md
index 8283de95994..377283bc006 100644
--- a/docs/en/sql-reference/functions/tuple-map-functions.md
+++ b/docs/en/sql-reference/functions/tuple-map-functions.md
@@ -16,7 +16,7 @@ map(key1, value1[, key2, value2, ...])
**Arguments**
-- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md), [LowCardinality](../../sql-reference/data-types/lowcardinality.md), [FixedString](../../sql-reference/data-types/fixedstring.md), [UUID](../../sql-reference/data-types/uuid.md), [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md), [Date32](../../sql-reference/data-types/date32.md), [Enum](../../sql-reference/data-types/enum.md).
+- `key` — The key part of the pair. Arbitrary type, except [Nullable](../../sql-reference/data-types/nullable.md) and [LowCardinality](../../sql-reference/data-types/lowcardinality.md) nested with [Nullable](../../sql-reference/data-types/nullable.md).
- `value` — The value part of the pair. Arbitrary type, including [Map](../../sql-reference/data-types/map.md) and [Array](../../sql-reference/data-types/array.md).
**Returned value**
diff --git a/docs/en/sql-reference/statements/select/with.md b/docs/en/sql-reference/statements/select/with.md
index a59ef463419..aa0e0c61c4e 100644
--- a/docs/en/sql-reference/statements/select/with.md
+++ b/docs/en/sql-reference/statements/select/with.md
@@ -5,21 +5,21 @@ sidebar_label: WITH
# WITH Clause
-ClickHouse supports Common Table Expressions ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)) and substitutes the code defined in the `WITH` clause in all places of use for the rest of `SELECT` query. Named subqueries can be included to the current and child query context in places where table objects are allowed. Recursion is prevented by hiding the current level CTEs from the WITH expression.
+ClickHouse supports Common Table Expressions ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)) and substitutes the code defined in the `WITH` clause in all places of use for the rest of `SELECT` query. Named subqueries can be included to the current and child query context in places where table objects are allowed. Recursion is prevented by hiding the current level CTEs from the WITH expression.
Please note that CTEs do not guarantee the same results in all places they are called because the query will be re-executed for each use case.
An example of such behavior is below
``` sql
-with cte_numbers as
+with cte_numbers as
(
- select
- num
- from generateRandom('num UInt64', NULL)
+ select
+ num
+ from generateRandom('num UInt64', NULL)
limit 1000000
)
select
- count()
+ count()
from cte_numbers
where num in (select num from cte_numbers)
```
@@ -87,3 +87,226 @@ LIMIT 10;
WITH test1 AS (SELECT i + 1, j + 1 FROM test1)
SELECT * FROM test1;
```
+
+## Recursive Queries
+
+The optional RECURSIVE modifier allows for a WITH query to refer to its own output. Example:
+
+**Example:** Sum integers from 1 through 100
+
+```sql
+WITH RECURSIVE test_table AS (
+ SELECT 1 AS number
+UNION ALL
+ SELECT number + 1 FROM test_table WHERE number < 100
+)
+SELECT sum(number) FROM test_table;
+```
+
+``` text
+┌─sum(number)─┐
+│ 5050 │
+└─────────────┘
+```
+
+The general form of a recursive `WITH` query is always a non-recursive term, then `UNION ALL`, then a recursive term, where only the recursive term can contain a reference to the query's own output. Recursive CTE query is executed as follows:
+
+1. Evaluate the non-recursive term. Place result of non-recursive term query in a temporary working table.
+2. As long as the working table is not empty, repeat these steps:
+ 1. Evaluate the recursive term, substituting the current contents of the working table for the recursive self-reference. Place result of recursive term query in a temporary intermediate table.
+ 2. Replace the contents of the working table with the contents of the intermediate table, then empty the intermediate table.
+
+Recursive queries are typically used to work with hierarchical or tree-structured data. For example, we can write a query that performs tree traversal:
+
+**Example:** Tree traversal
+
+First let's create tree table:
+
+```sql
+DROP TABLE IF EXISTS tree;
+CREATE TABLE tree
+(
+ id UInt64,
+ parent_id Nullable(UInt64),
+ data String
+) ENGINE = MergeTree ORDER BY id;
+
+INSERT INTO tree VALUES (0, NULL, 'ROOT'), (1, 0, 'Child_1'), (2, 0, 'Child_2'), (3, 1, 'Child_1_1');
+```
+
+We can traverse those tree with such query:
+
+**Example:** Tree traversal
+```sql
+WITH RECURSIVE search_tree AS (
+ SELECT id, parent_id, data
+ FROM tree t
+ WHERE t.id = 0
+UNION ALL
+ SELECT t.id, t.parent_id, t.data
+ FROM tree t, search_tree st
+ WHERE t.parent_id = st.id
+)
+SELECT * FROM search_tree;
+```
+
+```text
+┌─id─┬─parent_id─┬─data──────┐
+│ 0 │ ᴺᵁᴸᴸ │ ROOT │
+│ 1 │ 0 │ Child_1 │
+│ 2 │ 0 │ Child_2 │
+│ 3 │ 1 │ Child_1_1 │
+└────┴───────────┴───────────┘
+```
+
+### Search order
+
+To create a depth-first order, we compute for each result row an array of rows that we have already visited:
+
+**Example:** Tree traversal depth-first order
+```sql
+WITH RECURSIVE search_tree AS (
+ SELECT id, parent_id, data, [t.id] AS path
+ FROM tree t
+ WHERE t.id = 0
+UNION ALL
+ SELECT t.id, t.parent_id, t.data, arrayConcat(path, [t.id])
+ FROM tree t, search_tree st
+ WHERE t.parent_id = st.id
+)
+SELECT * FROM search_tree ORDER BY path;
+```
+
+```text
+┌─id─┬─parent_id─┬─data──────┬─path────┐
+│ 0 │ ᴺᵁᴸᴸ │ ROOT │ [0] │
+│ 1 │ 0 │ Child_1 │ [0,1] │
+│ 3 │ 1 │ Child_1_1 │ [0,1,3] │
+│ 2 │ 0 │ Child_2 │ [0,2] │
+└────┴───────────┴───────────┴─────────┘
+```
+
+To create a breadth-first order, standard approach is to add column that tracks the depth of the search:
+
+**Example:** Tree traversal breadth-first order
+```sql
+WITH RECURSIVE search_tree AS (
+ SELECT id, parent_id, data, [t.id] AS path, toUInt64(0) AS depth
+ FROM tree t
+ WHERE t.id = 0
+UNION ALL
+ SELECT t.id, t.parent_id, t.data, arrayConcat(path, [t.id]), depth + 1
+ FROM tree t, search_tree st
+ WHERE t.parent_id = st.id
+)
+SELECT * FROM search_tree ORDER BY depth;
+```
+
+```text
+┌─id─┬─link─┬─data──────┬─path────┬─depth─┐
+│ 0 │ ᴺᵁᴸᴸ │ ROOT │ [0] │ 0 │
+│ 1 │ 0 │ Child_1 │ [0,1] │ 1 │
+│ 2 │ 0 │ Child_2 │ [0,2] │ 1 │
+│ 3 │ 1 │ Child_1_1 │ [0,1,3] │ 2 │
+└────┴──────┴───────────┴─────────┴───────┘
+```
+
+### Cycle detection
+
+First let's create graph table:
+
+```sql
+DROP TABLE IF EXISTS graph;
+CREATE TABLE graph
+(
+ from UInt64,
+ to UInt64,
+ label String
+) ENGINE = MergeTree ORDER BY (from, to);
+
+INSERT INTO graph VALUES (1, 2, '1 -> 2'), (1, 3, '1 -> 3'), (2, 3, '2 -> 3'), (1, 4, '1 -> 4'), (4, 5, '4 -> 5');
+```
+
+We can traverse that graph with such query:
+
+**Example:** Graph traversal without cycle detection
+```sql
+WITH RECURSIVE search_graph AS (
+ SELECT from, to, label FROM graph g
+ UNION ALL
+ SELECT g.from, g.to, g.label
+ FROM graph g, search_graph sg
+ WHERE g.from = sg.to
+)
+SELECT DISTINCT * FROM search_graph ORDER BY from;
+```
+```text
+┌─from─┬─to─┬─label──┐
+│ 1 │ 4 │ 1 -> 4 │
+│ 1 │ 2 │ 1 -> 2 │
+│ 1 │ 3 │ 1 -> 3 │
+│ 2 │ 3 │ 2 -> 3 │
+│ 4 │ 5 │ 4 -> 5 │
+└──────┴────┴────────┘
+```
+
+But if we add cycle in that graph, previous query will fail with `Maximum recursive CTE evaluation depth` error:
+
+```sql
+INSERT INTO graph VALUES (5, 1, '5 -> 1');
+
+WITH RECURSIVE search_graph AS (
+ SELECT from, to, label FROM graph g
+UNION ALL
+ SELECT g.from, g.to, g.label
+ FROM graph g, search_graph sg
+ WHERE g.from = sg.to
+)
+SELECT DISTINCT * FROM search_graph ORDER BY from;
+```
+
+```text
+Code: 306. DB::Exception: Received from localhost:9000. DB::Exception: Maximum recursive CTE evaluation depth (1000) exceeded, during evaluation of search_graph AS (SELECT from, to, label FROM graph AS g UNION ALL SELECT g.from, g.to, g.label FROM graph AS g, search_graph AS sg WHERE g.from = sg.to). Consider raising max_recursive_cte_evaluation_depth setting.: While executing RecursiveCTESource. (TOO_DEEP_RECURSION)
+```
+
+The standard method for handling cycles is to compute an array of the already visited nodes:
+
+**Example:** Graph traversal with cycle detection
+```sql
+WITH RECURSIVE search_graph AS (
+ SELECT from, to, label, false AS is_cycle, [tuple(g.from, g.to)] AS path FROM graph g
+UNION ALL
+ SELECT g.from, g.to, g.label, has(path, tuple(g.from, g.to)), arrayConcat(sg.path, [tuple(g.from, g.to)])
+ FROM graph g, search_graph sg
+ WHERE g.from = sg.to AND NOT is_cycle
+)
+SELECT * FROM search_graph WHERE is_cycle ORDER BY from;
+```
+
+```text
+┌─from─┬─to─┬─label──┬─is_cycle─┬─path──────────────────────┐
+│ 1 │ 4 │ 1 -> 4 │ true │ [(1,4),(4,5),(5,1),(1,4)] │
+│ 4 │ 5 │ 4 -> 5 │ true │ [(4,5),(5,1),(1,4),(4,5)] │
+│ 5 │ 1 │ 5 -> 1 │ true │ [(5,1),(1,4),(4,5),(5,1)] │
+└──────┴────┴────────┴──────────┴───────────────────────────┘
+```
+
+### Infinite queries
+
+It is also possible to use infinite recursive CTE queries if `LIMIT` is used in outer query:
+
+**Example:** Infinite recursive CTE query
+```sql
+WITH RECURSIVE test_table AS (
+ SELECT 1 AS number
+UNION ALL
+ SELECT number + 1 FROM test_table
+)
+SELECT sum(number) FROM (SELECT number FROM test_table LIMIT 100);
+```
+
+```text
+┌─sum(number)─┐
+│ 5050 │
+└─────────────┘
+```
diff --git a/docs/en/sql-reference/statements/truncate.md b/docs/en/sql-reference/statements/truncate.md
index 8cd5a6a1424..fb5d4a3055a 100644
--- a/docs/en/sql-reference/statements/truncate.md
+++ b/docs/en/sql-reference/statements/truncate.md
@@ -25,7 +25,7 @@ If the `alter_sync` is set to `2` and some replicas are not active for more than
## TRUNCATE ALL TABLES
``` sql
-TRUNCATE ALL TABLES [IF EXISTS] db [ON CLUSTER cluster]
+TRUNCATE ALL TABLES FROM [IF EXISTS] db [ON CLUSTER cluster]
```
Removes all data from all tables in a database.
diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp
index 2736d13e751..2a658d7aaa2 100644
--- a/src/Access/ContextAccess.cpp
+++ b/src/Access/ContextAccess.cpp
@@ -570,11 +570,8 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
if (params.full_access)
return true;
- auto access_granted = [&]
+ auto access_granted = []
{
- if (trace_log)
- LOG_TRACE(trace_log, "Access granted: {}{}", (AccessRightsElement{flags, args...}.toStringWithoutOptions()),
- (grant_option ? " WITH GRANT OPTION" : ""));
return true;
};
@@ -582,9 +579,6 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
FormatStringHelper fmt_string [[maybe_unused]],
FmtArgs && ...fmt_args [[maybe_unused]])
{
- if (trace_log)
- LOG_TRACE(trace_log, "Access denied: {}{}", (AccessRightsElement{flags, args...}.toStringWithoutOptions()),
- (grant_option ? " WITH GRANT OPTION" : ""));
if constexpr (throw_if_denied)
throw Exception(error_code, std::move(fmt_string), getUserName(), std::forward(fmt_args)...);
return false;
diff --git a/src/AggregateFunctions/AggregateFunctionAggThrow.cpp b/src/AggregateFunctions/AggregateFunctionAggThrow.cpp
index 27432bc94ba..ebf6cabbcfd 100644
--- a/src/AggregateFunctions/AggregateFunctionAggThrow.cpp
+++ b/src/AggregateFunctions/AggregateFunctionAggThrow.cpp
@@ -16,7 +16,7 @@ struct Settings;
namespace ErrorCodes
{
extern const int AGGREGATE_FUNCTION_THROW;
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+ extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
}
namespace
@@ -116,7 +116,7 @@ void registerAggregateFunctionAggThrow(AggregateFunctionFactory & factory)
if (parameters.size() == 1)
throw_probability = parameters[0].safeGet();
else if (parameters.size() > 1)
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} cannot have more than one parameter", name);
+ throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION, "Aggregate function {} cannot have more than one parameter", name);
return std::make_shared(argument_types, parameters, throw_probability);
});
diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.cpp b/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.cpp
index 60e8df64283..8674aac2e90 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.cpp
@@ -27,6 +27,7 @@ struct Settings;
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+ extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
extern const int TOO_LARGE_ARRAY_SIZE;
extern const int CANNOT_CONVERT_TYPE;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
@@ -74,7 +75,7 @@ public:
if (!params.empty())
{
if (params.size() > 2)
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at most two parameters.", getName());
+ throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION, "Aggregate function {} requires at most two parameters.", getName());
default_value = params[0];
diff --git a/src/AggregateFunctions/AggregateFunctionMLMethod.cpp b/src/AggregateFunctions/AggregateFunctionMLMethod.cpp
index d2178437770..730b5cfa5cc 100644
--- a/src/AggregateFunctions/AggregateFunctionMLMethod.cpp
+++ b/src/AggregateFunctions/AggregateFunctionMLMethod.cpp
@@ -22,7 +22,8 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
extern const int LOGICAL_ERROR;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+ extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
+ extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
}
namespace
@@ -34,12 +35,12 @@ namespace
const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *)
{
if (parameters.size() > 4)
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+ throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION,
"Aggregate function {} requires at most four parameters: "
"learning_rate, l2_regularization_coef, mini-batch size and weights_updater method", name);
if (argument_types.size() < 2)
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+ throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION,
"Aggregate function {} requires at least two arguments: target and model's parameters", name);
for (size_t i = 0; i < argument_types.size(); ++i)
diff --git a/src/AggregateFunctions/AggregateFunctionMannWhitney.cpp b/src/AggregateFunctions/AggregateFunctionMannWhitney.cpp
index e7bc5df335f..8434f11eae6 100644
--- a/src/AggregateFunctions/AggregateFunctionMannWhitney.cpp
+++ b/src/AggregateFunctions/AggregateFunctionMannWhitney.cpp
@@ -17,19 +17,19 @@
#include
-namespace ErrorCodes
-{
- extern const int NOT_IMPLEMENTED;
- extern const int ILLEGAL_TYPE_OF_ARGUMENT;
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
- extern const int BAD_ARGUMENTS;
-}
-
namespace DB
{
struct Settings;
+namespace ErrorCodes
+{
+ extern const int NOT_IMPLEMENTED;
+ extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+ extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
+ extern const int BAD_ARGUMENTS;
+}
+
namespace
{
@@ -141,7 +141,7 @@ public:
: IAggregateFunctionDataHelper ({arguments}, {}, createResultType())
{
if (params.size() > 2)
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} require two parameter or less", getName());
+ throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION, "Aggregate function {} require two parameter or less", getName());
if (params.empty())
{
diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.cpp b/src/AggregateFunctions/AggregateFunctionQuantile.cpp
index 4d37ec69d26..f72b28030c3 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantile.cpp
+++ b/src/AggregateFunctions/AggregateFunctionQuantile.cpp
@@ -14,7 +14,7 @@ struct Settings;
namespace ErrorCodes
{
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+ extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int NOT_IMPLEMENTED;
}
@@ -118,7 +118,7 @@ AggregateFunctionPtr createAggregateFunctionQuantile(
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
{
if (argument_types.empty())
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
+ throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Aggregate function {} requires at least one argument", name);
const DataTypePtr & argument_type = argument_types[0];
WhichDataType which(argument_type);
diff --git a/src/AggregateFunctions/AggregateFunctionQuantileBFloat16Weighted.cpp b/src/AggregateFunctions/AggregateFunctionQuantileBFloat16Weighted.cpp
index 6d881b77c16..b0a39b1fdab 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantileBFloat16Weighted.cpp
+++ b/src/AggregateFunctions/AggregateFunctionQuantileBFloat16Weighted.cpp
@@ -12,7 +12,7 @@ struct Settings;
namespace ErrorCodes
{
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+ extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
@@ -27,7 +27,7 @@ AggregateFunctionPtr createAggregateFunctionQuantile(
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
{
if (argument_types.empty())
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
+ throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Aggregate function {} requires at least one argument", name);
const DataTypePtr & argument_type = argument_types[0];
WhichDataType which(argument_type);
diff --git a/src/AggregateFunctions/AggregateFunctionQuantileDD.cpp b/src/AggregateFunctions/AggregateFunctionQuantileDD.cpp
index f3d6b26ee75..7b1ae43038a 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantileDD.cpp
+++ b/src/AggregateFunctions/AggregateFunctionQuantileDD.cpp
@@ -13,7 +13,7 @@ struct Settings;
namespace ErrorCodes
{
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+ extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
@@ -29,7 +29,7 @@ AggregateFunctionPtr createAggregateFunctionQuantile(
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
{
if (argument_types.empty())
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
+ throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Aggregate function {} requires at least one argument", name);
const DataTypePtr & argument_type = argument_types[0];
WhichDataType which(argument_type);
diff --git a/src/AggregateFunctions/AggregateFunctionQuantileExactHigh.cpp b/src/AggregateFunctions/AggregateFunctionQuantileExactHigh.cpp
index b44df755158..6fa4158076c 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantileExactHigh.cpp
+++ b/src/AggregateFunctions/AggregateFunctionQuantileExactHigh.cpp
@@ -13,7 +13,7 @@ struct Settings;
namespace ErrorCodes
{
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+ extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
@@ -29,7 +29,7 @@ AggregateFunctionPtr createAggregateFunctionQuantile(
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
{
if (argument_types.empty())
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
+ throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Aggregate function {} requires at least one argument", name);
const DataTypePtr & argument_type = argument_types[0];
WhichDataType which(argument_type);
diff --git a/src/AggregateFunctions/AggregateFunctionQuantileExactInclusive.cpp b/src/AggregateFunctions/AggregateFunctionQuantileExactInclusive.cpp
index d8287be86ca..bed5cf95bd8 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantileExactInclusive.cpp
+++ b/src/AggregateFunctions/AggregateFunctionQuantileExactInclusive.cpp
@@ -13,7 +13,7 @@ struct Settings;
namespace ErrorCodes
{
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+ extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
@@ -28,7 +28,7 @@ AggregateFunctionPtr createAggregateFunctionQuantile(
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
{
if (argument_types.empty())
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
+ throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Aggregate function {} requires at least one argument", name);
const DataTypePtr & argument_type = argument_types[0];
WhichDataType which(argument_type);
diff --git a/src/AggregateFunctions/AggregateFunctionQuantileTiming.cpp b/src/AggregateFunctions/AggregateFunctionQuantileTiming.cpp
index 1bb77892c0d..e293fc6e59d 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantileTiming.cpp
+++ b/src/AggregateFunctions/AggregateFunctionQuantileTiming.cpp
@@ -13,7 +13,7 @@ struct Settings;
namespace ErrorCodes
{
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+ extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
@@ -28,7 +28,7 @@ AggregateFunctionPtr createAggregateFunctionQuantile(
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
{
if (argument_types.empty())
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
+ throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Aggregate function {} requires at least one argument", name);
const DataTypePtr & argument_type = argument_types[0];
WhichDataType which(argument_type);
diff --git a/src/AggregateFunctions/AggregateFunctionSparkbar.cpp b/src/AggregateFunctions/AggregateFunctionSparkbar.cpp
index b6e538520a8..362ffbe20d2 100644
--- a/src/AggregateFunctions/AggregateFunctionSparkbar.cpp
+++ b/src/AggregateFunctions/AggregateFunctionSparkbar.cpp
@@ -253,9 +253,9 @@ private:
else
{
Y scaled;
- bool has_overfllow = common::mulOverflow(y, levels_num, scaled);
+ bool has_overflow = common::mulOverflow(y, levels_num, scaled);
- if (has_overfllow)
+ if (has_overflow)
y = y / (y_max / levels_num) + 1;
else
y = scaled / y_max + 1;
diff --git a/src/AggregateFunctions/AggregateFunctionTopK.cpp b/src/AggregateFunctions/AggregateFunctionTopK.cpp
index d0e4f507d46..26f756abe18 100644
--- a/src/AggregateFunctions/AggregateFunctionTopK.cpp
+++ b/src/AggregateFunctions/AggregateFunctionTopK.cpp
@@ -35,7 +35,7 @@ namespace ErrorCodes
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int BAD_ARGUMENTS;
extern const int LOGICAL_ERROR;
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+ extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
}
@@ -467,7 +467,7 @@ AggregateFunctionPtr createAggregateFunctionTopK(const std::string & name, const
if (!params.empty())
{
if (params.size() > 3)
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+ throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION,
"Aggregate function '{}' requires three parameters or less", name);
threshold = applyVisitor(FieldVisitorConvertToNumber(), params[0]);
diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp
index 849f81279e7..7efd6154227 100644
--- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp
+++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp
@@ -4,16 +4,16 @@
#include
-namespace ErrorCodes
-{
- extern const int BAD_ARGUMENTS;
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-}
-
namespace DB
{
struct Settings;
+namespace ErrorCodes
+{
+ extern const int BAD_ARGUMENTS;
+ extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
+}
+
namespace
{
@@ -80,7 +80,7 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(
assertBinary(name, argument_types);
if (parameters.size() > 1)
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires zero or one parameter.", name);
+ throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION, "Aggregate function {} requires zero or one parameter.", name);
if (!isNumber(argument_types[0]) || !isNumber(argument_types[1]))
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function {} only supports numerical types", name);
diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionArray.cpp b/src/AggregateFunctions/Combinators/AggregateFunctionArray.cpp
index a54092f0bcb..16646c702bf 100644
--- a/src/AggregateFunctions/Combinators/AggregateFunctionArray.cpp
+++ b/src/AggregateFunctions/Combinators/AggregateFunctionArray.cpp
@@ -9,7 +9,7 @@ struct Settings;
namespace ErrorCodes
{
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+ extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
@@ -26,7 +26,7 @@ public:
DataTypes transformArguments(const DataTypes & arguments) const override
{
if (arguments.empty())
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "-Array aggregate functions require at least one argument");
+ throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Array aggregate functions require at least one argument");
DataTypes nested_arguments;
for (const auto & type : arguments)
diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionIf.h b/src/AggregateFunctions/Combinators/AggregateFunctionIf.h
index a893fc91780..f57cfa41752 100644
--- a/src/AggregateFunctions/Combinators/AggregateFunctionIf.h
+++ b/src/AggregateFunctions/Combinators/AggregateFunctionIf.h
@@ -18,7 +18,7 @@ struct Settings;
namespace ErrorCodes
{
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+ extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
@@ -42,7 +42,7 @@ public:
, nested_func(nested), num_arguments(types.size())
{
if (num_arguments == 0)
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} require at least one argument", getName());
+ throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Aggregate function {} require at least one argument", getName());
only_null_condition = types.back()->onlyNull();
diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp
index 04b2c6de19e..52efee03ae4 100644
--- a/src/Analyzer/Passes/QueryAnalysisPass.cpp
+++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp
@@ -4093,6 +4093,21 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifier(const IdentifierLook
{
bool prefer_column_name_to_alias = scope.context->getSettingsRef().prefer_column_name_to_alias;
+ if (identifier_lookup.isExpressionLookup())
+ {
+ /* For aliases from ARRAY JOIN we prefer column from join tree:
+ * SELECT id FROM ( SELECT ... ) AS subquery ARRAY JOIN [0] AS id INNER JOIN second_table USING (id)
+ * In the example, identifier `id` should be resolved into one from USING (id) column.
+ */
+ auto alias_it = scope.alias_name_to_expression_node.find(identifier_lookup.identifier.getFullName());
+ if (alias_it != scope.alias_name_to_expression_node.end() && alias_it->second->getNodeType() == QueryTreeNodeType::COLUMN)
+ {
+ const auto & column_node = alias_it->second->as();
+ if (column_node.getColumnSource()->getNodeType() == QueryTreeNodeType::ARRAY_JOIN)
+ prefer_column_name_to_alias = true;
+ }
+ }
+
if (unlikely(prefer_column_name_to_alias))
{
if (identifier_resolve_settings.allow_to_check_join_tree)
diff --git a/src/Analyzer/ValidationUtils.cpp b/src/Analyzer/ValidationUtils.cpp
index 60cc1dd521f..e17639367eb 100644
--- a/src/Analyzer/ValidationUtils.cpp
+++ b/src/Analyzer/ValidationUtils.cpp
@@ -266,6 +266,9 @@ void validateAggregates(const QueryTreeNodePtr & query_node, AggregatesValidatio
if (query_node_typed.hasHaving())
validate_group_by_columns_visitor.visit(query_node_typed.getHaving());
+ if (query_node_typed.hasQualify())
+ validate_group_by_columns_visitor.visit(query_node_typed.getQualify());
+
if (query_node_typed.hasOrderBy())
validate_group_by_columns_visitor.visit(query_node_typed.getOrderByNode());
diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp
index 9964de2ad6e..efdc18cc29c 100644
--- a/src/Backups/BackupCoordinationLocal.cpp
+++ b/src/Backups/BackupCoordinationLocal.cpp
@@ -33,42 +33,42 @@ Strings BackupCoordinationLocal::waitForStage(const String &, std::chrono::milli
return {};
}
-void BackupCoordinationLocal::addReplicatedPartNames(const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums)
+void BackupCoordinationLocal::addReplicatedPartNames(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums)
{
std::lock_guard lock{replicated_tables_mutex};
- replicated_tables.addPartNames({table_shared_id, table_name_for_logs, replica_name, part_names_and_checksums});
+ replicated_tables.addPartNames({table_zk_path, table_name_for_logs, replica_name, part_names_and_checksums});
}
-Strings BackupCoordinationLocal::getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const
+Strings BackupCoordinationLocal::getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const
{
std::lock_guard lock{replicated_tables_mutex};
- return replicated_tables.getPartNames(table_shared_id, replica_name);
+ return replicated_tables.getPartNames(table_zk_path, replica_name);
}
-void BackupCoordinationLocal::addReplicatedMutations(const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & mutations)
+void BackupCoordinationLocal::addReplicatedMutations(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name, const std::vector & mutations)
{
std::lock_guard lock{replicated_tables_mutex};
- replicated_tables.addMutations({table_shared_id, table_name_for_logs, replica_name, mutations});
+ replicated_tables.addMutations({table_zk_path, table_name_for_logs, replica_name, mutations});
}
-std::vector BackupCoordinationLocal::getReplicatedMutations(const String & table_shared_id, const String & replica_name) const
+std::vector BackupCoordinationLocal::getReplicatedMutations(const String & table_zk_path, const String & replica_name) const
{
std::lock_guard lock{replicated_tables_mutex};
- return replicated_tables.getMutations(table_shared_id, replica_name);
+ return replicated_tables.getMutations(table_zk_path, replica_name);
}
-void BackupCoordinationLocal::addReplicatedDataPath(const String & table_shared_id, const String & data_path)
+void BackupCoordinationLocal::addReplicatedDataPath(const String & table_zk_path, const String & data_path)
{
std::lock_guard lock{replicated_tables_mutex};
- replicated_tables.addDataPath({table_shared_id, data_path});
+ replicated_tables.addDataPath({table_zk_path, data_path});
}
-Strings BackupCoordinationLocal::getReplicatedDataPaths(const String & table_shared_id) const
+Strings BackupCoordinationLocal::getReplicatedDataPaths(const String & table_zk_path) const
{
std::lock_guard lock{replicated_tables_mutex};
- return replicated_tables.getDataPaths(table_shared_id);
+ return replicated_tables.getDataPaths(table_zk_path);
}
diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h
index e0aa5dc67a4..a7f15c79649 100644
--- a/src/Backups/BackupCoordinationLocal.h
+++ b/src/Backups/BackupCoordinationLocal.h
@@ -29,16 +29,16 @@ public:
Strings waitForStage(const String & stage_to_wait) override;
Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;
- void addReplicatedPartNames(const String & table_shared_id, const String & table_name_for_logs, const String & replica_name,
+ void addReplicatedPartNames(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name,
const std::vector & part_names_and_checksums) override;
- Strings getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const override;
+ Strings getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const override;
- void addReplicatedMutations(const String & table_shared_id, const String & table_name_for_logs, const String & replica_name,
+ void addReplicatedMutations(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name,
const std::vector & mutations) override;
- std::vector getReplicatedMutations(const String & table_shared_id, const String & replica_name) const override;
+ std::vector getReplicatedMutations(const String & table_zk_path, const String & replica_name) const override;
- void addReplicatedDataPath(const String & table_shared_id, const String & data_path) override;
- Strings getReplicatedDataPaths(const String & table_shared_id) const override;
+ void addReplicatedDataPath(const String & table_zk_path, const String & data_path) override;
+ Strings getReplicatedDataPaths(const String & table_zk_path) const override;
void addReplicatedAccessFilePath(const String & access_zk_path, AccessEntityType access_entity_type, const String & file_path) override;
Strings getReplicatedAccessFilePaths(const String & access_zk_path, AccessEntityType access_entity_type) const override;
diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp
index 455f45a7a77..f353062f628 100644
--- a/src/Backups/BackupCoordinationRemote.cpp
+++ b/src/Backups/BackupCoordinationRemote.cpp
@@ -358,7 +358,7 @@ String BackupCoordinationRemote::deserializeFromMultipleZooKeeperNodes(const Str
void BackupCoordinationRemote::addReplicatedPartNames(
- const String & table_shared_id,
+ const String & table_zk_path,
const String & table_name_for_logs,
const String & replica_name,
const std::vector & part_names_and_checksums)
@@ -374,22 +374,22 @@ void BackupCoordinationRemote::addReplicatedPartNames(
[&, &zk = holder.faulty_zookeeper]()
{
with_retries.renewZooKeeper(zk);
- String path = zookeeper_path + "/repl_part_names/" + escapeForFileName(table_shared_id);
+ String path = zookeeper_path + "/repl_part_names/" + escapeForFileName(table_zk_path);
zk->createIfNotExists(path, "");
path += "/" + escapeForFileName(replica_name);
zk->createIfNotExists(path, ReplicatedPartNames::serialize(part_names_and_checksums, table_name_for_logs));
});
}
-Strings BackupCoordinationRemote::getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const
+Strings BackupCoordinationRemote::getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const
{
std::lock_guard lock{replicated_tables_mutex};
prepareReplicatedTables();
- return replicated_tables->getPartNames(table_shared_id, replica_name);
+ return replicated_tables->getPartNames(table_zk_path, replica_name);
}
void BackupCoordinationRemote::addReplicatedMutations(
- const String & table_shared_id,
+ const String & table_zk_path,
const String & table_name_for_logs,
const String & replica_name,
const std::vector & mutations)
@@ -405,23 +405,23 @@ void BackupCoordinationRemote::addReplicatedMutations(
[&, &zk = holder.faulty_zookeeper]()
{
with_retries.renewZooKeeper(zk);
- String path = zookeeper_path + "/repl_mutations/" + escapeForFileName(table_shared_id);
+ String path = zookeeper_path + "/repl_mutations/" + escapeForFileName(table_zk_path);
zk->createIfNotExists(path, "");
path += "/" + escapeForFileName(replica_name);
zk->createIfNotExists(path, ReplicatedMutations::serialize(mutations, table_name_for_logs));
});
}
-std::vector BackupCoordinationRemote::getReplicatedMutations(const String & table_shared_id, const String & replica_name) const
+std::vector BackupCoordinationRemote::getReplicatedMutations(const String & table_zk_path, const String & replica_name) const
{
std::lock_guard lock{replicated_tables_mutex};
prepareReplicatedTables();
- return replicated_tables->getMutations(table_shared_id, replica_name);
+ return replicated_tables->getMutations(table_zk_path, replica_name);
}
void BackupCoordinationRemote::addReplicatedDataPath(
- const String & table_shared_id, const String & data_path)
+ const String & table_zk_path, const String & data_path)
{
{
std::lock_guard lock{replicated_tables_mutex};
@@ -434,18 +434,18 @@ void BackupCoordinationRemote::addReplicatedDataPath(
[&, &zk = holder.faulty_zookeeper]()
{
with_retries.renewZooKeeper(zk);
- String path = zookeeper_path + "/repl_data_paths/" + escapeForFileName(table_shared_id);
+ String path = zookeeper_path + "/repl_data_paths/" + escapeForFileName(table_zk_path);
zk->createIfNotExists(path, "");
path += "/" + escapeForFileName(data_path);
zk->createIfNotExists(path, "");
});
}
-Strings BackupCoordinationRemote::getReplicatedDataPaths(const String & table_shared_id) const
+Strings BackupCoordinationRemote::getReplicatedDataPaths(const String & table_zk_path) const
{
std::lock_guard lock{replicated_tables_mutex};
prepareReplicatedTables();
- return replicated_tables->getDataPaths(table_shared_id);
+ return replicated_tables->getDataPaths(table_zk_path);
}
@@ -464,16 +464,16 @@ void BackupCoordinationRemote::prepareReplicatedTables() const
with_retries.renewZooKeeper(zk);
String path = zookeeper_path + "/repl_part_names";
- for (const String & escaped_table_shared_id : zk->getChildren(path))
+ for (const String & escaped_table_zk_path : zk->getChildren(path))
{
- String table_shared_id = unescapeForFileName(escaped_table_shared_id);
- String path2 = path + "/" + escaped_table_shared_id;
+ String table_zk_path = unescapeForFileName(escaped_table_zk_path);
+ String path2 = path + "/" + escaped_table_zk_path;
for (const String & escaped_replica_name : zk->getChildren(path2))
{
String replica_name = unescapeForFileName(escaped_replica_name);
auto part_names = ReplicatedPartNames::deserialize(zk->get(path2 + "/" + escaped_replica_name));
part_names_for_replicated_tables.push_back(
- {table_shared_id, part_names.table_name_for_logs, replica_name, part_names.part_names_and_checksums});
+ {table_zk_path, part_names.table_name_for_logs, replica_name, part_names.part_names_and_checksums});
}
}
});
@@ -489,16 +489,16 @@ void BackupCoordinationRemote::prepareReplicatedTables() const
with_retries.renewZooKeeper(zk);
String path = zookeeper_path + "/repl_mutations";
- for (const String & escaped_table_shared_id : zk->getChildren(path))
+ for (const String & escaped_table_zk_path : zk->getChildren(path))
{
- String table_shared_id = unescapeForFileName(escaped_table_shared_id);
- String path2 = path + "/" + escaped_table_shared_id;
+ String table_zk_path = unescapeForFileName(escaped_table_zk_path);
+ String path2 = path + "/" + escaped_table_zk_path;
for (const String & escaped_replica_name : zk->getChildren(path2))
{
String replica_name = unescapeForFileName(escaped_replica_name);
auto mutations = ReplicatedMutations::deserialize(zk->get(path2 + "/" + escaped_replica_name));
mutations_for_replicated_tables.push_back(
- {table_shared_id, mutations.table_name_for_logs, replica_name, mutations.mutations});
+ {table_zk_path, mutations.table_name_for_logs, replica_name, mutations.mutations});
}
}
});
@@ -514,14 +514,14 @@ void BackupCoordinationRemote::prepareReplicatedTables() const
with_retries.renewZooKeeper(zk);
String path = zookeeper_path + "/repl_data_paths";
- for (const String & escaped_table_shared_id : zk->getChildren(path))
+ for (const String & escaped_table_zk_path : zk->getChildren(path))
{
- String table_shared_id = unescapeForFileName(escaped_table_shared_id);
- String path2 = path + "/" + escaped_table_shared_id;
+ String table_zk_path = unescapeForFileName(escaped_table_zk_path);
+ String path2 = path + "/" + escaped_table_zk_path;
for (const String & escaped_data_path : zk->getChildren(path2))
{
String data_path = unescapeForFileName(escaped_data_path);
- data_paths_for_replicated_tables.push_back({table_shared_id, data_path});
+ data_paths_for_replicated_tables.push_back({table_zk_path, data_path});
}
}
});
diff --git a/src/Backups/BackupCoordinationRemote.h b/src/Backups/BackupCoordinationRemote.h
index ce891699bd2..7a56b1a4eb8 100644
--- a/src/Backups/BackupCoordinationRemote.h
+++ b/src/Backups/BackupCoordinationRemote.h
@@ -41,23 +41,23 @@ public:
Strings waitForStage(const String & stage_to_wait, std::chrono::milliseconds timeout) override;
void addReplicatedPartNames(
- const String & table_shared_id,
+ const String & table_zk_path,
const String & table_name_for_logs,
const String & replica_name,
const std::vector & part_names_and_checksums) override;
- Strings getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const override;
+ Strings getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const override;
void addReplicatedMutations(
- const String & table_shared_id,
+ const String & table_zk_path,
const String & table_name_for_logs,
const String & replica_name,
const std::vector & mutations) override;
- std::vector getReplicatedMutations(const String & table_shared_id, const String & replica_name) const override;
+ std::vector getReplicatedMutations(const String & table_zk_path, const String & replica_name) const override;
- void addReplicatedDataPath(const String & table_shared_id, const String & data_path) override;
- Strings getReplicatedDataPaths(const String & table_shared_id) const override;
+ void addReplicatedDataPath(const String & table_zk_path, const String & data_path) override;
+ Strings getReplicatedDataPaths(const String & table_zk_path) const override;
void addReplicatedAccessFilePath(const String & access_zk_path, AccessEntityType access_entity_type, const String & file_path) override;
Strings getReplicatedAccessFilePaths(const String & access_zk_path, AccessEntityType access_entity_type) const override;
diff --git a/src/Backups/BackupCoordinationReplicatedTables.cpp b/src/Backups/BackupCoordinationReplicatedTables.cpp
index 1cbb88acb82..a435667f79a 100644
--- a/src/Backups/BackupCoordinationReplicatedTables.cpp
+++ b/src/Backups/BackupCoordinationReplicatedTables.cpp
@@ -151,7 +151,7 @@ BackupCoordinationReplicatedTables::~BackupCoordinationReplicatedTables() = defa
void BackupCoordinationReplicatedTables::addPartNames(PartNamesForTableReplica && part_names)
{
- const auto & table_shared_id = part_names.table_shared_id;
+ const auto & table_zk_path = part_names.table_zk_path;
const auto & table_name_for_logs = part_names.table_name_for_logs;
const auto & replica_name = part_names.replica_name;
const auto & part_names_and_checksums = part_names.part_names_and_checksums;
@@ -159,7 +159,7 @@ void BackupCoordinationReplicatedTables::addPartNames(PartNamesForTableReplica &
if (prepared)
throw Exception(ErrorCodes::LOGICAL_ERROR, "addPartNames() must not be called after preparing");
- auto & table_info = table_infos[table_shared_id];
+ auto & table_info = table_infos[table_zk_path];
table_info.table_name_for_logs = table_name_for_logs;
if (!table_info.covered_parts_finder)
@@ -200,11 +200,11 @@ void BackupCoordinationReplicatedTables::addPartNames(PartNamesForTableReplica &
}
}
-Strings BackupCoordinationReplicatedTables::getPartNames(const String & table_shared_id, const String & replica_name) const
+Strings BackupCoordinationReplicatedTables::getPartNames(const String & table_zk_path, const String & replica_name) const
{
prepare();
- auto it = table_infos.find(table_shared_id);
+ auto it = table_infos.find(table_zk_path);
if (it == table_infos.end())
return {};
@@ -218,7 +218,7 @@ Strings BackupCoordinationReplicatedTables::getPartNames(const String & table_sh
void BackupCoordinationReplicatedTables::addMutations(MutationsForTableReplica && mutations_for_table_replica)
{
- const auto & table_shared_id = mutations_for_table_replica.table_shared_id;
+ const auto & table_zk_path = mutations_for_table_replica.table_zk_path;
const auto & table_name_for_logs = mutations_for_table_replica.table_name_for_logs;
const auto & replica_name = mutations_for_table_replica.replica_name;
const auto & mutations = mutations_for_table_replica.mutations;
@@ -226,7 +226,7 @@ void BackupCoordinationReplicatedTables::addMutations(MutationsForTableReplica &
if (prepared)
throw Exception(ErrorCodes::LOGICAL_ERROR, "addMutations() must not be called after preparing");
- auto & table_info = table_infos[table_shared_id];
+ auto & table_info = table_infos[table_zk_path];
table_info.table_name_for_logs = table_name_for_logs;
for (const auto & [mutation_id, mutation_entry] : mutations)
table_info.mutations.emplace(mutation_id, mutation_entry);
@@ -236,11 +236,11 @@ void BackupCoordinationReplicatedTables::addMutations(MutationsForTableReplica &
}
std::vector
-BackupCoordinationReplicatedTables::getMutations(const String & table_shared_id, const String & replica_name) const
+BackupCoordinationReplicatedTables::getMutations(const String & table_zk_path, const String & replica_name) const
{
prepare();
- auto it = table_infos.find(table_shared_id);
+ auto it = table_infos.find(table_zk_path);
if (it == table_infos.end())
return {};
@@ -257,16 +257,16 @@ BackupCoordinationReplicatedTables::getMutations(const String & table_shared_id,
void BackupCoordinationReplicatedTables::addDataPath(DataPathForTableReplica && data_path_for_table_replica)
{
- const auto & table_shared_id = data_path_for_table_replica.table_shared_id;
+ const auto & table_zk_path = data_path_for_table_replica.table_zk_path;
const auto & data_path = data_path_for_table_replica.data_path;
- auto & table_info = table_infos[table_shared_id];
+ auto & table_info = table_infos[table_zk_path];
table_info.data_paths.emplace(data_path);
}
-Strings BackupCoordinationReplicatedTables::getDataPaths(const String & table_shared_id) const
+Strings BackupCoordinationReplicatedTables::getDataPaths(const String & table_zk_path) const
{
- auto it = table_infos.find(table_shared_id);
+ auto it = table_infos.find(table_zk_path);
if (it == table_infos.end())
return {};
diff --git a/src/Backups/BackupCoordinationReplicatedTables.h b/src/Backups/BackupCoordinationReplicatedTables.h
index 74f21eb9c7c..50ab56aef75 100644
--- a/src/Backups/BackupCoordinationReplicatedTables.h
+++ b/src/Backups/BackupCoordinationReplicatedTables.h
@@ -40,7 +40,7 @@ public:
struct PartNamesForTableReplica
{
- String table_shared_id;
+ String table_zk_path;
String table_name_for_logs;
String replica_name;
std::vector part_names_and_checksums;
@@ -55,13 +55,13 @@ public:
/// Returns the names of the parts which a specified replica of a replicated table should put to the backup.
/// This is the same list as it was added by call of the function addPartNames() but without duplications and without
/// parts covered by another parts.
- Strings getPartNames(const String & table_shared_id, const String & replica_name) const;
+ Strings getPartNames(const String & table_zk_path, const String & replica_name) const;
using MutationInfo = IBackupCoordination::MutationInfo;
struct MutationsForTableReplica
{
- String table_shared_id;
+ String table_zk_path;
String table_name_for_logs;
String replica_name;
std::vector mutations;
@@ -71,11 +71,11 @@ public:
void addMutations(MutationsForTableReplica && mutations_for_table_replica);
/// Returns all mutations of a replicated table which are not finished for some data parts added by addReplicatedPartNames().
- std::vector getMutations(const String & table_shared_id, const String & replica_name) const;
+ std::vector getMutations(const String & table_zk_path, const String & replica_name) const;
struct DataPathForTableReplica
{
- String table_shared_id;
+ String table_zk_path;
String data_path;
};
@@ -85,7 +85,7 @@ public:
void addDataPath(DataPathForTableReplica && data_path_for_table_replica);
/// Returns all the data paths in backup added for a replicated table (see also addReplicatedDataPath()).
- Strings getDataPaths(const String & table_shared_id) const;
+ Strings getDataPaths(const String & table_zk_path) const;
private:
void prepare() const;
@@ -110,7 +110,7 @@ private:
std::unordered_set data_paths;
};
- std::map table_infos; /// Should be ordered because we need this map to be in the same order on every replica.
+ std::map table_infos; /// Should be ordered because we need this map to be in the same order on every replica.
mutable bool prepared = false;
};
diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp
index cc014c279cc..d91cf47c4d3 100644
--- a/src/Backups/BackupEntriesCollector.cpp
+++ b/src/Backups/BackupEntriesCollector.cpp
@@ -11,6 +11,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -758,7 +759,7 @@ void BackupEntriesCollector::makeBackupEntriesForDatabasesDefs()
checkIsQueryCancelled();
ASTPtr new_create_query = database_info.create_database_query;
- adjustCreateQueryForBackup(new_create_query, context->getGlobalContext(), nullptr);
+ adjustCreateQueryForBackup(new_create_query, context->getGlobalContext());
renameDatabaseAndTableNameInCreateQuery(new_create_query, renaming_map, context->getGlobalContext());
const String & metadata_path_in_backup = database_info.metadata_path_in_backup;
@@ -775,7 +776,8 @@ void BackupEntriesCollector::makeBackupEntriesForTablesDefs()
checkIsQueryCancelled();
ASTPtr new_create_query = table_info.create_table_query;
- adjustCreateQueryForBackup(new_create_query, context->getGlobalContext(), &table_info.replicated_table_shared_id);
+ table_info.replicated_table_zk_path = extractZooKeeperPathFromReplicatedTableDef(new_create_query->as(), context);
+ adjustCreateQueryForBackup(new_create_query, context->getGlobalContext());
renameDatabaseAndTableNameInCreateQuery(new_create_query, renaming_map, context->getGlobalContext());
const String & metadata_path_in_backup = table_info.metadata_path_in_backup;
@@ -814,8 +816,8 @@ void BackupEntriesCollector::makeBackupEntriesForTableData(const QualifiedTableN
/// If this table is replicated in this case we call IBackupCoordination::addReplicatedDataPath() which will cause
/// other replicas to fill the storage's data in the backup.
/// If this table is not replicated we'll do nothing leaving the storage's data empty in the backup.
- if (table_info.replicated_table_shared_id)
- backup_coordination->addReplicatedDataPath(*table_info.replicated_table_shared_id, data_path_in_backup);
+ if (table_info.replicated_table_zk_path)
+ backup_coordination->addReplicatedDataPath(*table_info.replicated_table_zk_path, data_path_in_backup);
return;
}
diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h
index 01e8d594334..c7bce077a2d 100644
--- a/src/Backups/BackupEntriesCollector.h
+++ b/src/Backups/BackupEntriesCollector.h
@@ -164,7 +164,7 @@ private:
ASTPtr create_table_query;
String metadata_path_in_backup;
std::filesystem::path data_path_in_backup;
- std::optional replicated_table_shared_id;
+ std::optional replicated_table_zk_path;
std::optional partitions;
};
diff --git a/src/Backups/BackupUtils.cpp b/src/Backups/BackupUtils.cpp
index fb448fb64ad..fa8ed5855dd 100644
--- a/src/Backups/BackupUtils.cpp
+++ b/src/Backups/BackupUtils.cpp
@@ -103,7 +103,7 @@ bool compareRestoredTableDef(const IAST & restored_table_create_query, const IAS
auto adjust_before_comparison = [&](const IAST & query) -> ASTPtr
{
auto new_query = query.clone();
- adjustCreateQueryForBackup(new_query, global_context, nullptr);
+ adjustCreateQueryForBackup(new_query, global_context);
ASTCreateQuery & create = typeid_cast(*new_query);
create.setUUID({});
create.if_not_exists = false;
diff --git a/src/Backups/DDLAdjustingForBackupVisitor.cpp b/src/Backups/DDLAdjustingForBackupVisitor.cpp
index 5ea91094b75..7e5ce91629b 100644
--- a/src/Backups/DDLAdjustingForBackupVisitor.cpp
+++ b/src/Backups/DDLAdjustingForBackupVisitor.cpp
@@ -27,9 +27,6 @@ namespace
{
/// Precondition: engine_name.starts_with("Replicated") && engine_name.ends_with("MergeTree")
- if (data.replicated_table_shared_id)
- *data.replicated_table_shared_id = StorageReplicatedMergeTree::tryGetTableSharedIDFromCreateQuery(*data.create_query, data.global_context);
-
/// Before storing the metadata in a backup we have to find a zookeeper path in its definition and turn the table's UUID in there
/// back into "{uuid}", and also we probably can remove the zookeeper path and replica name if they're default.
/// So we're kind of reverting what we had done to the table's definition in registerStorageMergeTree.cpp before we created this table.
@@ -98,12 +95,9 @@ void DDLAdjustingForBackupVisitor::visit(ASTPtr ast, const Data & data)
visitCreateQuery(*create, data);
}
-void adjustCreateQueryForBackup(ASTPtr ast, const ContextPtr & global_context, std::optional * replicated_table_shared_id)
+void adjustCreateQueryForBackup(ASTPtr ast, const ContextPtr & global_context)
{
- if (replicated_table_shared_id)
- *replicated_table_shared_id = {};
-
- DDLAdjustingForBackupVisitor::Data data{ast, global_context, replicated_table_shared_id};
+ DDLAdjustingForBackupVisitor::Data data{ast, global_context};
DDLAdjustingForBackupVisitor::Visitor{data}.visit(ast);
}
diff --git a/src/Backups/DDLAdjustingForBackupVisitor.h b/src/Backups/DDLAdjustingForBackupVisitor.h
index 63353dcc000..f0508434e02 100644
--- a/src/Backups/DDLAdjustingForBackupVisitor.h
+++ b/src/Backups/DDLAdjustingForBackupVisitor.h
@@ -12,9 +12,7 @@ class Context;
using ContextPtr = std::shared_ptr;
/// Changes a create query to a form which is appropriate or suitable for saving in a backup.
-/// Also extracts a replicated table's shared ID from the create query if this is a create query for a replicated table.
-/// `replicated_table_shared_id` can be null if you don't need that.
-void adjustCreateQueryForBackup(ASTPtr ast, const ContextPtr & global_context, std::optional * replicated_table_shared_id);
+void adjustCreateQueryForBackup(ASTPtr ast, const ContextPtr & global_context);
/// Visits ASTCreateQuery and changes it to a form which is appropriate or suitable for saving in a backup.
class DDLAdjustingForBackupVisitor
@@ -24,7 +22,6 @@ public:
{
ASTPtr create_query;
ContextPtr global_context;
- std::optional * replicated_table_shared_id = nullptr;
};
using Visitor = InDepthNodeVisitor;
diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h
index f80b5dee883..4a9f8a23855 100644
--- a/src/Backups/IBackupCoordination.h
+++ b/src/Backups/IBackupCoordination.h
@@ -36,13 +36,13 @@ public:
/// Multiple replicas of the replicated table call this function and then the added part names can be returned by call of the function
/// getReplicatedPartNames().
/// Checksums are used only to control that parts under the same names on different replicas are the same.
- virtual void addReplicatedPartNames(const String & table_shared_id, const String & table_name_for_logs, const String & replica_name,
+ virtual void addReplicatedPartNames(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name,
const std::vector & part_names_and_checksums) = 0;
/// Returns the names of the parts which a specified replica of a replicated table should put to the backup.
/// This is the same list as it was added by call of the function addReplicatedPartNames() but without duplications and without
/// parts covered by another parts.
- virtual Strings getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const = 0;
+ virtual Strings getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const = 0;
struct MutationInfo
{
@@ -51,10 +51,10 @@ public:
};
/// Adds information about mutations of a replicated table.
- virtual void addReplicatedMutations(const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & mutations) = 0;
+ virtual void addReplicatedMutations(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name, const std::vector & mutations) = 0;
/// Returns all mutations of a replicated table which are not finished for some data parts added by addReplicatedPartNames().
- virtual std::vector getReplicatedMutations(const String & table_shared_id, const String & replica_name) const = 0;
+ virtual std::vector getReplicatedMutations(const String & table_zk_path, const String & replica_name) const = 0;
/// Adds information about KeeperMap tables
virtual void addKeeperMapTable(const String & table_zookeeper_root_path, const String & table_id, const String & data_path_in_backup) = 0;
@@ -65,10 +65,10 @@ public:
/// Adds a data path in backup for a replicated table.
/// Multiple replicas of the replicated table call this function and then all the added paths can be returned by call of the function
/// getReplicatedDataPaths().
- virtual void addReplicatedDataPath(const String & table_shared_id, const String & data_path) = 0;
+ virtual void addReplicatedDataPath(const String & table_zk_path, const String & data_path) = 0;
/// Returns all the data paths in backup added for a replicated table (see also addReplicatedDataPath()).
- virtual Strings getReplicatedDataPaths(const String & table_shared_id) const = 0;
+ virtual Strings getReplicatedDataPaths(const String & table_zk_path) const = 0;
/// Adds a path to access.txt file keeping access entities of a ReplicatedAccessStorage.
virtual void addReplicatedAccessFilePath(const String & access_zk_path, AccessEntityType access_entity_type, const String & file_path) = 0;
diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 46887d6cefe..402195ca944 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -2616,7 +2616,7 @@ void ClientBase::runInteractive()
{
// If a separate connection loading suggestions failed to open a new session,
// use the main session to receive them.
- suggest->load(*connection, connection_parameters.timeouts, config().getInt("suggestion_limit"));
+ suggest->load(*connection, connection_parameters.timeouts, config().getInt("suggestion_limit"), global_context->getClientInfo());
}
try
diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp
index 483201509c4..30f16d26443 100644
--- a/src/Client/Connection.cpp
+++ b/src/Client/Connection.cpp
@@ -34,6 +34,7 @@
#include
#include
#include
+#include
#include
#include "config.h"
@@ -51,6 +52,11 @@ namespace CurrentMetrics
namespace DB
{
+namespace FailPoints
+{
+ extern const char receive_timeout_on_table_status_response[];
+}
+
namespace ErrorCodes
{
extern const int NETWORK_ERROR;
@@ -607,6 +613,11 @@ TablesStatusResponse Connection::getTablesStatus(const ConnectionTimeouts & time
if (!connected)
connect(timeouts);
+ fiu_do_on(FailPoints::receive_timeout_on_table_status_response, {
+ sleepForSeconds(5);
+ throw NetException(ErrorCodes::SOCKET_TIMEOUT, "Injected timeout exceeded while reading from socket ({}:{})", host, port);
+ });
+
TimeoutSetter timeout_setter(*socket, timeouts.sync_request_timeout, true);
writeVarUInt(Protocol::Client::TablesStatusRequest, *out);
diff --git a/src/Client/Suggest.cpp b/src/Client/Suggest.cpp
index 736967f4a68..487271a2cf9 100644
--- a/src/Client/Suggest.cpp
+++ b/src/Client/Suggest.cpp
@@ -99,7 +99,10 @@ void Suggest::load(ContextPtr context, const ConnectionParameters & connection_p
try
{
auto connection = ConnectionType::createConnection(connection_parameters, my_context);
- fetch(*connection, connection_parameters.timeouts, getLoadSuggestionQuery(suggestion_limit, std::is_same_v));
+ fetch(*connection,
+ connection_parameters.timeouts,
+ getLoadSuggestionQuery(suggestion_limit, std::is_same_v),
+ my_context->getClientInfo());
}
catch (const Exception & e)
{
@@ -138,11 +141,12 @@ void Suggest::load(ContextPtr context, const ConnectionParameters & connection_p
void Suggest::load(IServerConnection & connection,
const ConnectionTimeouts & timeouts,
- Int32 suggestion_limit)
+ Int32 suggestion_limit,
+ const ClientInfo & client_info)
{
try
{
- fetch(connection, timeouts, getLoadSuggestionQuery(suggestion_limit, true));
+ fetch(connection, timeouts, getLoadSuggestionQuery(suggestion_limit, true), client_info);
}
catch (...)
{
@@ -151,10 +155,10 @@ void Suggest::load(IServerConnection & connection,
}
}
-void Suggest::fetch(IServerConnection & connection, const ConnectionTimeouts & timeouts, const std::string & query)
+void Suggest::fetch(IServerConnection & connection, const ConnectionTimeouts & timeouts, const std::string & query, const ClientInfo & client_info)
{
connection.sendQuery(
- timeouts, query, {} /* query_parameters */, "" /* query_id */, QueryProcessingStage::Complete, nullptr, nullptr, false, {});
+ timeouts, query, {} /* query_parameters */, "" /* query_id */, QueryProcessingStage::Complete, nullptr, &client_info, false, {});
while (true)
{
diff --git a/src/Client/Suggest.h b/src/Client/Suggest.h
index d04910f4d00..f1ba36c7f1c 100644
--- a/src/Client/Suggest.h
+++ b/src/Client/Suggest.h
@@ -31,7 +31,8 @@ public:
void load(IServerConnection & connection,
const ConnectionTimeouts & timeouts,
- Int32 suggestion_limit);
+ Int32 suggestion_limit,
+ const ClientInfo & client_info);
/// Older server versions cannot execute the query loading suggestions.
static constexpr int MIN_SERVER_REVISION = DBMS_MIN_PROTOCOL_VERSION_WITH_VIEW_IF_PERMITTED;
@@ -39,7 +40,7 @@ public:
int getLastError() const { return last_error.load(); }
private:
- void fetch(IServerConnection & connection, const ConnectionTimeouts & timeouts, const std::string & query);
+ void fetch(IServerConnection & connection, const ConnectionTimeouts & timeouts, const std::string & query, const ClientInfo & client_info);
void fillWordsFromBlock(const Block & block);
diff --git a/src/Common/ElapsedTimeProfileEventIncrement.h b/src/Common/ElapsedTimeProfileEventIncrement.h
index 731295a4cfd..aa944beeaa9 100644
--- a/src/Common/ElapsedTimeProfileEventIncrement.h
+++ b/src/Common/ElapsedTimeProfileEventIncrement.h
@@ -17,19 +17,18 @@ enum Time
template