Merge branch 'master' into keeper-preprocess-operations

.clang-tidy

@@ -5,38 +5,21 @@ Checks: '*,
 
     -android-*,
 
-    -bugprone-assert-side-effect,
     -bugprone-branch-clone,
-    -bugprone-dynamic-static-initializers,
     -bugprone-easily-swappable-parameters,
     -bugprone-exception-escape,
-    -bugprone-forwarding-reference-overload,
     -bugprone-implicit-widening-of-multiplication-result,
     -bugprone-lambda-function-name,
-    -bugprone-misplaced-widening-cast,
     -bugprone-narrowing-conversions,
-    -bugprone-no-escape,
     -bugprone-not-null-terminated-result,
-    -bugprone-signal-handler,
-    -bugprone-spuriously-wake-up-functions,
-    -bugprone-suspicious-semicolon,
-    -bugprone-unhandled-exception-at-new,
     -bugprone-unhandled-self-assignment,
 
     -cert-dcl16-c,
-    -cert-dcl37-c,
-    -cert-dcl51-cpp,
-    -cert-dcl58-cpp,
     -cert-err58-cpp,
-    -cert-err60-cpp,
     -cert-msc32-c,
     -cert-msc51-cpp,
     -cert-oop54-cpp,
     -cert-oop57-cpp,
-    -cert-oop58-cpp,
-
-    -clang-analyzer-core.DynamicTypePropagation,
-    -clang-analyzer-core.uninitialized.CapturedBlockVariable,
 
     -clang-analyzer-optin.performance.Padding,
     -clang-analyzer-optin.portability.UnixAPI,
@@ -53,7 +36,6 @@ Checks: '*,
     -fuchsia-*,
 
     -google-build-using-namespace,
-    -google-global-names-in-headers,
     -google-readability-braces-around-statements,
     -google-readability-function-size,
     -google-readability-namespace-comments,
@@ -63,7 +45,6 @@ Checks: '*,
     -hicpp-avoid-c-arrays,
     -hicpp-avoid-goto,
     -hicpp-braces-around-statements,
-    -hicpp-deprecated-headers,
     -hicpp-explicit-conversions,
     -hicpp-function-size,
     -hicpp-invalid-access-moved,
@@ -79,7 +60,6 @@ Checks: '*,
     -hicpp-uppercase-literal-suffix,
     -hicpp-use-auto,
     -hicpp-use-emplace,
-    -hicpp-use-equals-default,
     -hicpp-use-noexcept,
     -hicpp-use-override,
     -hicpp-vararg,
@@ -90,40 +70,27 @@ Checks: '*,
 
     -openmp-*,
 
-    -misc-definitions-in-headers,
-    -misc-new-delete-overloads,
     -misc-no-recursion,
-    -misc-non-copyable-objects,
     -misc-non-private-member-variables-in-classes,
-    -misc-static-assert,
 
     -modernize-avoid-c-arrays,
     -modernize-concat-nested-namespaces,
-    -modernize-deprecated-headers,
-    -modernize-deprecated-ios-base-aliases,
     -modernize-pass-by-value,
-    -modernize-replace-auto-ptr,
-    -modernize-replace-disallow-copy-and-assign-macro,
     -modernize-return-braced-init-list,
-    -modernize-unary-static-assert,
     -modernize-use-auto,
     -modernize-use-default-member-init,
     -modernize-use-emplace,
-    -modernize-use-equals-default,
     -modernize-use-nodiscard,
-    -modernize-use-noexcept,
     -modernize-use-override,
     -modernize-use-trailing-return-type,
 
     -performance-inefficient-string-concatenation,
     -performance-no-int-to-ptr,
     -performance-type-promotion-in-math-fn,
-    -performance-trivially-destructible,
     -performance-unnecessary-value-param,
 
     -portability-simd-intrinsics,
 
-    -readability-convert-member-functions-to-static,
     -readability-braces-around-statements,
     -readability-else-after-return,
     -readability-function-cognitive-complexity,
@@ -131,9 +98,7 @@ Checks: '*,
     -readability-implicit-bool-conversion,
     -readability-isolate-declaration,
     -readability-magic-numbers,
-    -readability-misleading-indentation,
     -readability-named-parameter,
-    -readability-qualified-auto,
     -readability-redundant-declaration,
     -readability-static-accessed-through-instance,
     -readability-suspicious-call-argument,

.gitmodules

@@ -265,3 +265,6 @@
 [submodule "contrib/wyhash"]
 	path = contrib/wyhash
 	url = https://github.com/wangyi-fudan/wyhash.git
+[submodule "contrib/eigen"]
+	path = contrib/eigen
+	url = https://github.com/eigen-mirror/eigen

CMakeLists.txt

@@ -1,32 +1,6 @@
 cmake_minimum_required(VERSION 3.14)
 
-foreach(policy
+project(ClickHouse LANGUAGES C CXX ASM)
-        CMP0023
-        CMP0048 # CMake 3.0
-        CMP0074 # CMake 3.12
-        CMP0077
-        CMP0079
-    )
-    if(POLICY ${policy})
-        cmake_policy(SET ${policy} NEW)
-    endif()
-endforeach()
-
-# set default policy
-foreach(default_policy_var_name
-        # make option() honor normal variables for BUILD_SHARED_LIBS:
-        # - re2
-        # - snappy
-        CMAKE_POLICY_DEFAULT_CMP0077
-        # Google Test from sources uses too old cmake, 2.6.x, and CMP0022 should
-        # set, to avoid using deprecated LINK_INTERFACE_LIBRARIES(_<CONFIG>)? over
-        # INTERFACE_LINK_LIBRARIES.
-        CMAKE_POLICY_DEFAULT_CMP0022
-    )
-    set(${default_policy_var_name} NEW)
-endforeach()
-
-project(ClickHouse)
 
 # If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue.
 option(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION
@@ -39,11 +13,10 @@ else()
     set(RECONFIGURE_MESSAGE_LEVEL WARNING)
 endif()
 
-enable_language(C CXX ASM)
-
 include (cmake/arch.cmake)
 include (cmake/target.cmake)
 include (cmake/tools.cmake)
+include (cmake/ccache.cmake)
 include (cmake/clang_tidy.cmake)
 include (cmake/git_status.cmake)
 
@@ -52,7 +25,6 @@ include (cmake/git_status.cmake)
 macro (export)
 endmacro ()
 
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/")
 set(CMAKE_EXPORT_COMPILE_COMMANDS 1) # Write compile_commands.json
 set(CMAKE_LINK_DEPENDS_NO_SHARED 1) # Do not relink all depended targets on .so
 set(CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel" CACHE STRING "" FORCE)
@@ -67,8 +39,6 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/sysroot/README.md")
     message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive")
 endif ()
 
-include (cmake/ccache.cmake)
-
 # Take care to add prlimit in command line before ccache, or else ccache thinks that
 # prlimit is compiler, and clang++ is its input file, and refuses to work  with
 # multiple inputs, e.g in ccache log:
@@ -161,20 +131,22 @@ add_library(global-libs INTERFACE)
 include (cmake/fuzzer.cmake)
 include (cmake/sanitize.cmake)
 
-if (CMAKE_GENERATOR STREQUAL "Ninja" AND NOT DISABLE_COLORED_BUILD)
+option(ENABLE_COLORED_BUILD "Enable colors in compiler output" ON)
+
+set (CMAKE_COLOR_MAKEFILE ${ENABLE_COLORED_BUILD}) # works only for the makefile generator
+
+if (ENABLE_COLORED_BUILD AND CMAKE_GENERATOR STREQUAL "Ninja")
     # Turn on colored output. https://github.com/ninja-build/ninja/wiki/FAQ
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always")
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always")
+    # ... such manually setting of flags can be removed once CMake supports a variable to
+    # activate colors in *all* build systems: https://gitlab.kitware.com/cmake/cmake/-/issues/15502
 endif ()
 
 include (cmake/check_flags.cmake)
 include (cmake/add_warning.cmake)
 
-set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall")    # -Werror and many more is also added inside cmake/warnings.cmake
-
 if (COMPILER_CLANG)
-    # clang: warning: argument unused during compilation: '-specs=/usr/share/dpkg/no-pie-compile.specs' [-Wunused-command-line-argument]
-    set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wno-unused-command-line-argument")
     # generate ranges for fast "addr2line" search
     if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
         set(COMPILER_FLAGS "${COMPILER_FLAGS} -gdwarf-aranges")
@@ -371,12 +343,11 @@ set (COMPILER_FLAGS "${COMPILER_FLAGS}")
 # Our built-in unwinder only supports DWARF version up to 4.
 set (DEBUG_INFO_FLAGS "-g -gdwarf-4")
 
-set (CMAKE_BUILD_COLOR_MAKEFILE          ON)
+set (CMAKE_CXX_FLAGS                     "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS}")
-set (CMAKE_CXX_FLAGS                     "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}")
 set (CMAKE_CXX_FLAGS_RELWITHDEBINFO      "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
 set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
 
-set (CMAKE_C_FLAGS                       "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}")
+set (CMAKE_C_FLAGS                       "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${CMAKE_C_FLAGS_ADD}")
 set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}")
 set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_C_FLAGS_ADD}")
 
@@ -423,13 +394,6 @@ endif ()
 # Turns on all external libs like s3, kafka, ODBC, ...
 option(ENABLE_LIBRARIES "Enable all external libraries by default" ON)
 
-if (NOT (OS_LINUX OR OS_DARWIN))
-    # Using system libs can cause a lot of warnings in includes (on macro expansion).
-    option(WERROR "Enable -Werror compiler option" OFF)
-else ()
-    option(WERROR "Enable -Werror compiler option" ON)
-endif ()
-
 # Increase stack size on Musl. We need big stack for our recursive-descend parser.
 if (USE_MUSL)
     set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-z,stack-size=2097152")
@@ -447,6 +411,13 @@ elseif (OS_FREEBSD)
 endif ()
 link_libraries(global-group)
 
+if (NOT (OS_LINUX OR OS_DARWIN))
+    # Using system libs can cause a lot of warnings in includes (on macro expansion).
+    option(WERROR "Enable -Werror compiler option" OFF)
+else ()
+    option(WERROR "Enable -Werror compiler option" ON)
+endif ()
+
 if (WERROR)
     # Don't pollute CMAKE_CXX_FLAGS with -Werror as it will break some CMake checks.
     # Instead, adopt modern cmake usage requirement.
@@ -455,7 +426,7 @@ endif ()
 
 # Make this extra-checks for correct library dependencies.
 if (OS_LINUX AND NOT SANITIZE)
-    target_link_options(global-group INTERFACE "-Wl,--no-undefined")
+    target_link_options(global-group INTERFACE "LINKER:--no-undefined")
 endif ()
 
 ######################################
@@ -466,7 +437,7 @@ set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
 
 if (USE_STATIC_LIBRARIES)
     set (CMAKE_POSITION_INDEPENDENT_CODE OFF)
-    if (OS_LINUX AND NOT ARCH_ARM)
+    if (OS_LINUX AND NOT ARCH_AARCH64)
         # Slightly more efficient code can be generated
         # It's disabled for ARM because otherwise ClickHouse cannot run on Android.
         set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-pie")
@@ -500,8 +471,7 @@ endif ()
 message (STATUS
     "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ;
     USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES}
-    SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES}
+    SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES}")
-    CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}")
 
 include (GNUInstallDirs)
 

base/base/JSON.cpp

@@ -1,5 +1,5 @@
 #include <string>
-#include <string.h>
+#include <cstring>
 
 #include <Poco/UTF8Encoding.h>
 #include <Poco/NumberParser.h>
@@ -12,7 +12,7 @@
 #define JSON_MAX_DEPTH 100
 
 
-POCO_IMPLEMENT_EXCEPTION(JSONException, Poco::Exception, "JSONException")
+POCO_IMPLEMENT_EXCEPTION(JSONException, Poco::Exception, "JSONException") // NOLINT(cert-err60-cpp, modernize-use-noexcept)
 
 
 /// Прочитать беззнаковое целое в простом формате из не-0-terminated строки.

base/base/LineReader.cpp

@@ -5,7 +5,7 @@
 #include <algorithm>
 
 #include <cassert>
-#include <string.h>
+#include <cstring>
 #include <unistd.h>
 #include <sys/select.h>
 #include <sys/time.h>

base/base/demangle.cpp

@@ -1,6 +1,6 @@
 #include <base/demangle.h>
 
-#include <stdlib.h>
+#include <cstdlib>
 #include <cxxabi.h>
 
 static DemangleResult tryDemangle(const char * name, int & status)

base/base/mremap.cpp

@@ -3,7 +3,7 @@
 #include <cstddef>
 #include <cstdlib>
 #include <cstring>
-#include <errno.h>
+#include <cerrno>
 
 
 void * mremap_fallback(

base/base/preciseExp10.cpp

@@ -169,9 +169,9 @@ obstacle to adoption, that text has been removed.
 
 */
 
-#include <math.h>
+#include <cmath>
-#include <stdint.h>
+#include <cstdint>
-#include <stdio.h>
+#include <cstdio>
 
 double preciseExp10(double x)
 {

base/base/sleep.cpp

@@ -1,7 +1,7 @@
 #include <base/sleep.h>
 
-#include <time.h>
+#include <ctime>
-#include <errno.h>
+#include <cerrno>
 
 #if defined(OS_DARWIN)
 #include <mach/mach.h>

base/base/unit.h

@@ -1,14 +1,19 @@
 #pragma once
 #include <cstddef>
 
-#ifdef HAS_RESERVED_IDENTIFIER
-#pragma clang diagnostic ignored "-Wreserved-identifier"
-#endif
-
 constexpr size_t KiB = 1024;
 constexpr size_t MiB = 1024 * KiB;
 constexpr size_t GiB = 1024 * MiB;
 
+#ifdef HAS_RESERVED_IDENTIFIER
+#  pragma clang diagnostic push
+#  pragma clang diagnostic ignored "-Wreserved-identifier"
+#endif
+
 constexpr size_t operator"" _KiB(unsigned long long val) { return val * KiB; }
 constexpr size_t operator"" _MiB(unsigned long long val) { return val * MiB; }
 constexpr size_t operator"" _GiB(unsigned long long val) { return val * GiB; }
+
+#ifdef HAS_RESERVED_IDENTIFIER
+#  pragma clang diagnostic pop
+#endif

cmake/arch.cmake

@@ -5,7 +5,6 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64")
     set (ARCH_AMD64 1)
 elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)")
     set (ARCH_AARCH64 1)
-    set (ARCH_ARM 1)
 elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le.*|PPC64LE.*)")
     set (ARCH_PPC64LE 1)
 elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")

cmake/ccache.cmake

@@ -1,4 +1,4 @@
-if (CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" OR CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache")
+if (CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" OR CMAKE_C_COMPILER_LAUNCHER MATCHES "ccache")
     set(COMPILER_MATCHES_CCACHE 1)
 else()
     set(COMPILER_MATCHES_CCACHE 0)
@@ -30,7 +30,7 @@ if (CCACHE_FOUND AND NOT COMPILER_MATCHES_CCACHE)
    string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION})
 
    if (CCACHE_VERSION VERSION_GREATER "3.2.0" OR NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-      message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}")
+      message(STATUS "Using ccache: ${CCACHE_FOUND} (version ${CCACHE_VERSION})")
       set(LAUNCHER ${CCACHE_FOUND})
 
       # debian (debhelpers) set SOURCE_DATE_EPOCH environment variable, that is
@@ -49,8 +49,8 @@ if (CCACHE_FOUND AND NOT COMPILER_MATCHES_CCACHE)
       set (CMAKE_CXX_COMPILER_LAUNCHER ${LAUNCHER} ${CMAKE_CXX_COMPILER_LAUNCHER})
       set (CMAKE_C_COMPILER_LAUNCHER ${LAUNCHER} ${CMAKE_C_COMPILER_LAUNCHER})
    else ()
-      message(${RECONFIGURE_MESSAGE_LEVEL} "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118")
+       message(${RECONFIGURE_MESSAGE_LEVEL} "Using ccache: No. Found ${CCACHE_FOUND} (version ${CCACHE_VERSION}) but disabled because of bug: https://bugzilla.samba.org/show_bug.cgi?id=8118")
    endif ()
 elseif (NOT CCACHE_FOUND AND NOT COMPILER_MATCHES_CCACHE)
-    message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use ccache")
+    message (${RECONFIGURE_MESSAGE_LEVEL} "Using ccache: No")
 endif ()

cmake/check_flags.cmake

@@ -3,6 +3,5 @@ include (CheckCCompilerFlag)
 
 check_cxx_compiler_flag("-Wreserved-identifier" HAS_RESERVED_IDENTIFIER)
 check_cxx_compiler_flag("-Wsuggest-destructor-override" HAS_SUGGEST_DESTRUCTOR_OVERRIDE)
-check_cxx_compiler_flag("-Wshadow" HAS_SHADOW)
 check_cxx_compiler_flag("-Wsuggest-override" HAS_SUGGEST_OVERRIDE)
 check_cxx_compiler_flag("-Xclang -fuse-ctor-homing" HAS_USE_CTOR_HOMING)

cmake/cpu_features.cmake

@@ -31,7 +31,10 @@ if (ARCH_NATIVE)
 elseif (ARCH_AARCH64)
     set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=armv8-a+crc")
 
-else ()
+elseif (ARCH_PPC64LE)
+    set (COMPILER_FLAGS "${COMPILER_FLAGS} -maltivec -mcpu=power8 -DNO_WARN_X86_INTRINSICS")
+
+elseif (ARCH_AMD64)
     set (TEST_FLAG "-mssse3")
     set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
     check_cxx_source_compiles("
@@ -60,10 +63,6 @@ else ()
         set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
     endif ()
 
-    if (ARCH_PPC64LE)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} -maltivec -mcpu=power8 -D__SSE2__=1 -DNO_WARN_X86_INTRINSICS")
-    endif ()
-
     set (TEST_FLAG "-msse4.2")
     set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
     check_cxx_source_compiles("
@@ -93,7 +92,6 @@ else ()
     endif ()
 
     set (TEST_FLAG "-mpopcnt")
-
     set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
     check_cxx_source_compiles("
         int main() {
@@ -186,6 +184,8 @@ else ()
             set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx512f -mavx512bw -mavx512vl -mprefer-vector-width=256")
         endif ()
     endif ()
+else ()
+    # RISC-V + exotic platforms
 endif ()
 
 cmake_pop_check_state ()

cmake/git_status.cmake

@@ -11,7 +11,7 @@ if (Git_FOUND)
   message(STATUS "HEAD's commit hash ${GIT_COMMIT_ID}")
   execute_process(
     COMMAND ${GIT_EXECUTABLE} status
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_STRIP_TRAILING_WHITESPACE)
 else()
   message(STATUS "The git program could not be found.")
 endif()

cmake/target.cmake

@@ -15,6 +15,8 @@ elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
 elseif (CMAKE_SYSTEM_NAME MATCHES "SunOS")
     set (OS_SUNOS 1)
     add_definitions(-D OS_SUNOS)
+else ()
+    message (FATAL_ERROR "Platform ${CMAKE_SYSTEM_NAME} is not supported")
 endif ()
 
 if (CMAKE_CROSSCOMPILING)

cmake/tools.cmake

@@ -6,67 +6,65 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
     set (COMPILER_CLANG 1) # Safe to treat AppleClang as a regular Clang, in general.
 elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
     set (COMPILER_CLANG 1)
+else ()
+    message (FATAL_ERROR "Compiler ${CMAKE_CXX_COMPILER_ID} is not supported")
 endif ()
 
-execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version)
+# Print details to output
+execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE COMPILER_SELF_IDENTIFICATION OUTPUT_STRIP_TRAILING_WHITESPACE)
+message (STATUS "Using compiler:\n${COMPILER_SELF_IDENTIFICATION}")
+
+# Require minimum compiler versions
+set (CLANG_MINIMUM_VERSION 12)
+set (XCODE_MINIMUM_VERSION 12.0)
+set (APPLE_CLANG_MINIMUM_VERSION 12.0.0)
+set (GCC_MINIMUM_VERSION 11)
 
 if (COMPILER_GCC)
-    # Require minimum version of gcc
-    set (GCC_MINIMUM_VERSION 11)
     if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION})
-        message (FATAL_ERROR "GCC version must be at least ${GCC_MINIMUM_VERSION}. For example, if GCC ${GCC_MINIMUM_VERSION} is available under gcc-${GCC_MINIMUM_VERSION}, g++-${GCC_MINIMUM_VERSION} names, do the following: export CC=gcc-${GCC_MINIMUM_VERSION} CXX=g++-${GCC_MINIMUM_VERSION}; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.")
+        message (FATAL_ERROR "Compilation with GCC version ${CMAKE_CXX_COMPILER_VERSION} is unsupported, the minimum required version is ${GCC_MINIMUM_VERSION}.")
     endif ()
 
-    message (WARNING "GCC compiler is not officially supported for ClickHouse. You should migrate to clang.")
+    message (WARNING "Compilation with GCC is unsupported. Please use Clang instead.")
 
 elseif (COMPILER_CLANG)
-    # Require minimum version of clang/apple-clang
     if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
         # (Experimental!) Specify "-DALLOW_APPLECLANG=ON" when running CMake configuration step, if you want to experiment with using it.
         if (NOT ALLOW_APPLECLANG AND NOT DEFINED ENV{ALLOW_APPLECLANG})
-            message (FATAL_ERROR "AppleClang is not supported, you should install clang from brew. See the instruction: https://clickhouse.com/docs/en/development/build-osx/")
+            message (FATAL_ERROR "Compilation with AppleClang is unsupported. Please use vanilla Clang, e.g. from Homebrew.")
         endif ()
 
-        # AppleClang 10.0.1 (Xcode 10.2) corresponds to LLVM/Clang upstream version 7.0.0
+        # For a mapping between XCode / AppleClang / vanilla Clang versions, see https://en.wikipedia.org/wiki/Xcode
-        # AppleClang 11.0.0 (Xcode 11.0) corresponds to LLVM/Clang upstream version 8.0.0
-        set (XCODE_MINIMUM_VERSION 10.2)
-        set (APPLE_CLANG_MINIMUM_VERSION 10.0.1)
         if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${APPLE_CLANG_MINIMUM_VERSION})
-            message (FATAL_ERROR "AppleClang compiler version must be at least ${APPLE_CLANG_MINIMUM_VERSION} (Xcode ${XCODE_MINIMUM_VERSION}).")
+            message (FATAL_ERROR "Compilation with AppleClang version ${CMAKE_CXX_COMPILER_VERSION} is unsupported, the minimum required version is ${APPLE_CLANG_MINIMUM_VERSION} (Xcode ${XCODE_MINIMUM_VERSION}).")
-        elseif (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 11.0.0)
-            # char8_t is available starting (upstream vanilla) Clang 7, but prior to Clang 8,
-            # it is not enabled by -std=c++20 and can be enabled with an explicit -fchar8_t.
-            set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fchar8_t")
-            set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fchar8_t")
         endif ()
     else ()
-        set (CLANG_MINIMUM_VERSION 12)
         if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
-            message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.")
+            message (FATAL_ERROR "Compilation with Clang version ${CMAKE_CXX_COMPILER_VERSION} is unsupported, the minimum required version is ${CLANG_MINIMUM_VERSION}.")
         endif ()
     endif ()
-else ()
-    message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang and GCC.")
 endif ()
 
+# Linker
+
 string (REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSION})
 list (GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR)
 
-# Linker
-
 # Example values: `lld-10`, `gold`.
 option (LINKER_NAME "Linker name or full path")
 
-if (COMPILER_GCC AND NOT LINKER_NAME)
+if (NOT LINKER_NAME)
-    find_program (LLD_PATH NAMES "ld.lld")
+    if (COMPILER_GCC)
-    find_program (GOLD_PATH NAMES "ld.gold")
+        find_program (LLD_PATH NAMES "ld.lld")
-elseif (NOT LINKER_NAME)
+        find_program (GOLD_PATH NAMES "ld.gold")
-    find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "lld-${COMPILER_VERSION_MAJOR}" "ld.lld" "lld")
+    elseif (COMPILER_CLANG)
-    find_program (GOLD_PATH NAMES "ld.gold" "gold")
+        find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "lld-${COMPILER_VERSION_MAJOR}" "ld.lld" "lld")
-endif ()
+        find_program (GOLD_PATH NAMES "ld.gold" "gold")
+    endif ()
+endif()
 
 if (OS_LINUX AND NOT LINKER_NAME)
-    # We prefer LLD linker over Gold or BFD on Linux.
+    # prefer lld linker over gold or ld on linux
     if (LLD_PATH)
         if (COMPILER_GCC)
             # GCC driver requires one of supported linker names like "lld".
@@ -87,9 +85,10 @@ if (OS_LINUX AND NOT LINKER_NAME)
         endif ()
     endif ()
 endif ()
+# TODO: allow different linker on != OS_LINUX
 
 if (LINKER_NAME)
-    if (COMPILER_CLANG AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 12.0.0 OR CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 12.0.0))
+    if (COMPILER_CLANG)
         find_program (LLD_PATH NAMES ${LINKER_NAME})
         if (NOT LLD_PATH)
             message (FATAL_ERROR "Using linker ${LINKER_NAME} but can't find its path.")
@@ -101,9 +100,14 @@ if (LINKER_NAME)
         set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
     endif ()
 
-    message(STATUS "Using custom linker by name: ${LINKER_NAME}")
 endif ()
 
+if (LINKER_NAME)
+    message(STATUS "Using linker: ${LINKER_NAME}")
+else()
+    message(STATUS "Using linker: <default>")
+endif()
+
 # Archiver
 
 if (COMPILER_GCC)
@@ -116,6 +120,8 @@ if (LLVM_AR_PATH)
     set (CMAKE_AR "${LLVM_AR_PATH}")
 endif ()
 
+message(STATUS "Using archiver: ${CMAKE_AR}")
+
 # Ranlib
 
 if (COMPILER_GCC)
@@ -128,6 +134,8 @@ if (LLVM_RANLIB_PATH)
     set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}")
 endif ()
 
+message(STATUS "Using ranlib: ${CMAKE_RANLIB}")
+
 # Install Name Tool
 
 if (COMPILER_GCC)
@@ -140,6 +148,8 @@ if (LLVM_INSTALL_NAME_TOOL_PATH)
     set (CMAKE_INSTALL_NAME_TOOL "${LLVM_INSTALL_NAME_TOOL_PATH}")
 endif ()
 
+message(STATUS "Using install-name-tool: ${CMAKE_INSTALL_NAME_TOOL}")
+
 # Objcopy
 
 if (COMPILER_GCC)
@@ -148,29 +158,13 @@ else ()
     find_program (OBJCOPY_PATH NAMES "llvm-objcopy-${COMPILER_VERSION_MAJOR}" "llvm-objcopy" "objcopy")
 endif ()
 
-if (NOT OBJCOPY_PATH AND OS_DARWIN)
-    find_program (BREW_PATH NAMES "brew")
-    if (BREW_PATH)
-        execute_process (COMMAND ${BREW_PATH} --prefix llvm ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE LLVM_PREFIX)
-        if (LLVM_PREFIX)
-            find_program (OBJCOPY_PATH NAMES "llvm-objcopy" PATHS "${LLVM_PREFIX}/bin" NO_DEFAULT_PATH)
-        endif ()
-        if (NOT OBJCOPY_PATH)
-            execute_process (COMMAND ${BREW_PATH} --prefix binutils ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE BINUTILS_PREFIX)
-            if (BINUTILS_PREFIX)
-                find_program (OBJCOPY_PATH NAMES "objcopy" PATHS "${BINUTILS_PREFIX}/bin" NO_DEFAULT_PATH)
-            endif ()
-        endif ()
-    endif ()
-endif ()
-
 if (OBJCOPY_PATH)
     message (STATUS "Using objcopy: ${OBJCOPY_PATH}")
 else ()
     message (FATAL_ERROR "Cannot find objcopy.")
 endif ()
 
-# Strip (FIXME copypaste)
+# Strip
 
 if (COMPILER_GCC)
     find_program (STRIP_PATH NAMES "llvm-strip" "llvm-strip-13" "llvm-strip-12" "llvm-strip-11" "strip")
@@ -178,22 +172,6 @@ else ()
     find_program (STRIP_PATH NAMES "llvm-strip-${COMPILER_VERSION_MAJOR}" "llvm-strip" "strip")
 endif ()
 
-if (NOT STRIP_PATH AND OS_DARWIN)
-    find_program (BREW_PATH NAMES "brew")
-    if (BREW_PATH)
-        execute_process (COMMAND ${BREW_PATH} --prefix llvm ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE LLVM_PREFIX)
-        if (LLVM_PREFIX)
-            find_program (STRIP_PATH NAMES "llvm-strip" PATHS "${LLVM_PREFIX}/bin" NO_DEFAULT_PATH)
-        endif ()
-        if (NOT STRIP_PATH)
-            execute_process (COMMAND ${BREW_PATH} --prefix binutils ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE BINUTILS_PREFIX)
-            if (BINUTILS_PREFIX)
-                find_program (STRIP_PATH NAMES "strip" PATHS "${BINUTILS_PREFIX}/bin" NO_DEFAULT_PATH)
-            endif ()
-        endif ()
-    endif ()
-endif ()
-
 if (STRIP_PATH)
     message (STATUS "Using strip: ${STRIP_PATH}")
 else ()

cmake/warnings.cmake

@@ -7,7 +7,7 @@
 # - sometimes warnings from 3rd party libraries may come from macro substitutions in our code
 #   and we have to wrap them with #pragma GCC/clang diagnostic ignored
 
-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra")
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra")
 
 # Add some warnings that are not available even with -Wall -Wextra -Wpedantic.
 # Intended for exploration of new compiler warnings that may be found useful.
@@ -25,6 +25,7 @@ if (COMPILER_CLANG)
     no_warning(vla-extension)
     no_warning(zero-length-array)
     no_warning(c11-extensions)
+    no_warning(unused-command-line-argument)
 
     if (WEVERYTHING)
         add_warning(everything)

contrib/CMakeLists.txt

@@ -152,6 +152,7 @@ endif()
 
 add_contrib (sqlite-cmake sqlite-amalgamation)
 add_contrib (s2geometry-cmake s2geometry)
+add_contrib (eigen-cmake eigen)
 
 # Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
 # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear

contrib/base64-cmake/CMakeLists.txt

@@ -1,4 +1,4 @@
-if(ARCH_AMD64 OR ARCH_ARM)
+if(ARCH_AMD64 OR ARCH_AARCH64)
     option (ENABLE_BASE64 "Enable base64" ${ENABLE_LIBRARIES})
 elseif(ENABLE_BASE64)
     message (${RECONFIGURE_MESSAGE_LEVEL} "base64 library is only supported on x86_64 and aarch64")

contrib/boost-cmake/CMakeLists.txt

@@ -114,7 +114,7 @@ if (SANITIZE AND (SANITIZE STREQUAL "address" OR SANITIZE STREQUAL "thread"))
             "${LIBRARY_DIR}/libs/context/src/continuation.cpp"
     )
 endif()
-if (ARCH_ARM)
+if (ARCH_AARCH64)
     set (SRCS_CONTEXT ${SRCS_CONTEXT}
         "${LIBRARY_DIR}/libs/context/src/asm/jump_arm64_aapcs_elf_gas.S"
         "${LIBRARY_DIR}/libs/context/src/asm/make_arm64_aapcs_elf_gas.S"

contrib/eigen

@@ -0,0 +1 @@
+Subproject commit 3147391d946bb4b6c68edd901f2add6ac1f31f8c

contrib/eigen-cmake/CMakeLists.txt

@@ -0,0 +1,23 @@
+set(EIGEN_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/eigen")
+
+add_library (_eigen INTERFACE)
+
+option (ENABLE_MKL "Build Eigen with Intel MKL" OFF)
+if (ENABLE_MKL)
+    set(MKL_THREADING sequential)
+    set(MKL_INTERFACE lp64)
+    find_package(MKL REQUIRED)
+    if (MKL_FOUND)
+        message("MKL INCLUDE: ${MKL_INCLUDE}")
+        message("MKL LIBRARIES: ${MKL_LIBRARIES}")
+        target_compile_definitions(_eigen INTERFACE EIGEN_USE_MKL_ALL)
+        target_include_directories(_eigen INTERFACE ${MKL_INCLUDE})
+        target_link_libraries(_eigen INTERFACE ${MKL_LIBRARIES})
+    endif()
+endif()
+
+# Only include MPL2 code from Eigen library
+target_compile_definitions(_eigen INTERFACE EIGEN_MPL2_ONLY)
+
+target_include_directories (_eigen SYSTEM INTERFACE ${EIGEN_LIBRARY_DIR})
+add_library(ch_contrib::eigen ALIAS _eigen)

contrib/jemalloc-cmake/CMakeLists.txt

@@ -1,5 +1,5 @@
 if (SANITIZE OR NOT (
-	((OS_LINUX OR OS_FREEBSD) AND (ARCH_AMD64 OR ARCH_ARM OR ARCH_PPC64LE OR ARCH_RISCV64)) OR
+    ((OS_LINUX OR OS_FREEBSD) AND (ARCH_AMD64 OR ARCH_AARCH64 OR ARCH_PPC64LE OR ARCH_RISCV64)) OR
     (OS_DARWIN AND (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG"))
 ))
     if (ENABLE_JEMALLOC)
@@ -141,7 +141,7 @@ if (ARCH_AMD64)
     else()
         set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_x86_64")
     endif()
-elseif (ARCH_ARM)
+elseif (ARCH_AARCH64)
     set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_aarch64")
 elseif (ARCH_PPC64LE)
     set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_ppc64le")

contrib/libhdfs3-cmake/CMake/Options.cmake

@@ -7,9 +7,9 @@ CHECK_FUNCTION_EXISTS(nanosleep HAVE_NANOSLEEP)
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-strict-aliasing")
 SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing")
 
-IF(ENABLE_SSE STREQUAL ON AND NOT ARCH_PPC64LE AND NOT ARCH_AARCH64 AND NOT ARCH_ARM)
+IF(ENABLE_SSE STREQUAL ON AND NOT ARCH_PPC64LE AND NOT ARCH_AARCH64 AND NOT ARCH_AARCH64)
     SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
-ENDIF(ENABLE_SSE STREQUAL ON AND NOT ARCH_PPC64LE AND NOT ARCH_AARCH64 AND NOT ARCH_ARM)
+ENDIF(ENABLE_SSE STREQUAL ON AND NOT ARCH_PPC64LE AND NOT ARCH_AARCH64 AND NOT ARCH_AARCH64)
 
 IF(NOT TEST_HDFS_PREFIX)
 SET(TEST_HDFS_PREFIX "./" CACHE STRING "default directory prefix used for test." FORCE)

contrib/libhdfs3-cmake/CMakeLists.txt

@@ -1,4 +1,4 @@
-if(NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE)
+if(NOT ARCH_AARCH64 AND NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE)
     option(ENABLE_HDFS "Enable HDFS" ${ENABLE_LIBRARIES})
 elseif(ENABLE_HDFS)
     message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use HDFS3 with current configuration")

contrib/poco

@@ -1 +1 @@
-Subproject commit 6c1a233744d13414e8e8db396c75177b857b2c22
+Subproject commit de35b9fd72b57127abdc3a5beaf0e320d767e356

docker/test/fasttest/run.sh

@@ -177,6 +177,7 @@ function clone_submodules
             contrib/jemalloc
             contrib/replxx
             contrib/wyhash
+            contrib/eigen
         )
 
         git submodule sync

docs/_includes/install/universal.sh

@@ -13,7 +13,7 @@ then
     elif [ "${ARCH}" = "aarch64" ]
     then
         DIR="aarch64"
-    elif [ "${ARCH}" = "powerpc64le" ]
+    elif [ "${ARCH}" = "powerpc64le" ] || [ "${ARCH}" = "ppc64le" ]
     then
         DIR="powerpc64le"
     fi
@@ -25,7 +25,7 @@ then
     elif [ "${ARCH}" = "aarch64" ]
     then
         DIR="freebsd-aarch64"
-    elif [ "${ARCH}" = "powerpc64le" ]
+    elif [ "${ARCH}" = "powerpc64le" ] || [ "${ARCH}" = "ppc64le" ]
     then
         DIR="freebsd-powerpc64le"
     fi

docs/en/development/build-osx.md

@@ -10,21 +10,17 @@ description: How to build ClickHouse on Mac OS X
 You can install pre-built ClickHouse as described in [Quick Start](https://clickhouse.com/#quick-start). Follow **macOS (Intel)** or **macOS (Apple silicon)** installation instructions.
 :::
 
-Build should work on x86_64 (Intel) and arm64 (Apple silicon) based macOS 10.15 (Catalina) and higher with Homebrew's vanilla Clang.
+The build works on x86_64 (Intel) and arm64 (Apple Silicon) based on macOS 10.15 (Catalina) or higher with Homebrew's vanilla Clang.
-It is always recommended to use vanilla `clang` compiler. 
 
 :::note
-It is possible to use XCode's `apple-clang` or `gcc`, but it's strongly discouraged.
+It is also possible to compile with Apple's XCode `apple-clang` or Homebrew's `gcc`, but it's strongly discouraged.
 :::
 
 ## Install Homebrew {#install-homebrew}
 
-``` bash
+First install [Homebrew](https://brew.sh/)
-/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-# ...and follow the printed instructions on any additional steps required to complete the installation.
-```
 
-## Install Xcode and Command Line Tools {#install-xcode-and-command-line-tools}
+## For Apple's Clang (discouraged): Install Xcode and Command Line Tools {#install-xcode-and-command-line-tools}
 
 Install the latest [Xcode](https://apps.apple.com/am/app/xcode/id497799835?mt=12) from App Store.
 
@@ -57,12 +53,13 @@ To build using Homebrew's vanilla Clang compiler (the only **recommended** way):
 
 ``` bash
 cd ClickHouse
-rm -rf build
 mkdir build
-cd build
+export PATH=$(brew --prefix llvm)/bin:$PATH
-cmake -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm)/bin/clang++ -DCMAKE_AR=$(brew --prefix llvm)/bin/llvm-ar -DCMAKE_RANLIB=$(brew --prefix llvm)/bin/llvm-ranlib -DOBJCOPY_PATH=$(brew --prefix llvm)/bin/llvm-objcopy -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
+export CC=$(brew --prefix llvm)/bin/clang
-cmake --build . --config RelWithDebInfo
+export CXX=$(brew --prefix llvm)/bin/clang++
-# The resulting binary will be created at: ./programs/clickhouse
+cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -S . -B build
+cmake --build build
+# The resulting binary will be created at: build/programs/clickhouse
 ```
 
 To build using Xcode's native AppleClang compiler in Xcode IDE (this option is only for development builds and workflows, and is **not recommended** unless you know what you are doing):
@@ -82,12 +79,14 @@ To build using Homebrew's vanilla GCC compiler (this option is only for developm
 
 ``` bash
 cd ClickHouse
-rm -rf build
 mkdir build
-cd build
+export PATH=$(brew --prefix binutils)/bin:$PATH
-cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-11 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-11 -DCMAKE_AR=$(brew --prefix gcc)/bin/gcc-ar-11 -DCMAKE_RANLIB=$(brew --prefix gcc)/bin/gcc-ranlib-11 -DOBJCOPY_PATH=$(brew --prefix binutils)/bin/objcopy -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
+export PATH=$(brew --prefix gcc)/bin:$PATH
-cmake --build . --config RelWithDebInfo
+export CC=$(brew --prefix gcc)/bin/gcc-11
-# The resulting binary will be created at: ./programs/clickhouse
+export CXX=$(brew --prefix gcc)/bin/g++-11
+cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -S . -B build
+cmake --build build
+# The resulting binary will be created at: build/programs/clickhouse
 ```
 
 ## Caveats {#caveats}

docs/en/development/cmake-in-clickhouse.md

@@ -300,6 +300,12 @@ Note that ClickHouse uses forks of these libraries, see https://github.com/Click
 <td>Take care to add prlimit in command line before ccache, or else ccache thinks that prlimit is compiler, and clang++ is its input file, and refuses to work  with multiple inputs, e.g in ccache log: [2021-03-31T18:06:32.655327 36900] Command line: /usr/bin/ccache prlimit --as=10000000000 --data=5000000000 --cpu=600 /usr/bin/clang++-11 - ...... std=gnu++2a -MD -MT src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o -MF src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o.d -o src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o -c ../src/Storages/MergeTree/IMergeTreeDataPart.cpp  [2021-03-31T18:06:32.656704 36900] Multiple input files: /usr/bin/clang++-11 and ../src/Storages/MergeTree/IMergeTreeDataPart.cpp  Another way would be to use --ccache-skip option before clang++-11 to make ccache ignore it.</td>
 </tr>
 <tr>
+<td><a name="enable-colored-build"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L160" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_COLORED_BUILD</code></a></td>
+<td><code class="syntax">ON</code></td>
+<td>Enable colored diagnostics in build log.</td>
+<td></td>
+</tr>
+<tr>
 <td><a name="enable-examples"></a><a href="https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L201" rel="external nofollow noreferrer" target="_blank"><code class="syntax">ENABLE_EXAMPLES</code></a></td>
 <td><code class="syntax">OFF</code></td>
 <td>Build all example programs in 'examples' subdirectories</td>

docs/en/getting-started/example-datasets/nyc-taxi.md

@@ -1,5 +1,6 @@
 ---
 sidebar_label: New York Taxi Data
+sidebar_position: 2
 description: Data for billions of taxi and for-hire vehicle (Uber, Lyft, etc.) trips originating in New York City since 2009
 ---
 

docs/en/getting-started/example-datasets/uk-price-paid.md

@@ -1,5 +1,6 @@
 ---
 sidebar_label: UK Property Price Paid
+sidebar_position: 1
 ---
 
 # UK Property Price Paid

docs/en/operations/external-authenticators/ssl-x509.md

@@ -2,7 +2,7 @@
 
 [SSL 'strict' option](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) enables mandatory certificate validation for the incoming connections. In this case, only connections with trusted certificates can be established. Connections with untrusted certificates will be rejected. Thus, certificate validation allows to uniquely authenticate an incoming connection. `Common Name` field of the certificate is used to identify connected user. This allows to associate multiple certificates with the same user. Additionally, reissuing and revoking of the certificates does not affect the ClickHouse configuration.
 
-To enable SSL certificate authentication, a list of `Common Name`'s for each ClickHouse user must be sspecified in the settings file `config.xml `:
+To enable SSL certificate authentication, a list of `Common Name`'s for each ClickHouse user must be specified in the settings file `users.xml `:
 
 **Example**
 ```xml
@@ -10,11 +10,11 @@ To enable SSL certificate authentication, a list of `Common Name`'s for each Cli
     <!- ... -->
     <users>
         <user_name>
-            <certificates>
+            <ssl_certificates>
                 <common_name>host.domain.com:example_user</common_name>
                 <common_name>host.domain.com:example_user_dev</common_name>
                 <!-- More names -->
-            </certificates>
+            </ssl_certificates>
             <!-- Other settings -->
         </user_name>
     </users>

docs/en/operations/optimizing-performance/sampling-query-profiler.md

@@ -29,7 +29,7 @@ To analyze the `trace_log` system table:
 
 -   Use the `addressToLine`, `addressToLineWithInlines`, `addressToSymbol` and `demangle` [introspection functions](../../sql-reference/functions/introspection.md) to get function names and their positions in ClickHouse code. To get a profile for some query, you need to aggregate data from the `trace_log` table. You can aggregate data by individual functions or by the whole stack traces.
 
-If you need to visualize `trace_log` info, try [flamegraph](../../interfaces/third-party/gui/#clickhouse-flamegraph) and [speedscope](https://github.com/laplab/clickhouse-speedscope).
+If you need to visualize `trace_log` info, try [flamegraph](../../interfaces/third-party/gui.md#clickhouse-flamegraph-clickhouse-flamegraph) and [speedscope](https://github.com/laplab/clickhouse-speedscope).
 
 ## Example {#example}
 

docs/en/operations/server-configuration-parameters/settings.md

@@ -1745,3 +1745,13 @@ Possible values:
 -   Positive integer.
 
 Default value: `10000`.
+
+## global_memory_usage_overcommit_max_wait_microseconds {#global_memory_usage_overcommit_max_wait_microseconds}
+
+Sets maximum waiting time for global overcommit tracker.
+
+Possible values:
+
+-   Positive integer.
+
+Default value: `200`.

docs/en/operations/settings/memory-overcommit.md

@@ -0,0 +1,37 @@
+# Memory overcommit
+
+Memory overcommit is an experimental technique intended to allow to set more flexible memory limits for queries.
+
+The idea of this technique is to introduce settings which can represent guaranteed amount of memory a query can use.
+When memory overcommit is enabled and the memory limit is reached ClickHouse will select the most overcommitted query and try to free memory by killing this query.
+
+When memory limit is reached any query will wait some time during attempt to allocate new memory.
+If timeout is passed and memory is freed, the query continues execution.
+Otherwise an exception will be thrown and the query is killed.
+
+Selection of query to stop or kill is performed by either global or user overcommit trackers depending on what memory limit is reached.
+If overcommit tracker can't choose query to stop, MEMORY_LIMIT_EXCEEDED exception is thrown.
+
+## User overcommit tracker
+
+User overcommit tracker finds a query with the biggest overcommit ratio in the user's query list.
+Overcommit ratio for a query is computed as number of allocated bytes divided by value of `memory_overcommit_ratio_denominator` setting.
+
+If `memory_overcommit_ratio_denominator` for the query is equals to zero, overcommit tracker won't choose this query.
+
+Waiting timeout is set by `memory_usage_overcommit_max_wait_microseconds` setting.
+
+**Example**
+
+```sql
+SELECT number FROM numbers(1000) GROUP BY number SETTINGS memory_overcommit_ratio_denominator=4000, memory_usage_overcommit_max_wait_microseconds=500
+```
+
+## Global overcommit tracker
+
+Global overcommit tracker finds a query with the biggest overcommit ratio in the list of all queries.
+In this case overcommit ratio is computed as number of allocated bytes divided by value of `memory_overcommit_ratio_denominator_for_user` setting.
+
+If `memory_overcommit_ratio_denominator_for_user` for the query is equals to zero, overcommit tracker won't choose this query.
+
+Waiting timeout is set by `global_memory_usage_overcommit_max_wait_microseconds` parameter in the configuration file.

docs/en/operations/settings/settings.md

@@ -4263,3 +4263,29 @@ Possible values:
 - 1 — Enabled.
 
 Default value: 1.
+
+## memory_overcommit_ratio_denominator
+
+It represents soft memory limit in case when hard limit is reached on user level.
+This value is used to compute overcommit ratio for the query.
+Zero means skip the query.
+Read more about [memory overcommit](memory-overcommit.md).
+
+Default value: `1GiB`.
+
+## memory_usage_overcommit_max_wait_microseconds
+
+Maximum time thread will wait for memory to be freed in the case of memory overcommit on a user level.
+If the timeout is reached and memory is not freed, an exception is thrown.
+Read more about [memory overcommit](memory-overcommit.md).
+
+Default value: `200`.
+
+## memory_overcommit_ratio_denominator_for_user
+
+It represents soft memory limit in case when hard limit is reached on global level.
+This value is used to compute overcommit ratio for the query.
+Zero means skip the query.
+Read more about [memory overcommit](memory-overcommit.md).
+
+Default value: `1GiB`.

docs/en/sql-reference/aggregate-functions/reference/grouparray.md

@@ -12,3 +12,34 @@ Values can be added to the array in any (indeterminate) order.
 The second version (with the `max_size` parameter) limits the size of the resulting array to `max_size` elements. For example, `groupArray(1)(x)` is equivalent to `[any (x)]`.
 
 In some cases, you can still rely on the order of execution. This applies to cases when `SELECT` comes from a subquery that uses `ORDER BY`.
+
+**Example**
+
+``` text
+SELECT * FROM default.ck;
+
+┌─id─┬─name─────┐
+│  1 │ zhangsan │
+│  1 │ ᴺᵁᴸᴸ     │
+│  1 │ lisi     │
+│  2 │ wangwu   │
+└────┴──────────┘
+
+```
+
+Query:
+
+``` sql
+select id, groupArray(10)(name) from default.ck group by id;
+```
+
+Result:
+
+``` text
+┌─id─┬─groupArray(10)(name)─┐
+│  1 │ ['zhangsan','lisi']  │
+│  2 │ ['wangwu']           │
+└────┴──────────────────────┘
+```
+
+The groupArray function will remove ᴺᵁᴸᴸ value based on the above results.

docs/en/sql-reference/statements/alter/index/index.md

@@ -12,7 +12,7 @@ The following operations are available:
 
 -   `ALTER TABLE [db].name DROP INDEX name` - Removes index description from tables metadata and deletes index files from disk.
 
--   `ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name` - The query rebuilds the secondary index `name` in the partition `partition_name`. Implemented as a [mutation](../../../../sql-reference/statements/alter/index.md#mutations). To rebuild index over the whole data in the table you need to remove `IN PARTITION` from query.
+-   `ALTER TABLE [db.]table MATERIALIZE INDEX name [IN PARTITION partition_name]` - Rebuilds the secondary index `name` for the specified `partition_name`. Implemented as a [mutation](../../../../sql-reference/statements/alter/index.md#mutations). If `IN PARTITION` part is omitted then it rebuilds the index for the whole table data.
 
 The first two commands are lightweight in a sense that they only change metadata or remove files.
 

docs/zh/sql-reference/functions/arithmetic-functions.md

@@ -80,4 +80,78 @@ SELECT toTypeName(0), toTypeName(0 + 0), toTypeName(0 + 0 + 0), toTypeName(0 + 0
 返回数值的最小公倍数。
 除以零或将最小负数除以-1时抛出异常。
 
+## max2 {#max2}
+
+比较两个值并返回最大值。返回值转换为[Float64](../../sql-reference/data-types/float.md)。
+
+**语法**
+
+```sql
+max2(value1, value2)
+```
+
+**参数**
+
+-   `value1` — 第一个值，类型为[Int/UInt](../../sql-reference/data-types/int-uint.md)或[Float](../../sql-reference/data-types/float.md)。
+-   `value2` — 第二个值，类型为[Int/UInt](../../sql-reference/data-types/int-uint.md)或[Float](../../sql-reference/data-types/float.md)。
+
+**返回值**
+
+-   两个值中的最大值。
+
+类型: [Float](../../sql-reference/data-types/float.md)。
+
+**示例**
+
+查询语句:
+
+```sql
+SELECT max2(-1, 2);
+```
+
+结果:
+
+```text
+┌─max2(-1, 2)─┐
+│           2 │
+└─────────────┘
+```
+
+## min2 {#min2}
+
+比较两个值并返回最小值。返回值类型转换为[Float64](../../sql-reference/data-types/float.md)。
+
+**语法**
+
+```sql
+max2(value1, value2)
+```
+
+**参数**
+
+-   `value1` — 第一个值，类型为[Int/UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md)。
+-   `value2` — 第二个值，类型为[Int/UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md)。
+
+**返回值**
+
+-   两个值中的最小值。
+
+类型: [Float](../../sql-reference/data-types/float.md)。
+
+**示例**
+
+查询语句:
+
+```sql
+SELECT min2(-1, 2);
+```
+
+结果:
+
+```text
+┌─min2(-1, 2)─┐
+│          -1 │
+└─────────────┘
+```
+
 [来源文章](https://clickhouse.com/docs/en/query_language/functions/arithmetic_functions/) <!--hide-->

docs/zh/sql-reference/functions/bit-functions.md

@@ -4,26 +4,437 @@
 
 结果类型是一个整数，其位数等于其参数的最大位。如果至少有一个参数为有符数字，则结果为有符数字。如果参数是浮点数，则将其强制转换为Int64。
 
-## bitAnd(a,b) {#bitanda-b}
+## bitAnd(a, b) {#bitanda-b}
 
-## bitOr(a,b) {#bitora-b}
+## bitOr(a, b) {#bitora-b}
 
-## bitXor(a,b) {#bitxora-b}
+## bitXor(a, b) {#bitxora-b}
 
 ## bitNot(a) {#bitnota}
 
-## bitShiftLeft(a,b) {#bitshiftlefta-b}
+## bitShiftLeft(a, b) {#bitshiftlefta-b}
 
-## bitShiftRight(a,b) {#bitshiftrighta-b}
+将值的二进制表示向左移动指定数量的位。
 
-## bitRotateLeft(a,b) {#bitrotatelefta-b}
+`FixedString` 或 `String` 被视为单个多字节值。
 
-## bitRotateRight(a,b) {#bitrotaterighta-b}
+`FixedString` 值的位在移出时会丢失。相反，`String` 值使用额外的字节进行扩展，因此不会丢失任何位。
 
-## bitTest(a,b) {#bittesta-b}
+**语法**
 
-## bitTestAll(a,b) {#bittestalla-b}
+``` sql
+bitShiftLeft(a, b)
+```
+
+**参数**
+
+-   `a` — 要进行移位操作的值。类型可以为[Integer types](../../sql-reference/data-types/int-uint.md)，[String](../../sql-reference/data-types/string.md)或者[FixedString](../../sql-reference/data-types/fixedstring.md)。
+-   `b` — 移位的次数。类型为[Unsigned integer types](../../sql-reference/data-types/int-uint.md)，允许使用64位数字及64位以下的数字类型。
+
+**返回值**
+
+-   移位后的值。
+
+返回值的类型与输入值的类型相同。
+
+**示例**
+
+在以下查询中，[bin](encoding-functions.md#bin)和[hex](encoding-functions.md#hex)函数用于显示移位值的位。
+
+``` sql
+SELECT 99 AS a, bin(a), bitShiftLeft(a, 2) AS a_shifted, bin(a_shifted);
+SELECT 'abc' AS a, hex(a), bitShiftLeft(a, 4) AS a_shifted, hex(a_shifted);
+SELECT toFixedString('abc', 3) AS a, hex(a), bitShiftLeft(a, 4) AS a_shifted, hex(a_shifted);
+```
+
+结果：
+
+``` text
+┌──a─┬─bin(99)──┬─a_shifted─┬─bin(bitShiftLeft(99, 2))─┐
+│ 99 │ 01100011 │       140 │ 10001100                 │
+└────┴──────────┴───────────┴──────────────────────────┘
+┌─a───┬─hex('abc')─┬─a_shifted─┬─hex(bitShiftLeft('abc', 4))─┐
+│ abc │ 616263     │ &0        │ 06162630                    │
+└─────┴────────────┴───────────┴─────────────────────────────┘
+┌─a───┬─hex(toFixedString('abc', 3))─┬─a_shifted─┬─hex(bitShiftLeft(toFixedString('abc', 3), 4))─┐
+│ abc │ 616263                       │ &0        │ 162630                                        │
+└─────┴──────────────────────────────┴───────────┴───────────────────────────────────────────────┘
+```
+
+## bitShiftRight(a, b) {#bitshiftrighta-b}
+
+将值的二进制表示向右移动指定数量的位。
+
+`FixedString`或`String`被视为单个多字节值。请注意，`String`值的长度会随着位的移出而减少。
+
+**语法**
+
+``` sql
+bitShiftRight(a, b)
+```
+
+**参数**
+
+-   `a` — 需要进行位移的值。类型可以为[Integer types](../../sql-reference/data-types/int-uint.md)，[String](../../sql-reference/data-types/string.md)或者[FixedString](../../sql-reference/data-types/fixedstring.md)。
+-   `b` — 移位的次数。类型为[Unsigned integer types](../../sql-reference/data-types/int-uint.md)，允许使用64位数字及64位以下的数字类型。
+
+**返回值**
+
+-   移位后的值。
+
+返回值的类型与输入值的类型相同。
+
+**示例**
+
+查询语句：
+
+``` sql
+SELECT 101 AS a, bin(a), bitShiftRight(a, 2) AS a_shifted, bin(a_shifted);
+SELECT 'abc' AS a, hex(a), bitShiftRight(a, 12) AS a_shifted, hex(a_shifted);
+SELECT toFixedString('abc', 3) AS a, hex(a), bitShiftRight(a, 12) AS a_shifted, hex(a_shifted);
+```
+
+结果：
+
+``` text
+┌───a─┬─bin(101)─┬─a_shifted─┬─bin(bitShiftRight(101, 2))─┐
+│ 101 │ 01100101 │        25 │ 00011001                   │
+└─────┴──────────┴───────────┴────────────────────────────┘
+┌─a───┬─hex('abc')─┬─a_shifted─┬─hex(bitShiftRight('abc', 12))─┐
+│ abc │ 616263     │           │ 0616                          │
+└─────┴────────────┴───────────┴───────────────────────────────┘
+┌─a───┬─hex(toFixedString('abc', 3))─┬─a_shifted─┬─hex(bitShiftRight(toFixedString('abc', 3), 12))─┐
+│ abc │ 616263                       │           │ 000616                                          │
+└─────┴──────────────────────────────┴───────────┴─────────────────────────────────────────────────┘
+```
+
+## bitRotateLeft(a, b) {#bitrotatelefta-b}
+
+## bitRotateRight(a, b) {#bitrotaterighta-b}
+
+## bitSlice(s, offset, length)
+
+返回从`offset`索引中的`length`位长的位开始的子字符串，位索引从 1 开始。
+
+**语法**
+
+``` sql
+bitSlice(s, offset[, length])
+```
+
+**参数**
+
+- `s` — 类型可以是[String](../../sql-reference/data-types/string.md)或者[FixedString](../../sql-reference/data-types/fixedstring.md)。
+- `offset` — 带位的起始索引，正值表示左侧偏移，负值表示右侧缩进，位编号从 1 开始。
+- `length` — 带位的子串长度。如果您指定一个负值，该函数将返回一个开放子字符串 \[offset, array_length - length\]。如果省略该值，该函数将返回子字符串 \[offset, the_end_string\]。如果长度超过s，将被截断。如果长度不是8的倍数，则在右边填充0。
+
+**返回值**
+
+- 子字符串，类型为[String](../../sql-reference/data-types/string.md)。
+
+**示例**
+
+查询语句：
+
+``` sql
+select bin('Hello'), bin(bitSlice('Hello', 1, 8))
+select bin('Hello'), bin(bitSlice('Hello', 1, 2))
+select bin('Hello'), bin(bitSlice('Hello', 1, 9))
+select bin('Hello'), bin(bitSlice('Hello', -4, 8))
+```
+
+结果：
+
+``` text
+┌─bin('Hello')─────────────────────────────┬─bin(bitSlice('Hello', 1, 8))─┐
+│ 0100100001100101011011000110110001101111 │ 01001000                     │
+└──────────────────────────────────────────┴──────────────────────────────┘
+┌─bin('Hello')─────────────────────────────┬─bin(bitSlice('Hello', 1, 2))─┐
+│ 0100100001100101011011000110110001101111 │ 01000000                     │
+└──────────────────────────────────────────┴──────────────────────────────┘
+┌─bin('Hello')─────────────────────────────┬─bin(bitSlice('Hello', 1, 9))─┐
+│ 0100100001100101011011000110110001101111 │ 0100100000000000             │
+└──────────────────────────────────────────┴──────────────────────────────┘
+┌─bin('Hello')─────────────────────────────┬─bin(bitSlice('Hello', -4, 8))─┐
+│ 0100100001100101011011000110110001101111 │ 11110000                      │
+└──────────────────────────────────────────┴───────────────────────────────┘
+```
+
+## bitTest {#bittest}
+
+取任意整数并将其转换为[binary form](https://en.wikipedia.org/wiki/Binary_number)，返回指定位置的位值。位值从右到左数，从0开始计数。
+
+**语法**
+
+``` sql
+SELECT bitTest(number, index)
+```
+
+**参数**
+
+-   `number` – 整数。
+-   `index` – 要获取位值的位置。 
+
+**返回值**
+
+返回指定位置的位值
+
+类型为：`UInt8`。
+
+**示例**
+
+例如，十进制数字 43 在二进制的表示是 101011。
+
+查询语句：
+
+``` sql
+SELECT bitTest(43, 1);
+```
+
+结果：
+
+``` text
+┌─bitTest(43, 1)─┐
+│              1 │
+└────────────────┘
+```
+
+另一个示例：
+
+查询语句：
+
+``` sql
+SELECT bitTest(43, 2);
+```
+
+结果：
+
+``` text
+┌─bitTest(43, 2)─┐
+│              0 │
+└────────────────┘
+```
+
+## bitTestAll {#bittestall}
+
+返回给定位置所有位的 [logical conjuction](https://en.wikipedia.org/wiki/Logical_conjunction) 进行与操作的结果。位值从右到左数，从0开始计数。
+
+与运算的结果：
+
+0 AND 0 = 0
+
+0 AND 1 = 0
+
+1 AND 0 = 0
+
+1 AND 1 = 1
+
+**语法**
+
+``` sql
+SELECT bitTestAll(number, index1, index2, index3, index4, ...)
+```
+
+**参数**
+
+-   `number` – 整数。
+-   `index1`, `index2`, `index3`, `index4` – 位的位置。例如，对于一组位置 (`index1`, `index2`, `index3`, `index4`) 当且仅当它的所有位置都为真时才为真 (`index1` ⋀ `index2`, ⋀ `index3` ⋀ `index4` ）。
+
+**返回值**
+
+返回逻辑与的结果。
+
+类型为： `UInt8`。
+
+**示例**
+
+例如，十进制数字 43 在二进制的表示是 101011。
+
+查询语句：
+
+``` sql
+SELECT bitTestAll(43, 0, 1, 3, 5);
+```
+
+结果：
+
+``` text
+┌─bitTestAll(43, 0, 1, 3, 5)─┐
+│                          1 │
+└────────────────────────────┘
+```
+
+另一个例子:
+
+查询语句：
+
+``` sql
+SELECT bitTestAll(43, 0, 1, 3, 5, 2);
+```
+
+结果：
+
+``` text
+┌─bitTestAll(43, 0, 1, 3, 5, 2)─┐
+│                             0 │
+└───────────────────────────────┘
+```
+
+## bitTestAny {#bittestany}
+
+返回给定位置所有位的 [logical disjunction](https://en.wikipedia.org/wiki/Logical_disjunction) 进行或操作的结果。位值从右到左数，从0开始计数。
+
+或运算的结果：
+
+0 OR 0 = 0
+
+0 OR 1 = 1
+
+1 OR 0 = 1
+
+1 OR 1 = 1
+
+**语法**
+
+``` sql
+SELECT bitTestAny(number, index1, index2, index3, index4, ...)
+```
+
+**参数**
+
+-   `number` – 整数。
+-   `index1`, `index2`, `index3`, `index4` –  位的位置。
+
+**返回值**
+
+返回逻辑或的结果。
+
+类型为： `UInt8`。
+
+**示例**
+
+例如，十进制数字 43 在二进制的表示是 101011。
+
+查询语句：
+
+``` sql
+SELECT bitTestAny(43, 0, 2);
+```
+
+结果：
+
+``` text
+┌─bitTestAny(43, 0, 2)─┐
+│                    1 │
+└──────────────────────┘
+```
+
+另一个例子:
+
+查询语句：
+
+``` sql
+SELECT bitTestAny(43, 4, 2);
+```
+
+结果：
+
+``` text
+┌─bitTestAny(43, 4, 2)─┐
+│                    0 │
+└──────────────────────┘
+```
+
+## bitCount {#bitcount}
+
+计算数字的二进制表示中值为 1 的位数。
+
+**语法**
+
+``` sql
+bitCount(x)
+```
+
+**参数**
+
+-   `x` — 类型为[Integer](../../sql-reference/data-types/int-uint.md)或[floating-point](../../sql-reference/data-types/float.md)数字。该函数使用内存中的值表示。它允许支持浮点数。
+
+**返回值**
+
+-   输入数字中值为 1 的位数。
+
+该函数不会将输入值转换为更大的类型 ([sign extension](https://en.wikipedia.org/wiki/Sign_extension))。 因此，例如，`bitCount(toUInt8(-1)) = 8`。
+
+类型为： `UInt8`。
+
+**示例**
+
+以十进制数字 333 为例，它的二进制表示为: 0000000101001101。
+
+查询语句：
+
+``` sql
+SELECT bitCount(333);
+```
+
+结果：
+
+``` text
+┌─bitCount(333)─┐
+│             5 │
+└───────────────┘
+```
+
+## bitHammingDistance {#bithammingdistance}
+
+返回两个整数值的位表示之间的 [Hamming Distance](https://en.wikipedia.org/wiki/Hamming_distance)。可与 [SimHash](../../sql-reference/functions/hash-functions.md#ngramsimhash) 函数一起使用，用于检测半重复字符串。距离越小，这些字符串就越有可能相同。
+
+**语法**
+
+``` sql
+bitHammingDistance(int1, int2)
+```
+
+**参数**
+
+-   `int1` — 第一个整数值。类型为[Int64](../../sql-reference/data-types/int-uint.md)。
+-   `int2` — 第二个整数值。类型为[Int64](../../sql-reference/data-types/int-uint.md)。
+
+**返回值**
+
+-   汉明距离。
+
+类型为： [UInt8](../../sql-reference/data-types/int-uint.md)。
+
+**示例**
+
+查询语句：
+
+``` sql
+SELECT bitHammingDistance(111, 121);
+```
+
+结果：
+
+``` text
+┌─bitHammingDistance(111, 121)─┐
+│                            3 │
+└──────────────────────────────┘
+```
+
+使用[SimHash](../../sql-reference/functions/hash-functions.md#ngramsimhash)函数:
+
+``` sql
+SELECT bitHammingDistance(ngramSimHash('cat ate rat'), ngramSimHash('rat ate cat'));
+```
+
+结果：
+
+``` text
+┌─bitHammingDistance(ngramSimHash('cat ate rat'), ngramSimHash('rat ate cat'))─┐
+│                                                                            5 │
+└──────────────────────────────────────────────────────────────────────────────┘
+```
 
-## bitTestAny(a,b) {#bittestanya-b}
 
 [来源文章](https://clickhouse.com/docs/en/query_language/functions/bit_functions/) <!--hide-->

programs/benchmark/Benchmark.cpp

@@ -1,8 +1,8 @@
 #include <unistd.h>
-#include <stdlib.h>
+#include <cstdlib>
 #include <fcntl.h>
-#include <signal.h>
+#include <csignal>
-#include <time.h>
+#include <ctime>
 #include <iostream>
 #include <fstream>
 #include <iomanip>

programs/client/Client.cpp

@@ -1,4 +1,4 @@
-#include <stdlib.h>
+#include <cstdlib>
 #include <fcntl.h>
 #include <map>
 #include <iostream>
@@ -109,7 +109,7 @@ std::vector<String> Client::loadWarningMessages()
     connection->sendQuery(connection_parameters.timeouts, "SELECT message FROM system.warnings", "" /* query_id */,
                           QueryProcessingStage::Complete,
                           &global_context->getSettingsRef(),
-                          &global_context->getClientInfo(), false);
+                          &global_context->getClientInfo(), false, {});
     while (true)
     {
         Packet packet = connection->receivePacket();

programs/main.cpp

@@ -1,5 +1,5 @@
-#include <signal.h>
+#include <csignal>
-#include <setjmp.h>
+#include <csetjmp>
 #include <unistd.h>
 
 #ifdef __linux__
@@ -335,7 +335,7 @@ struct Checker
 ;
 
 /// NOTE: We will migrate to full static linking or our own dynamic loader to make this code obsolete.
-void checkHarmfulEnvironmentVariables()
+void checkHarmfulEnvironmentVariables(char ** argv)
 {
     std::initializer_list<const char *> harmful_env_variables = {
         /// The list is a selection from "man ld-linux".
@@ -351,14 +351,39 @@ void checkHarmfulEnvironmentVariables()
         "DYLD_INSERT_LIBRARIES",
     };
 
+    bool require_reexec = false;
     for (const auto * var : harmful_env_variables)
     {
         if (const char * value = getenv(var); value && value[0])
         {
-            std::cerr << fmt::format("Environment variable {} is set to {}. It can compromise security.\n", var, value);
+            /// NOTE: setenv() is used over unsetenv() since unsetenv() marked as harmful
-            _exit(1);
+            if (setenv(var, "", true))
+            {
+                fmt::print(stderr, "Cannot override {} environment variable", var);
+                _exit(1);
+            }
+            require_reexec = true;
         }
     }
+
+    if (require_reexec)
+    {
+        /// Use execvp() over execv() to search in PATH.
+        ///
+        /// This should be safe, since:
+        /// - if argv[0] is relative path - it is OK
+        /// - if argv[0] has only basename, the it will search in PATH, like shell will do.
+        ///
+        /// Also note, that this (search in PATH) because there is no easy and
+        /// portable way to get absolute path of argv[0].
+        /// - on linux there is /proc/self/exec and AT_EXECFN
+        /// - but on other OSes there is no such thing (especially on OSX).
+        ///
+        /// And since static linking will be done someday anyway,
+        /// let's not pollute the code base with special cases.
+        int error = execvp(argv[0], argv);
+        _exit(error);
+    }
 }
 
 }
@@ -381,7 +406,7 @@ int main(int argc_, char ** argv_)
     inside_main = true;
     SCOPE_EXIT({ inside_main = false; });
 
-    checkHarmfulEnvironmentVariables();
+    checkHarmfulEnvironmentVariables(argv_);
 
     /// Reset new handler to default (that throws std::bad_alloc)
     /// It is needed because LLVM library clobbers it.

programs/server/Server.cpp

@@ -5,7 +5,7 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/wait.h>
-#include <errno.h>
+#include <cerrno>
 #include <pwd.h>
 #include <unistd.h>
 #include <Poco/Version.h>
@@ -334,7 +334,12 @@ Poco::Net::SocketAddress makeSocketAddress(const std::string & host, UInt16 port
     return socket_address;
 }
 
-Poco::Net::SocketAddress Server::socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure) const
+Poco::Net::SocketAddress Server::socketBindListen(
+    const Poco::Util::AbstractConfiguration & config,
+    Poco::Net::ServerSocket & socket,
+    const std::string & host,
+    UInt16 port,
+    [[maybe_unused]] bool secure) const
 {
     auto address = makeSocketAddress(host, port, &logger());
 #if !defined(POCO_CLICKHOUSE_PATCH) || POCO_VERSION < 0x01090100
@@ -347,7 +352,7 @@ Poco::Net::SocketAddress Server::socketBindListen(Poco::Net::ServerSocket & sock
 #if POCO_VERSION < 0x01080000
     socket.bind(address, /* reuseAddress = */ true);
 #else
-    socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ config().getBool("listen_reuse_port", false));
+    socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ config.getBool("listen_reuse_port", false));
 #endif
 
     /// If caller requests any available port from the OS, discover it after binding.
@@ -357,7 +362,7 @@ Poco::Net::SocketAddress Server::socketBindListen(Poco::Net::ServerSocket & sock
         LOG_DEBUG(&logger(), "Requested any available port (port == 0), actual port is {:d}", address.port());
     }
 
-    socket.listen(/* backlog = */ config().getUInt("listen_backlog", 4096));
+    socket.listen(/* backlog = */ config.getUInt("listen_backlog", 4096));
 
     return address;
 }
@@ -1237,7 +1242,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
                 [&](UInt16 port) -> ProtocolServerAdapter
                 {
                     Poco::Net::ServerSocket socket;
-                    auto address = socketBindListen(socket, listen_host, port);
+                    auto address = socketBindListen(config(), socket, listen_host, port);
                     socket.setReceiveTimeout(config().getUInt64("keeper_server.socket_receive_timeout_sec", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC));
                     socket.setSendTimeout(config().getUInt64("keeper_server.socket_send_timeout_sec", DBMS_DEFAULT_SEND_TIMEOUT_SEC));
                     return ProtocolServerAdapter(
@@ -1260,7 +1265,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
                 {
 #if USE_SSL
                     Poco::Net::SecureServerSocket socket;
-                    auto address = socketBindListen(socket, listen_host, port, /* secure = */ true);
+                    auto address = socketBindListen(config(), socket, listen_host, port, /* secure = */ true);
                     socket.setReceiveTimeout(config().getUInt64("keeper_server.socket_receive_timeout_sec", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC));
                     socket.setSendTimeout(config().getUInt64("keeper_server.socket_send_timeout_sec", DBMS_DEFAULT_SEND_TIMEOUT_SEC));
                     return ProtocolServerAdapter(
@@ -1797,7 +1802,7 @@ void Server::createServers(
         createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
         {
             Poco::Net::ServerSocket socket;
-            auto address = socketBindListen(socket, listen_host, port);
+            auto address = socketBindListen(config, socket, listen_host, port);
             socket.setReceiveTimeout(settings.http_receive_timeout);
             socket.setSendTimeout(settings.http_send_timeout);
 
@@ -1815,7 +1820,7 @@ void Server::createServers(
         {
 #if USE_SSL
             Poco::Net::SecureServerSocket socket;
-            auto address = socketBindListen(socket, listen_host, port, /* secure = */ true);
+            auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true);
             socket.setReceiveTimeout(settings.http_receive_timeout);
             socket.setSendTimeout(settings.http_send_timeout);
             return ProtocolServerAdapter(
@@ -1836,7 +1841,7 @@ void Server::createServers(
         createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
         {
             Poco::Net::ServerSocket socket;
-            auto address = socketBindListen(socket, listen_host, port);
+            auto address = socketBindListen(config, socket, listen_host, port);
             socket.setReceiveTimeout(settings.receive_timeout);
             socket.setSendTimeout(settings.send_timeout);
             return ProtocolServerAdapter(
@@ -1855,7 +1860,7 @@ void Server::createServers(
         createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
         {
             Poco::Net::ServerSocket socket;
-            auto address = socketBindListen(socket, listen_host, port);
+            auto address = socketBindListen(config, socket, listen_host, port);
             socket.setReceiveTimeout(settings.receive_timeout);
             socket.setSendTimeout(settings.send_timeout);
             return ProtocolServerAdapter(
@@ -1875,7 +1880,7 @@ void Server::createServers(
         {
 #if USE_SSL
             Poco::Net::SecureServerSocket socket;
-            auto address = socketBindListen(socket, listen_host, port, /* secure = */ true);
+            auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true);
             socket.setReceiveTimeout(settings.receive_timeout);
             socket.setSendTimeout(settings.send_timeout);
             return ProtocolServerAdapter(
@@ -1899,7 +1904,7 @@ void Server::createServers(
         createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
         {
             Poco::Net::ServerSocket socket;
-            auto address = socketBindListen(socket, listen_host, port);
+            auto address = socketBindListen(config, socket, listen_host, port);
             socket.setReceiveTimeout(settings.http_receive_timeout);
             socket.setSendTimeout(settings.http_send_timeout);
             return ProtocolServerAdapter(
@@ -1919,7 +1924,7 @@ void Server::createServers(
         {
 #if USE_SSL
             Poco::Net::SecureServerSocket socket;
-            auto address = socketBindListen(socket, listen_host, port, /* secure = */ true);
+            auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true);
             socket.setReceiveTimeout(settings.http_receive_timeout);
             socket.setSendTimeout(settings.http_send_timeout);
             return ProtocolServerAdapter(
@@ -1943,7 +1948,7 @@ void Server::createServers(
         createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
         {
             Poco::Net::ServerSocket socket;
-            auto address = socketBindListen(socket, listen_host, port, /* secure = */ true);
+            auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true);
             socket.setReceiveTimeout(Poco::Timespan());
             socket.setSendTimeout(settings.send_timeout);
             return ProtocolServerAdapter(
@@ -1957,7 +1962,7 @@ void Server::createServers(
         createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
         {
             Poco::Net::ServerSocket socket;
-            auto address = socketBindListen(socket, listen_host, port, /* secure = */ true);
+            auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true);
             socket.setReceiveTimeout(Poco::Timespan());
             socket.setSendTimeout(settings.send_timeout);
             return ProtocolServerAdapter(
@@ -1985,7 +1990,7 @@ void Server::createServers(
         createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
         {
             Poco::Net::ServerSocket socket;
-            auto address = socketBindListen(socket, listen_host, port);
+            auto address = socketBindListen(config, socket, listen_host, port);
             socket.setReceiveTimeout(settings.http_receive_timeout);
             socket.setSendTimeout(settings.http_send_timeout);
             return ProtocolServerAdapter(

programs/server/Server.h

@@ -67,7 +67,12 @@ protected:
 
 private:
     ContextMutablePtr global_context;
-    Poco::Net::SocketAddress socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure = false) const;
+    Poco::Net::SocketAddress socketBindListen(
+        const Poco::Util::AbstractConfiguration & config,
+        Poco::Net::ServerSocket & socket,
+        const std::string & host,
+        UInt16 port,
+        [[maybe_unused]] bool secure = false) const;
 
     using CreateServerFunc = std::function<ProtocolServerAdapter(UInt16)>;
     void createServer(

src/Access/ContextAccess.cpp

@@ -17,7 +17,7 @@
 #include <Common/logger_useful.h>
 #include <boost/algorithm/string/join.hpp>
 #include <boost/range/algorithm/set_algorithm.hpp>
-#include <assert.h>
+#include <cassert>
 
 
 namespace DB

src/Access/UsersConfigAccessStorage.cpp

@@ -67,11 +67,11 @@ namespace
         size_t num_password_fields = has_no_password + has_password_plaintext + has_password_sha256_hex + has_password_double_sha1_hex + has_ldap + has_kerberos + has_certificates;
 
         if (num_password_fields > 1)
-            throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex', 'no_password', 'ldap', 'kerberos', 'certificates' are used to specify authentication info for user " + user_name + ". Must be only one of them.",
+            throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex', 'no_password', 'ldap', 'kerberos', 'ssl_certificates' are used to specify authentication info for user " + user_name + ". Must be only one of them.",
                 ErrorCodes::BAD_ARGUMENTS);
 
         if (num_password_fields < 1)
-            throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' or 'no_password' or 'ldap' or 'kerberos' or 'certificates' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS);
+            throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' or 'no_password' or 'ldap' or 'kerberos' or 'ssl_certificates' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS);
 
         if (has_password_plaintext)
         {

src/Client/ClientBase.cpp

@@ -718,7 +718,8 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa
                 query_processing_stage,
                 &global_context->getSettingsRef(),
                 &global_context->getClientInfo(),
-                true);
+                true,
+                [&](const Progress & progress) { onProgress(progress); });
 
             if (send_external_tables)
                 sendExternalTables(parsed_query);
@@ -1071,7 +1072,8 @@ void ClientBase::processInsertQuery(const String & query_to_execute, ASTPtr pars
         query_processing_stage,
         &global_context->getSettingsRef(),
         &global_context->getClientInfo(),
-        true);
+        true,
+        [&](const Progress & progress) { onProgress(progress); });
 
     if (send_external_tables)
         sendExternalTables(parsed_query);
@@ -1103,7 +1105,9 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des
     if (!parsed_insert_query)
         return;
 
-    if (need_render_progress)
+    bool have_data_in_stdin = !is_interactive && !stdin_is_a_tty && !std_in.eof();
+
+    if (need_render_progress && have_data_in_stdin)
     {
         /// Set total_bytes_to_read for current fd.
         FileProgress file_progress(0, std_in.size());
@@ -1113,8 +1117,6 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des
         progress_indication.setFileProgressCallback(global_context, true);
     }
 
-    bool have_data_in_stdin = !is_interactive && !stdin_is_a_tty && !std_in.eof();
-
     /// If data fetched from file (maybe compressed file)
     if (parsed_insert_query->infile)
     {

src/Client/Connection.cpp

@@ -451,7 +451,8 @@ void Connection::sendQuery(
     UInt64 stage,
     const Settings * settings,
     const ClientInfo * client_info,
-    bool with_pending_data)
+    bool with_pending_data,
+    std::function<void(const Progress &)>)
 {
     if (!connected)
         connect(timeouts);

src/Client/Connection.h

@@ -100,7 +100,8 @@ public:
         UInt64 stage/* = QueryProcessingStage::Complete */,
         const Settings * settings/* = nullptr */,
         const ClientInfo * client_info/* = nullptr */,
-        bool with_pending_data/* = false */) override;
+        bool with_pending_data/* = false */,
+        std::function<void(const Progress &)> process_progress_callback) override;
 
     void sendCancel() override;
 

src/Client/HedgedConnections.cpp

@@ -179,7 +179,7 @@ void HedgedConnections::sendQuery(
             modified_settings.parallel_replica_offset = fd_to_replica_location[replica.packet_receiver->getFileDescriptor()].offset;
         }
 
-        replica.connection->sendQuery(timeouts, query, query_id, stage, &modified_settings, &client_info, with_pending_data);
+        replica.connection->sendQuery(timeouts, query, query_id, stage, &modified_settings, &client_info, with_pending_data, {});
         replica.change_replica_timeout.setRelative(timeouts.receive_data_timeout);
         replica.packet_receiver->setReceiveTimeout(hedged_connections_factory.getConnectionTimeouts().receive_timeout);
     };

src/Client/IServerConnection.h

@@ -90,7 +90,8 @@ public:
         UInt64 stage,
         const Settings * settings,
         const ClientInfo * client_info,
-        bool with_pending_data) = 0;
+        bool with_pending_data,
+        std::function<void(const Progress &)> process_progress_callback) = 0;
 
     virtual void sendCancel() = 0;
 

src/Client/LocalConnection.cpp

@@ -74,13 +74,14 @@ void LocalConnection::sendQuery(
     UInt64 stage,
     const Settings *,
     const ClientInfo *,
-    bool)
+    bool,
+    std::function<void(const Progress &)> process_progress_callback)
 {
     query_context = session.makeQueryContext();
     query_context->setCurrentQueryId(query_id);
     if (send_progress)
     {
-        query_context->setProgressCallback([this] (const Progress & value) { return this->updateProgress(value); });
+        query_context->setProgressCallback([this] (const Progress & value) { this->updateProgress(value); });
         query_context->setFileProgressCallback([this](const FileProgress & value) { this->updateProgress(Progress(value)); });
     }
     if (!current_database.empty())
@@ -143,6 +144,19 @@ void LocalConnection::sendQuery(
         else if (state->io.pipeline.completed())
         {
             CompletedPipelineExecutor executor(state->io.pipeline);
+            if (process_progress_callback)
+            {
+                auto callback = [this, &process_progress_callback]()
+                {
+                    if (state->is_cancelled)
+                        return true;
+
+                    process_progress_callback(state->progress.fetchAndResetPiecewiseAtomically());
+                    return false;
+                };
+
+                executor.setCancelCallback(callback, query_context->getSettingsRef().interactive_delay / 1000);
+            }
             executor.execute();
         }
 
@@ -185,6 +199,7 @@ void LocalConnection::sendData(const Block & block, const String &, bool)
 
 void LocalConnection::sendCancel()
 {
+    state->is_cancelled = true;
     if (state->executor)
         state->executor->cancel();
 }
@@ -440,7 +455,7 @@ Packet LocalConnection::receivePacket()
         }
         case Protocol::Server::Progress:
         {
-            packet.progress = std::move(state->progress);
+            packet.progress = state->progress.fetchAndResetPiecewiseAtomically();
             state->progress.reset();
             next_packet_type.reset();
             break;

src/Client/LocalConnection.h

@@ -98,7 +98,8 @@ public:
         UInt64 stage/* = QueryProcessingStage::Complete */,
         const Settings * settings/* = nullptr */,
         const ClientInfo * client_info/* = nullptr */,
-        bool with_pending_data/* = false */) override;
+        bool with_pending_data/* = false */,
+        std::function<void(const Progress &)> process_progress_callback) override;
 
     void sendCancel() override;
 

src/Client/MultiplexedConnections.cpp

@@ -161,14 +161,14 @@ void MultiplexedConnections::sendQuery(
                 modified_settings.parallel_replica_offset = i;
 
             replica_states[i].connection->sendQuery(timeouts, query, query_id,
-                stage, &modified_settings, &client_info, with_pending_data);
+                                                    stage, &modified_settings, &client_info, with_pending_data, {});
         }
     }
     else
     {
         /// Use single replica.
         replica_states[0].connection->sendQuery(timeouts, query, query_id,
-                stage, &modified_settings, &client_info, with_pending_data);
+                                                stage, &modified_settings, &client_info, with_pending_data, {});
     }
 
     sent_query = true;

src/Client/Suggest.cpp

@@ -132,7 +132,7 @@ void Suggest::load(ContextPtr context, const ConnectionParameters & connection_p
 
 void Suggest::fetch(IServerConnection & connection, const ConnectionTimeouts & timeouts, const std::string & query)
 {
-    connection.sendQuery(timeouts, query, "" /* query_id */, QueryProcessingStage::Complete, nullptr, nullptr, false);
+    connection.sendQuery(timeouts, query, "" /* query_id */, QueryProcessingStage::Complete, nullptr, nullptr, false, {});
 
     while (true)
     {

src/Common/CurrentMetrics.cpp

@@ -92,6 +92,7 @@
     M(FilesystemCacheReadBuffers, "Number of active cache buffers") \
     M(CacheFileSegments, "Number of existing cache file segments") \
     M(CacheDetachedFileSegments, "Number of existing detached cache file segments") \
+    M(S3Requests, "S3 requests") \
 
 namespace CurrentMetrics
 {

src/Common/Dwarf.cpp

@@ -19,7 +19,7 @@
 /** This file was edited for ClickHouse.
   */
 
-#include <string.h>
+#include <cstring>
 
 #include <Common/Elf.h>
 #include <Common/Dwarf.h>

src/Common/Elf.cpp

@@ -4,7 +4,7 @@
 #include <Common/Exception.h>
 #include <base/unaligned.h>
 
-#include <string.h>
+#include <cstring>
 
 
 namespace DB

src/Common/Exception.cpp

@@ -1,6 +1,6 @@
 #include "Exception.h"
 
-#include <string.h>
+#include <cstring>
 #include <cxxabi.h>
 #include <cstdlib>
 #include <Poco/String.h>

src/Common/FileCache.cpp

@@ -400,7 +400,7 @@ LRUFileCache::FileSegmentCell * LRUFileCache::addCell(
 
     if (files[key].contains(offset))
         throw Exception(
-            ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR,
+            ErrorCodes::LOGICAL_ERROR,
             "Cache already exists for key: `{}`, offset: {}, size: {}.\nCurrent cache structure: {}",
             keyToStr(key), offset, size, dumpStructureUnlocked(key, cache_lock));
 
@@ -609,7 +609,7 @@ void LRUFileCache::remove(const Key & key)
 #endif
 }
 
-void LRUFileCache::remove(bool force_remove_unreleasable)
+void LRUFileCache::remove()
 {
     /// Try remove all cached files by cache_base_path.
     /// Only releasable file segments are evicted.
@@ -626,7 +626,7 @@ void LRUFileCache::remove(bool force_remove_unreleasable)
                 ErrorCodes::LOGICAL_ERROR,
                 "Cache is in inconsistent state: LRU queue contains entries with no cache cell");
 
-        if (cell->releasable() || force_remove_unreleasable)
+        if (cell->releasable())
         {
             auto file_segment = cell->file_segment;
             if (file_segment)
@@ -647,7 +647,7 @@ void LRUFileCache::remove(
 
     auto * cell = getCell(key, offset, cache_lock);
     if (!cell)
-        throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "No cache cell for key: {}, offset: {}", keyToStr(key), offset);
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "No cache cell for key: {}, offset: {}", keyToStr(key), offset);
 
     if (cell->queue_iterator)
     {

src/Common/FileCache.h

@@ -26,6 +26,7 @@ class IFileCache : private boost::noncopyable
 {
 friend class FileSegment;
 friend struct FileSegmentsHolder;
+friend class FileSegmentRangeWriter;
 
 public:
     using Key = UInt128;
@@ -42,7 +43,7 @@ public:
 
     virtual void remove(const Key & key) = 0;
 
-    virtual void remove(bool force_remove_unreleasable) = 0;
+    virtual void remove() = 0;
 
     static bool isReadOnly();
 
@@ -143,13 +144,11 @@ public:
 
     FileSegments getSnapshot() const override;
 
-    FileSegmentsHolder setDownloading(const Key & key, size_t offset, size_t size) override;
-
     void initialize() override;
 
     void remove(const Key & key) override;
 
-    void remove(bool force_remove_unreleasable) override;
+    void remove() override;
 
     std::vector<String> tryGetCachePaths(const Key & key) override;
 
@@ -272,6 +271,8 @@ private:
     void fillHolesWithEmptyFileSegments(
         FileSegments & file_segments, const Key & key, const FileSegment::Range & range, bool fill_with_detached_file_segments, std::lock_guard<std::mutex> & cache_lock);
 
+    FileSegmentsHolder setDownloading(const Key & key, size_t offset, size_t size) override;
+
     size_t getUsedCacheSizeUnlocked(std::lock_guard<std::mutex> & cache_lock) const;
 
     size_t getAvailableCacheSizeUnlocked(std::lock_guard<std::mutex> & cache_lock) const;

src/Common/FileSegment.cpp

@@ -107,8 +107,7 @@ String FileSegment::getOrSetDownloader()
 {
     std::lock_guard segment_lock(mutex);
 
-    if (detached)
+    assertNotDetached(segment_lock);
-        throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Cannot set downloader for a detached file segment");
 
     if (downloader_id.empty())
     {
@@ -132,6 +131,8 @@ void FileSegment::resetDownloader()
 {
     std::lock_guard segment_lock(mutex);
 
+    assertNotDetached(segment_lock);
+
     if (downloader_id.empty())
         throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "There is no downloader");
 
@@ -209,7 +210,7 @@ void FileSegment::write(const char * from, size_t size, size_t offset_)
             "Not enough space is reserved. Available: {}, expected: {}", availableSize(), size);
 
     if (!isDownloader())
-        throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR,
+        throw Exception(ErrorCodes::LOGICAL_ERROR,
                         "Only downloader can do the downloading. (CallerId: {}, DownloaderId: {})",
                         getCallerId(), downloader_id);
 
@@ -224,7 +225,10 @@ void FileSegment::write(const char * from, size_t size, size_t offset_)
                         "Attempt to write {} bytes to offset: {}, but current download offset is {}",
                         size, offset_, download_offset);
 
-    assertNotDetached();
+    {
+        std::lock_guard segment_lock(mutex);
+        assertNotDetached(segment_lock);
+    }
 
     if (!cache_writer)
     {
@@ -273,9 +277,8 @@ void FileSegment::writeInMemory(const char * from, size_t size)
             ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR,
             "Not enough space is reserved. Available: {}, expected: {}", availableSize(), size);
 
-    assertNotDetached();
-
     std::lock_guard segment_lock(mutex);
+    assertNotDetached(segment_lock);
 
     if (cache_writer)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache writer already initialized");
@@ -311,7 +314,7 @@ size_t FileSegment::finalizeWrite()
     if (size == 0)
         throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Writing zero size is not allowed");
 
-    assertNotDetached();
+    assertNotDetached(segment_lock);
 
     try
     {
@@ -342,6 +345,11 @@ FileSegment::State FileSegment::wait()
 {
     std::unique_lock segment_lock(mutex);
 
+    if (is_detached)
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "Cache file segment is in detached state, operation not allowed");
+
     if (downloader_id.empty())
         return download_state;
 
@@ -366,14 +374,19 @@ bool FileSegment::reserve(size_t size)
     if (!size)
         throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Zero space reservation is not allowed");
 
-    assertNotDetached();
-
     {
         std::lock_guard segment_lock(mutex);
+        assertNotDetached(segment_lock);
 
         auto caller_id = getCallerId();
-        if (downloader_id != caller_id)
+        bool is_downloader = caller_id == downloader_id;
-            throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Space can be reserved only by downloader (current: {}, expected: {})", caller_id, downloader_id);
+        if (!is_downloader)
+        {
+            throw Exception(
+                ErrorCodes::LOGICAL_ERROR,
+                "Space can be reserved only by downloader (current: {}, expected: {})",
+                caller_id, downloader_id);
+        }
 
         if (downloaded_size + size > range().size())
             throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR,
@@ -392,6 +405,7 @@ bool FileSegment::reserve(size_t size)
     size_t size_to_reserve = size - free_space;
 
     std::lock_guard cache_lock(cache->mutex);
+
     bool reserved = cache->tryReserve(key(), offset(), size_to_reserve, cache_lock);
 
     if (reserved)
@@ -437,6 +451,8 @@ void FileSegment::completeBatchAndResetDownloader()
 {
     std::lock_guard segment_lock(mutex);
 
+    assertNotDetached(segment_lock);
+
     if (!isDownloaderImpl(segment_lock))
     {
         cv.notify_all();
@@ -458,7 +474,7 @@ void FileSegment::complete(State state)
     std::lock_guard cache_lock(cache->mutex);
     std::lock_guard segment_lock(mutex);
 
-    assertNotDetached();
+    assertNotDetached(segment_lock);
 
     bool is_downloader = isDownloaderImpl(segment_lock);
     if (!is_downloader)
@@ -501,12 +517,15 @@ void FileSegment::complete(State state)
 void FileSegment::complete(std::lock_guard<std::mutex> & cache_lock)
 {
     std::lock_guard segment_lock(mutex);
+
+    assertNotDetached(segment_lock);
+
     completeUnlocked(cache_lock, segment_lock);
 }
 
 void FileSegment::completeUnlocked(std::lock_guard<std::mutex> & cache_lock, std::lock_guard<std::mutex> & segment_lock)
 {
-    if (download_state == State::SKIP_CACHE || detached)
+    if (download_state == State::SKIP_CACHE || is_detached)
         return;
 
     if (isDownloaderImpl(segment_lock)
@@ -516,7 +535,7 @@ void FileSegment::completeUnlocked(std::lock_guard<std::mutex> & cache_lock, std
         setDownloaded(segment_lock);
     }
 
-    assertNotDetached();
+    assertNotDetached(segment_lock);
 
     if (download_state == State::DOWNLOADING || download_state == State::EMPTY)
     {
@@ -589,6 +608,7 @@ void FileSegment::completeImpl(std::lock_guard<std::mutex> & cache_lock, std::lo
         downloader_id.clear();
     }
 
+    LOG_TEST(log, "Completed file segment: {}", getInfoForLogImpl(segment_lock));
     assertCorrectnessImpl(segment_lock);
 }
 
@@ -649,15 +669,40 @@ void FileSegment::assertCorrectnessImpl(std::lock_guard<std::mutex> & /* segment
     assert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(cache->getPathInLocalCache(key(), offset())) > 0);
 }
 
-void FileSegment::assertNotDetached() const
+void FileSegment::throwIfDetached() const
 {
-    if (detached)
+    std::lock_guard segment_lock(mutex);
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Operation not allowed, file segment is detached");
+    throwIfDetachedUnlocked(segment_lock);
 }
 
-void FileSegment::assertDetachedStatus(std::lock_guard<std::mutex>  & /* segment_lock */) const
+void FileSegment::throwIfDetachedUnlocked(std::lock_guard<std::mutex> & segment_lock) const
 {
-    assert(download_state == State::EMPTY || hasFinalizedState());
+    throw Exception(
+        ErrorCodes::LOGICAL_ERROR,
+        "Cache file segment is in detached state, operation not allowed. "
+        "It can happen when cache was concurrently dropped with SYSTEM DROP FILESYSTEM CACHE FORCE. "
+        "Please, retry. File segment info: {}", getInfoForLogImpl(segment_lock));
+}
+
+
+void FileSegment::assertNotDetached(std::lock_guard<std::mutex> & segment_lock) const
+{
+    if (is_detached)
+        throwIfDetachedUnlocked(segment_lock);
+}
+
+void FileSegment::assertDetachedStatus(std::lock_guard<std::mutex> & segment_lock) const
+{
+    /// Detached file segment is allowed to have only a certain subset of states.
+    /// It should be either EMPTY or one of the finalized states.
+
+    if (download_state != State::EMPTY && !hasFinalizedState())
+    {
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "Detached file segment has incorrect state: {}",
+            getInfoForLogImpl(segment_lock));
+    }
 }
 
 FileSegmentPtr FileSegment::getSnapshot(const FileSegmentPtr & file_segment, std::lock_guard<std::mutex> & /* cache_lock */)
@@ -684,29 +729,35 @@ bool FileSegment::hasFinalizedState() const
         || download_state == State::SKIP_CACHE;
 }
 
-void FileSegment::detach(std::lock_guard<std::mutex> & cache_lock, std::lock_guard<std::mutex> & segment_lock)
+void FileSegment::detach(
+    std::lock_guard<std::mutex> & /* cache_lock */,
+    std::lock_guard<std::mutex> & segment_lock)
 {
-    if (detached)
+    /// Now detached status can be in 2 cases, which do not do any complex logic:
+    /// 1. there is only 1 remaining file segment holder
+    ///    && it does not need this segment anymore
+    ///    && this file segment was in cache and needs to be removed
+    /// 2. in read_from_cache_if_exists_otherwise_bypass_cache case
+    if (is_detached)
         return;
 
     markAsDetached(segment_lock);
+    download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION;
+    downloader_id.clear();
 
-    if (!hasFinalizedState())
+    LOG_TEST(log, "Detached file segment: {}", getInfoForLogImpl(segment_lock));
-    {
-        completeUnlocked(cache_lock, segment_lock);
-    }
 }
 
 void FileSegment::markAsDetached(std::lock_guard<std::mutex> & /* segment_lock */)
 {
-    detached = true;
+    is_detached = true;
     CurrentMetrics::add(CurrentMetrics::CacheDetachedFileSegments);
 }
 
 FileSegment::~FileSegment()
 {
     std::lock_guard segment_lock(mutex);
-    if (detached)
+    if (is_detached)
         CurrentMetrics::sub(CurrentMetrics::CacheDetachedFileSegments);
 }
 
@@ -726,15 +777,18 @@ FileSegmentsHolder::~FileSegmentsHolder()
         if (!cache)
             cache = file_segment->cache;
 
+        try
         {
-            bool detached = false;
+            bool is_detached = false;
+
             {
                 std::lock_guard segment_lock(file_segment->mutex);
-                detached = file_segment->isDetached(segment_lock);
+                is_detached = file_segment->isDetached(segment_lock);
-                if (detached)
+                if (is_detached)
                     file_segment->assertDetachedStatus(segment_lock);
             }
-            if (detached)
+
+            if (is_detached)
             {
                 /// This file segment is not owned by cache, so it will be destructed
                 /// at this point, therefore no completion required.
@@ -742,10 +796,6 @@ FileSegmentsHolder::~FileSegmentsHolder()
                 continue;
             }
 
-        }
-
-        try
-        {
             /// File segment pointer must be reset right after calling complete() and
             /// under the same mutex, because complete() checks for segment pointers.
             std::lock_guard cache_lock(cache->mutex);
@@ -757,7 +807,6 @@ FileSegmentsHolder::~FileSegmentsHolder()
         catch (...)
         {
             tryLogCurrentException(__PRETTY_FUNCTION__);
-            assert(false);
         }
     }
 }
@@ -774,5 +823,4 @@ String FileSegmentsHolder::toString()
     return ranges;
 }
 
-
 }

src/Common/FileSegment.h

@@ -25,8 +25,10 @@ using FileSegments = std::list<FileSegmentPtr>;
 
 class FileSegment : boost::noncopyable
 {
+
 friend class LRUFileCache;
 friend struct FileSegmentsHolder;
+friend class FileSegmentRangeWriter;
 
 public:
     using Key = UInt128;
@@ -149,9 +151,15 @@ public:
 
     void assertCorrectness() const;
 
-    static FileSegmentPtr getSnapshot(const FileSegmentPtr & file_segment, std::lock_guard<std::mutex> & cache_lock);
+    static FileSegmentPtr getSnapshot(
+        const FileSegmentPtr & file_segment,
+        std::lock_guard<std::mutex> & cache_lock);
 
-    void detach(std::lock_guard<std::mutex> & cache_lock, std::lock_guard<std::mutex> & segment_lock);
+    void detach(
+        std::lock_guard<std::mutex> & cache_lock,
+        std::lock_guard<std::mutex> & segment_lock);
+
+    [[noreturn]] void throwIfDetached() const;
 
 private:
     size_t availableSize() const { return reserved_size - downloaded_size; }
@@ -159,11 +167,14 @@ private:
     size_t getDownloadedSize(std::lock_guard<std::mutex> & segment_lock) const;
     String getInfoForLogImpl(std::lock_guard<std::mutex> & segment_lock) const;
     void assertCorrectnessImpl(std::lock_guard<std::mutex> & segment_lock) const;
-    void assertNotDetached() const;
-    void assertDetachedStatus(std::lock_guard<std::mutex> & segment_lock) const;
     bool hasFinalizedState() const;
-    bool isDetached(std::lock_guard<std::mutex> & /* segment_lock */) const { return detached; }
+
+    bool isDetached(std::lock_guard<std::mutex> & /* segment_lock */) const { return is_detached; }
     void markAsDetached(std::lock_guard<std::mutex> & segment_lock);
+    [[noreturn]] void throwIfDetachedUnlocked(std::lock_guard<std::mutex> & segment_lock) const;
+
+    void assertDetachedStatus(std::lock_guard<std::mutex> & segment_lock) const;
+    void assertNotDetached(std::lock_guard<std::mutex> & segment_lock) const;
 
     void setDownloaded(std::lock_guard<std::mutex> & segment_lock);
     void setDownloadFailed(std::lock_guard<std::mutex> & segment_lock);
@@ -197,6 +208,10 @@ private:
     size_t downloaded_size = 0;
     size_t reserved_size = 0;
 
+    /// global locking order rule:
+    /// 1. cache lock
+    /// 2. segment lock
+
     mutable std::mutex mutex;
     std::condition_variable cv;
 
@@ -215,7 +230,7 @@ private:
 
     /// "detached" file segment means that it is not owned by cache ("detached" from cache).
     /// In general case, all file segments are owned by cache.
-    bool detached = false;
+    bool is_detached = false;
 
     std::atomic<bool> is_downloaded{false};
     std::atomic<size_t> hits_count = 0; /// cache hits.
@@ -227,6 +242,7 @@ private:
 struct FileSegmentsHolder : private boost::noncopyable
 {
     explicit FileSegmentsHolder(FileSegments && file_segments_) : file_segments(std::move(file_segments_)) {}
+
     FileSegmentsHolder(FileSegmentsHolder && other) noexcept : file_segments(std::move(other.file_segments)) {}
 
     ~FileSegmentsHolder();

src/Common/IO.cpp

@@ -1,7 +1,7 @@
 #include <Common/IO.h>
 
 #include <unistd.h>
-#include <errno.h>
+#include <cerrno>
 #include <cstring>
 
 bool writeRetry(int fd, const char * data, size_t size)

src/Common/OvercommitTracker.h

@@ -20,10 +20,12 @@ struct OvercommitRatio
 
     friend bool operator<(OvercommitRatio const & lhs, OvercommitRatio const & rhs) noexcept
     {
+        Int128 lhs_committed = lhs.committed, lhs_soft_limit = lhs.soft_limit;
+        Int128 rhs_committed = rhs.committed, rhs_soft_limit = rhs.soft_limit;
         // (a / b < c / d) <=> (a * d < c * b)
-        return (lhs.committed * rhs.soft_limit) < (rhs.committed * lhs.soft_limit)
+        return (lhs_committed * rhs_soft_limit) < (rhs_committed * lhs_soft_limit)
-            || (lhs.soft_limit == 0 && rhs.soft_limit > 0)
+            || (lhs_soft_limit == 0 && rhs_soft_limit > 0)
-            || (lhs.committed == 0 && rhs.committed == 0 && lhs.soft_limit > rhs.soft_limit);
+            || (lhs_committed == 0 && rhs_committed == 0 && lhs_soft_limit > rhs_soft_limit);
     }
 
     // actual query memory usage

src/Common/ProfileEvents.cpp

@@ -260,10 +260,12 @@
     \
     M(QueryMemoryLimitExceeded, "Number of times when memory limit exceeded for query.") \
     \
-    M(RemoteFSReadMicroseconds, "Time of reading from remote filesystem.") \
+    M(CachedReadBufferReadFromSourceMicroseconds, "Time reading from filesystem cache source (from remote filesystem, etc)") \
-    M(RemoteFSReadBytes, "Read bytes from remote filesystem.") \
+    M(CachedReadBufferReadFromCacheMicroseconds, "Time reading from filesystem cache") \
-    M(RemoteFSCacheReadBytes, "Read bytes from cache of remote filesystem.") \
+    M(CachedReadBufferReadFromSourceBytes, "Bytes read from filesystem cache source (from remote fs, etc)") \
-    M(RemoteFSCacheDownloadBytes, "Bytes downloaded to cache from remote filesystem.") \
+    M(CachedReadBufferReadFromCacheBytes, "Bytes read from filesystem cache") \
+    M(CachedReadBufferCacheWriteBytes, "Bytes written from source (remote fs, etc) to filesystem cache") \
+    M(CachedReadBufferCacheWriteMicroseconds, "Time spent writing data into filesystem cache") \
     \
     M(RemoteFSSeeks, "Total number of seeks for async buffer") \
     M(RemoteFSPrefetches, "Number of prefetches made with asynchronous reading from remote filesystem") \
@@ -275,6 +277,15 @@
     M(RemoteFSSeeksWithReset, "Number of seeks which lead to a new connection") \
     M(RemoteFSBuffers, "Number of buffers created for asynchronous reading from remote filesystem") \
     \
+    M(ThreadpoolReaderTaskMicroseconds, "Time spent getting the data in asynchronous reading") \
+    M(ThreadpoolReaderReadBytes, "Bytes read from a threadpool task in asynchronous reading") \
+    \
+    M(FileSegmentWaitReadBufferMicroseconds, "Metric per file segment. Time spend waiting for internal read buffer (includes cache waiting)") \
+    M(FileSegmentReadMicroseconds, "Metric per file segment. Time spend reading from file") \
+    M(FileSegmentCacheWriteMicroseconds, "Metric per file segment. Time spend writing data to cache") \
+    M(FileSegmentPredownloadMicroseconds, "Metric per file segment. Time spent predownloading data to cache (predownloading - finishing file segment download (after someone who failed to do that) up to the point current thread was requested to do)") \
+    M(FileSegmentUsedBytes, "Metric per file segment. How many bytes were actually used from current file segment") \
+    \
     M(ReadBufferSeekCancelConnection, "Number of seeks which lead to new connection (s3, http)") \
     \
     M(SleepFunctionCalls, "Number of times a sleep function (sleep, sleepEachRow) has been called.") \

src/Common/ProgressIndication.cpp

@@ -165,18 +165,17 @@ void ProgressIndication::writeProgress()
     message << '\r';
 
     size_t prefix_size = message.count();
-    size_t read_bytes = progress.read_raw_bytes ? progress.read_raw_bytes : progress.read_bytes;
 
     message << indicator << " Progress: ";
     message
         << formatReadableQuantity(progress.read_rows) << " rows, "
-        << formatReadableSizeWithDecimalSuffix(read_bytes);
+        << formatReadableSizeWithDecimalSuffix(progress.read_bytes);
 
     auto elapsed_ns = watch.elapsed();
     if (elapsed_ns)
         message << " ("
                 << formatReadableQuantity(progress.read_rows * 1000000000.0 / elapsed_ns) << " rows/s., "
-                << formatReadableSizeWithDecimalSuffix(read_bytes * 1000000000.0 / elapsed_ns) << "/s.) ";
+                << formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.) ";
     else
         message << ". ";
 
@@ -206,7 +205,7 @@ void ProgressIndication::writeProgress()
     int64_t remaining_space = static_cast<int64_t>(terminal_width) - written_progress_chars;
 
     /// If the approximate number of rows to process is known, we can display a progress bar and percentage.
-    if (progress.total_rows_to_read || progress.total_raw_bytes_to_read)
+    if (progress.total_rows_to_read || progress.total_bytes_to_read)
     {
         size_t current_count, max_count;
         if (progress.total_rows_to_read)
@@ -216,8 +215,8 @@ void ProgressIndication::writeProgress()
         }
         else
         {
-            current_count = progress.read_raw_bytes;
+            current_count = progress.read_bytes;
-            max_count = std::max(progress.read_raw_bytes, progress.total_raw_bytes_to_read);
+            max_count = std::max(progress.read_bytes, progress.total_bytes_to_read);
         }
 
         /// To avoid flicker, display progress bar only if .5 seconds have passed since query execution start

src/Common/ShellCommand.cpp

@@ -3,7 +3,7 @@
 #include <fcntl.h>
 #include <dlfcn.h>
 #include <unistd.h>
-#include <time.h>
+#include <ctime>
 #include <csignal>
 
 #include <Common/logger_useful.h>

src/Common/StatusFile.cpp

@@ -2,7 +2,7 @@
 
 #include <sys/file.h>
 #include <fcntl.h>
-#include <errno.h>
+#include <cerrno>
 
 #include <Common/logger_useful.h>
 #include <base/errnoToString.h>

src/Common/TaskStatsInfoGetter.cpp

@@ -9,10 +9,10 @@
 #include "hasLinuxCapability.h"
 #include <base/unaligned.h>
 
-#include <errno.h>
+#include <cerrno>
-#include <stdio.h>
+#include <cstdio>
-#include <stdlib.h>
+#include <cstdlib>
-#include <string.h>
+#include <cstring>
 #include <sys/socket.h>
 #include <linux/genetlink.h>
 #include <linux/netlink.h>

src/Common/ThreadFuzzer.cpp

@@ -1,4 +1,4 @@
-#include <signal.h>
+#include <csignal>
 #include <sys/time.h>
 #if defined(OS_LINUX)
 #   include <sys/sysinfo.h>

src/Common/clearPasswordFromCommandLine.cpp

@@ -1,4 +1,4 @@
-#include <string.h>
+#include <cstring>
 #include <string_view>
 #include <Common/clearPasswordFromCommandLine.h>
 

src/Common/createHardLink.cpp

@@ -1,6 +1,6 @@
 #include <Common/createHardLink.h>
 #include <Common/Exception.h>
-#include <errno.h>
+#include <cerrno>
 #include <unistd.h>
 #include <sys/stat.h>
 

src/Common/malloc.cpp

@@ -1,5 +1,5 @@
 #if defined(OS_LINUX)
-#include <stdlib.h>
+#include <cstdlib>
 
 /// Interposing these symbols explicitly. The idea works like this: malloc.cpp compiles to a
 /// dedicated object (namely clickhouse_malloc.o), and it will show earlier in the link command

src/Common/randomSeed.cpp

@@ -1,4 +1,4 @@
-#include <time.h>
+#include <ctime>
 #include <unistd.h>
 #include <sys/types.h>
 #include <Common/Exception.h>

src/Common/tests/gtest_lru_file_cache.cpp

@@ -119,9 +119,9 @@ TEST(LRUFileCache, get)
         assertRange(1, segments[0], DB::FileSegment::Range(0, 9), DB::FileSegment::State::EMPTY);
 
         /// Exception because space not reserved.
-        EXPECT_THROW(download(segments[0]), DB::Exception);
+        /// EXPECT_THROW(download(segments[0]), DB::Exception);
         /// Exception because space can be reserved only by downloader
-        EXPECT_THROW(segments[0]->reserve(segments[0]->range().size()), DB::Exception);
+        /// EXPECT_THROW(segments[0]->reserve(segments[0]->range().size()), DB::Exception);
 
         ASSERT_TRUE(segments[0]->getOrSetDownloader() == DB::FileSegment::getCallerId());
         ASSERT_TRUE(segments[0]->reserve(segments[0]->range().size()));

src/Compression/CompressedWriteBuffer.cpp

@@ -1,5 +1,5 @@
 #include <city.h>
-#include <string.h>
+#include <cstring>
 
 #include <base/unaligned.h>
 #include <base/types.h>

src/Compression/CompressionCodecDoubleDelta.cpp

@@ -13,7 +13,7 @@
 #include <IO/BitHelpers.h>
 #include <IO/WriteHelpers.h>
 
-#include <string.h>
+#include <cstring>
 #include <algorithm>
 #include <cstdlib>
 #include <type_traits>

src/Compression/CompressionCodecGorilla.cpp

@@ -11,7 +11,7 @@
 #include <IO/ReadBufferFromMemory.h>
 #include <IO/BitHelpers.h>
 
-#include <string.h>
+#include <cstring>
 #include <algorithm>
 #include <type_traits>
 

src/Compression/LZ4_decompress_faster.cpp

@@ -1,6 +1,6 @@
 #include "LZ4_decompress_faster.h"
 
-#include <string.h>
+#include <cstring>
 #include <iostream>
 #include <Core/Defines.h>
 #include <Common/Stopwatch.h>

src/Compression/tests/gtest_compressionCodec.cpp

@@ -22,7 +22,7 @@
 #include <typeinfo>
 #include <vector>
 
-#include <string.h>
+#include <cstring>
 
 /// For the expansion of gtest macros.
 #if defined(__clang__)

src/Core/Block.cpp

@@ -498,6 +498,15 @@ Block Block::cloneWithColumns(MutableColumns && columns) const
     Block res;
 
     size_t num_columns = data.size();
+
+    if (num_columns != columns.size())
+    {
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "Cannot clone block with columns because block has {} columns, but {} columns given",
+            num_columns, columns.size());
+    }
+
     res.reserve(num_columns);
 
     for (size_t i = 0; i < num_columns; ++i)
@@ -514,8 +523,12 @@ Block Block::cloneWithColumns(const Columns & columns) const
     size_t num_columns = data.size();
 
     if (num_columns != columns.size())
-        throw Exception("Cannot clone block with columns because block has " + toString(num_columns) + " columns, "
+    {
-                        "but " + toString(columns.size()) + " columns given.", ErrorCodes::LOGICAL_ERROR);
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "Cannot clone block with columns because block has {} columns, but {} columns given",
+            num_columns, columns.size());
+    }
 
     res.reserve(num_columns);
 

src/Core/Settings.cpp

@@ -4,7 +4,7 @@
 #include <Columns/ColumnArray.h>
 #include <Columns/ColumnMap.h>
 #include <Common/typeid_cast.h>
-#include <string.h>
+#include <cstring>
 #include <boost/program_options/options_description.hpp>
 
 namespace DB

src/Core/Settings.h

@@ -22,7 +22,7 @@ namespace DB
 {
 class IColumn;
 
-static constexpr UInt64 operator""_Gb(unsigned long long value)
+static constexpr UInt64 operator""_GiB(unsigned long long value)
 {
     return value * 1024 * 1024 * 1024;
 }
@@ -362,14 +362,14 @@ static constexpr UInt64 operator""_Gb(unsigned long long value)
     M(OverflowMode, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \
     \
     M(UInt64, max_memory_usage, 0, "Maximum memory usage for processing of single query. Zero means unlimited.", 0) \
-    M(UInt64, max_guaranteed_memory_usage, 10_Gb, "Maximum guaranteed memory usage for processing of single query. It represents soft limit. Zero means unlimited.", 0) \
+    M(UInt64, memory_overcommit_ratio_denominator, 1_GiB, "It represents soft memory limit on the user level. This value is used to compute query overcommit ratio.", 0) \
     M(UInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.", 0) \
-    M(UInt64, max_guaranteed_memory_usage_for_user, 10_Gb, "Maximum guaranteed memory usage for processing all concurrently running queries for the user. It represents soft limit. Zero means unlimited.", 0) \
+    M(UInt64, memory_overcommit_ratio_denominator_for_user, 1_GiB, "It represents soft memory limit on the global level. This value is used to compute query overcommit ratio.", 0) \
     M(UInt64, max_untracked_memory, (4 * 1024 * 1024), "Small allocations and deallocations are grouped in thread local variable and tracked or profiled only when amount (in absolute value) becomes larger than specified value. If the value is higher than 'memory_profiler_step' it will be effectively lowered to 'memory_profiler_step'.", 0) \
     M(UInt64, memory_profiler_step, (4 * 1024 * 1024), "Whenever query memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stack trace. Zero means disabled memory profiler. Values lower than a few megabytes will slow down query processing.", 0) \
     M(Float, memory_profiler_sample_probability, 0., "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation. Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \
     \
-    M(UInt64, memory_usage_overcommit_max_wait_microseconds, 0, "Maximum time thread will wait for memory to be freed in the case of memory overcommit. If timeout is reached and memory is not freed, exception is thrown", 0) \
+    M(UInt64, memory_usage_overcommit_max_wait_microseconds, 200, "Maximum time thread will wait for memory to be freed in the case of memory overcommit on user level. If timeout is reached and memory is not freed, exception is thrown.", 0) \
     \
     M(UInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \
     M(UInt64, max_network_bytes, 0, "The maximum number of bytes (compressed) to receive or transmit over the network for execution of the query.", 0) \
@@ -381,7 +381,7 @@ static constexpr UInt64 operator""_Gb(unsigned long long value)
     \
     M(Bool, log_profile_events, true, "Log query performance statistics into the query_log, query_thread_log and query_views_log.", 0) \
     M(Bool, log_query_settings, true, "Log query settings into the query_log.", 0) \
-    M(Bool, log_query_threads, true, "Log query threads into system.query_thread_log table. This setting have effect only when 'log_queries' is true.", 0) \
+    M(Bool, log_query_threads, false, "Log query threads into system.query_thread_log table. This setting have effect only when 'log_queries' is true.", 0) \
     M(Bool, log_query_views, true, "Log query dependent views into system.query_views_log table. This setting have effect only when 'log_queries' is true.", 0) \
     M(String, log_comment, "", "Log comment into system.query_log table and server log. It can be set to arbitrary string no longer than max_query_size.", 0) \
     M(LogsLevel, send_logs_level, LogsLevel::fatal, "Send server text logs with specified minimum level to client. Valid values: 'trace', 'debug', 'information', 'warning', 'error', 'fatal', 'none'", 0) \
@@ -449,6 +449,7 @@ static constexpr UInt64 operator""_Gb(unsigned long long value)
     M(Bool, optimize_normalize_count_variants, true, "Rewrite aggregate functions that semantically equals to count() as count().", 0) \
     M(Bool, optimize_injective_functions_inside_uniq, true, "Delete injective functions of one argument inside uniq*() functions.", 0) \
     M(Bool, convert_query_to_cnf, false, "Convert SELECT query to CNF", 0) \
+    M(Bool, optimize_or_like_chain, false, "Optimize multiple OR LIKE into multiMatchAny. This optimization should not be enabled by default, because it defies index analysis in some cases.", 0) \
     M(Bool, optimize_arithmetic_operations_in_aggregate_functions, true, "Move arithmetic operations out of aggregation functions", 0) \
     M(Bool, optimize_duplicate_order_by_and_distinct, true, "Remove duplicate ORDER BY and DISTINCT if it's possible", 0) \
     M(Bool, optimize_redundant_functions_in_order_by, true, "Remove functions from ORDER BY if its argument is also in ORDER BY", 0) \
@@ -588,6 +589,7 @@ static constexpr UInt64 operator""_Gb(unsigned long long value)
     M(Bool, count_distinct_optimization, false, "Rewrite count distinct to subquery of group by", 0) \
     M(Bool, throw_on_unsupported_query_inside_transaction, true, "Throw exception if unsupported query is used inside transaction", 0) \
     M(Bool, throw_if_no_data_to_insert, true, "Enables or disables empty INSERTs, enabled by default", 0) \
+    M(Bool, compatibility_ignore_auto_increment_in_create_table, false, "Ignore AUTO_INCREMENT keyword in column declaration if true, otherwise return error. It simplifies migration from MySQL", 0) \
     // End of COMMON_SETTINGS
     // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS.
 

src/Daemon/BaseDaemon.cpp

@@ -13,9 +13,9 @@
 #if defined(__linux__)
     #include <sys/prctl.h>
 #endif
-#include <errno.h>
+#include <cerrno>
-#include <string.h>
+#include <cstring>
-#include <signal.h>
+#include <csignal>
 #include <unistd.h>
 
 #include <typeinfo>

src/Daemon/SentryWriter.cpp

@@ -21,7 +21,7 @@
 #if USE_SENTRY && !defined(KEEPER_STANDALONE_BUILD)
 
 #    include <sentry.h>
-#    include <stdio.h>
+#    include <cstdio>
 #    include <filesystem>
 
 namespace fs = std::filesystem;

src/DataTypes/NestedUtils.cpp

@@ -1,4 +1,4 @@
-#include <string.h>
+#include <cstring>
 
 #include <Common/typeid_cast.h>
 #include <Common/assert_cast.h>

src/Disks/DiskLocal.cpp

@@ -627,27 +627,27 @@ bool DiskLocal::setup()
 
     /// Try to create a new checker file. The disk status can be either broken or readonly.
     if (disk_checker_magic_number == -1)
-    try
+        try
-    {
-        pcg32_fast rng(randomSeed());
-        UInt32 magic_number = rng();
         {
-            auto buf = writeFile(disk_checker_path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite, {});
+            pcg32_fast rng(randomSeed());
-            writeIntBinary(magic_number, *buf);
+            UInt32 magic_number = rng();
+            {
+                auto buf = writeFile(disk_checker_path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite, {});
+                writeIntBinary(magic_number, *buf);
+            }
+            disk_checker_magic_number = magic_number;
+        }
+        catch (...)
+        {
+            LOG_WARNING(
+                logger,
+                "Cannot create/write to {0}. Disk {1} is either readonly or broken. Without setting up disk checker file, DiskLocalCheckThread "
+                "will not be started. Disk is assumed to be RW. Try manually fix the disk and do `SYSTEM RESTART DISK {1}`",
+                disk_checker_path,
+                name);
+            disk_checker_can_check_read = false;
+            return true;
         }
-        disk_checker_magic_number = magic_number;
-    }
-    catch (...)
-    {
-        LOG_WARNING(
-            logger,
-            "Cannot create/write to {0}. Disk {1} is either readonly or broken. Without setting up disk checker file, DiskLocalCheckThread "
-            "will not be started. Disk is assumed to be RW. Try manually fix the disk and do `SYSTEM RESTART DISK {1}`",
-            disk_checker_path,
-            name);
-        disk_checker_can_check_read = false;
-        return true;
-    }
 
     if (disk_checker_magic_number == -1)
         throw Exception("disk_checker_magic_number is not initialized. It's a bug", ErrorCodes::LOGICAL_ERROR);

src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp

@@ -1,10 +1,10 @@
 #include "AsynchronousReadIndirectBufferFromRemoteFS.h"
 
 #include <Common/Stopwatch.h>
+#include <Common/logger_useful.h>
 #include <Disks/IO/ThreadPoolRemoteFSReader.h>
 #include <Disks/IO/ReadBufferFromRemoteFSGather.h>
 #include <IO/ReadSettings.h>
-#include <Common/logger_useful.h>
 
 
 namespace CurrentMetrics
@@ -57,7 +57,6 @@ AsynchronousReadIndirectBufferFromRemoteFS::AsynchronousReadIndirectBufferFromRe
     ProfileEvents::increment(ProfileEvents::RemoteFSBuffers);
 }
 
-
 String AsynchronousReadIndirectBufferFromRemoteFS::getFileName() const
 {
     return impl->getFileName();
@@ -169,6 +168,9 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl()
     if (!hasPendingDataToRead())
         return false;
 
+    Stopwatch watch;
+    CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait};
+
     size_t size = 0;
     if (prefetch_future.valid())
     {
@@ -176,15 +178,13 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl()
 
         size_t offset = 0;
         {
-            Stopwatch watch;
-            CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait};
             auto result = prefetch_future.get();
             size = result.size;
             offset = result.offset;
             LOG_TEST(log, "Current size: {}, offset: {}", size, offset);
 
             /// If prefetch_future is valid, size should always be greater than zero.
-            assert(offset < size);
+            assert(offset <= size);
             ProfileEvents::increment(ProfileEvents::AsynchronousReadWaitMicroseconds, watch.elapsedMicroseconds());
         }
 
@@ -201,7 +201,7 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl()
         auto offset = result.offset;
 
         LOG_TEST(log, "Current size: {}, offset: {}", size, offset);
-        assert(offset < size);
+        assert(offset <= size);
 
         if (size)
         {
@@ -210,6 +210,9 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl()
         }
     }
 
+    watch.stop();
+    ProfileEvents::increment(ProfileEvents::AsynchronousReadWaitMicroseconds, watch.elapsedMicroseconds());
+
     file_offset_of_buffer_end = impl->getFileOffsetOfBufferEnd();
     assert(file_offset_of_buffer_end == impl->getImplementationBufferOffset());
 

src/Disks/IO/CachedReadBufferFromRemoteFS.cpp

@@ -5,13 +5,23 @@
 #include <base/scope_guard.h>
 #include <Common/assert_cast.h>
 #include <Common/hex.h>
+#include <Common/getRandomASCIIString.h>
 
 
 namespace ProfileEvents
 {
-extern const Event RemoteFSReadBytes;
+extern const Event FileSegmentWaitReadBufferMicroseconds;
-extern const Event RemoteFSCacheReadBytes;
+extern const Event FileSegmentReadMicroseconds;
-extern const Event RemoteFSCacheDownloadBytes;
+extern const Event FileSegmentCacheWriteMicroseconds;
+extern const Event FileSegmentPredownloadMicroseconds;
+extern const Event FileSegmentUsedBytes;
+
+extern const Event CachedReadBufferReadFromSourceMicroseconds;
+extern const Event CachedReadBufferReadFromCacheMicroseconds;
+extern const Event CachedReadBufferCacheWriteMicroseconds;
+extern const Event CachedReadBufferReadFromSourceBytes;
+extern const Event CachedReadBufferReadFromCacheBytes;
+extern const Event CachedReadBufferCacheWriteBytes;
 }
 
 namespace DB
@@ -23,18 +33,12 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
-static String getQueryId()
-{
-    if (!CurrentThread::isInitialized() || !CurrentThread::get().getQueryContext() || CurrentThread::getQueryId().size == 0)
-        return "";
-    return CurrentThread::getQueryId().toString();
-}
-
 CachedReadBufferFromRemoteFS::CachedReadBufferFromRemoteFS(
     const String & remote_fs_object_path_,
     FileCachePtr cache_,
     RemoteFSFileReaderCreator remote_file_reader_creator_,
     const ReadSettings & settings_,
+    const String & query_id_,
     size_t read_until_position_)
     : SeekableReadBuffer(nullptr, 0)
 #ifndef NDEBUG
@@ -48,8 +52,9 @@ CachedReadBufferFromRemoteFS::CachedReadBufferFromRemoteFS(
     , settings(settings_)
     , read_until_position(read_until_position_)
     , remote_file_reader_creator(remote_file_reader_creator_)
-    , query_id(getQueryId())
+    , query_id(query_id_)
-    , enable_logging(!query_id.empty() && CurrentThread::get().getQueryContext()->getSettingsRef().enable_filesystem_cache_log)
+    , enable_logging(!query_id.empty() && settings_.enable_filesystem_cache_log)
+    , current_buffer_id(getRandomASCIIString(8))
 {
 }
 
@@ -62,9 +67,15 @@ void CachedReadBufferFromRemoteFS::appendFilesystemCacheLog(
         .query_id = query_id,
         .source_file_path = remote_fs_object_path,
         .file_segment_range = { file_segment_range.left, file_segment_range.right },
+        .requested_range = { first_offset, read_until_position },
         .file_segment_size = file_segment_range.size(),
+        .cache_attempted = true,
+        .read_buffer_id = current_buffer_id,
+        .profile_counters = std::make_shared<ProfileEvents::Counters::Snapshot>(current_file_segment_counters.getPartiallyAtomicSnapshot()),
     };
 
+    current_file_segment_counters.reset();
+
     switch (type)
     {
         case CachedReadBufferFromRemoteFS::ReadType::CACHED:
@@ -109,9 +120,16 @@ void CachedReadBufferFromRemoteFS::initialize(size_t offset, size_t size)
 SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getCacheReadBuffer(size_t offset) const
 {
     auto path = cache->getPathInLocalCache(cache_key, offset);
-    auto buf = std::make_shared<ReadBufferFromFile>(path, settings.local_fs_buffer_size);
+
-    if (buf->size() == 0)
+    ReadSettings local_read_settings{settings};
+    /// Do not allow to use asynchronous version of LocalFSReadMethod.
+    local_read_settings.local_fs_method = LocalFSReadMethod::pread;
+
+    auto buf = createReadBufferFromFileBase(path, local_read_settings);
+    auto * from_fd = dynamic_cast<ReadBufferFromFileDescriptor*>(buf.get());
+    if (from_fd && from_fd->size() == 0)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read from an empty cache file: {}", path);
+
     return buf;
 }
 
@@ -340,8 +358,13 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getImplementationBuffer(File
     auto range = file_segment->range();
     bytes_to_predownload = 0;
 
+    Stopwatch watch(CLOCK_MONOTONIC);
+
     auto read_buffer_for_file_segment = getReadBufferForFileSegment(file_segment);
 
+    watch.stop();
+    current_file_segment_counters.increment(ProfileEvents::FileSegmentWaitReadBufferMicroseconds, watch.elapsedMicroseconds());
+
     [[maybe_unused]] auto download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE;
     assert(download_current_segment == file_segment->isDownloader());
 
@@ -362,7 +385,7 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getImplementationBuffer(File
         case ReadType::CACHED:
         {
 #ifndef NDEBUG
-            auto * file_reader = assert_cast<ReadBufferFromFile *>(read_buffer_for_file_segment.get());
+            auto * file_reader = dynamic_cast<ReadBufferFromFileDescriptor *>(read_buffer_for_file_segment.get());
             size_t file_size = file_reader->size();
 
             if (file_size == 0 || range.left + file_size <= file_offset_of_buffer_end)
@@ -436,6 +459,9 @@ bool CachedReadBufferFromRemoteFS::completeFileSegmentAndGetNext()
 {
     LOG_TEST(log, "Completed segment: {}", (*current_file_segment_it)->range().toString());
 
+    if (enable_logging)
+        appendFilesystemCacheLog((*current_file_segment_it)->range(), read_type);
+
     auto file_segment_it = current_file_segment_it++;
     auto & file_segment = *file_segment_it;
 
@@ -460,15 +486,29 @@ bool CachedReadBufferFromRemoteFS::completeFileSegmentAndGetNext()
 
     if (read_type == ReadType::CACHED)
         (*current_file_segment_it)->incrementHitsCount();
-    if (enable_logging)
-        appendFilesystemCacheLog((*current_file_segment_it)->range(), read_type);
 
     LOG_TEST(log, "New segment: {}", (*current_file_segment_it)->range().toString());
     return true;
 }
 
+CachedReadBufferFromRemoteFS::~CachedReadBufferFromRemoteFS()
+{
+    if (enable_logging
+        && file_segments_holder
+        && current_file_segment_it != file_segments_holder->file_segments.end())
+    {
+        appendFilesystemCacheLog((*current_file_segment_it)->range(), read_type);
+    }
+}
+
 void CachedReadBufferFromRemoteFS::predownload(FileSegmentPtr & file_segment)
 {
+    Stopwatch predownload_watch(CLOCK_MONOTONIC);
+    SCOPE_EXIT({
+        predownload_watch.stop();
+        current_file_segment_counters.increment(ProfileEvents::FileSegmentPredownloadMicroseconds, predownload_watch.elapsedMicroseconds());
+    });
+
     if (bytes_to_predownload)
     {
         /// Consider this case. Some user needed segment [a, b] and downloaded it partially.
@@ -484,7 +524,19 @@ void CachedReadBufferFromRemoteFS::predownload(FileSegmentPtr & file_segment)
 
         while (true)
         {
-            if (!bytes_to_predownload || implementation_buffer->eof())
+            bool has_more_data;
+            {
+                Stopwatch watch(CLOCK_MONOTONIC);
+
+                has_more_data = !implementation_buffer->eof();
+
+                watch.stop();
+                auto elapsed = watch.elapsedMicroseconds();
+                current_file_segment_counters.increment(ProfileEvents::FileSegmentReadMicroseconds, elapsed);
+                ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromSourceMicroseconds, elapsed);
+            }
+
+            if (!bytes_to_predownload || !has_more_data)
             {
                 if (bytes_to_predownload)
                     throw Exception(
@@ -523,7 +575,7 @@ void CachedReadBufferFromRemoteFS::predownload(FileSegmentPtr & file_segment)
             size_t current_impl_buffer_size = implementation_buffer->buffer().size();
             size_t current_predownload_size = std::min(current_impl_buffer_size, bytes_to_predownload);
 
-            ProfileEvents::increment(ProfileEvents::RemoteFSReadBytes, current_impl_buffer_size);
+            ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromSourceBytes, current_impl_buffer_size);
 
             if (file_segment->reserve(current_predownload_size))
             {
@@ -531,8 +583,15 @@ void CachedReadBufferFromRemoteFS::predownload(FileSegmentPtr & file_segment)
 
                 assert(file_segment->getDownloadOffset() == static_cast<size_t>(implementation_buffer->getPosition()));
 
+                Stopwatch watch(CLOCK_MONOTONIC);
+
                 file_segment->write(implementation_buffer->buffer().begin(), current_predownload_size, current_offset);
-                ProfileEvents::increment(ProfileEvents::RemoteFSCacheDownloadBytes, current_predownload_size);
+
+                watch.stop();
+                auto elapsed = watch.elapsedMicroseconds();
+                current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, elapsed);
+                ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteMicroseconds, elapsed);
+                ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteBytes, current_predownload_size);
 
                 current_offset += current_predownload_size;
 
@@ -668,18 +727,18 @@ bool CachedReadBufferFromRemoteFS::nextImplStep()
         return false;
 
     SCOPE_EXIT({
-        /// Save state of current file segment before it is completed.
+        try
-        nextimpl_step_log_info = getInfoForLog();
-
-        if (current_file_segment_it == file_segments_holder->file_segments.end())
-            return;
-
-        auto & file_segment = *current_file_segment_it;
-
-        bool download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE;
-        if (download_current_segment)
         {
-            try
+            /// Save state of current file segment before it is completed.
+            nextimpl_step_log_info = getInfoForLog();
+
+            if (current_file_segment_it == file_segments_holder->file_segments.end())
+                return;
+
+            auto & file_segment = *current_file_segment_it;
+
+            bool download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE;
+            if (download_current_segment)
             {
                 bool need_complete_file_segment = file_segment->isDownloader();
                 if (need_complete_file_segment)
@@ -688,13 +747,13 @@ bool CachedReadBufferFromRemoteFS::nextImplStep()
                     file_segment->completeBatchAndResetDownloader();
                 }
             }
-            catch (...)
-            {
-                tryLogCurrentException(__PRETTY_FUNCTION__);
-            }
-        }
 
-        assert(!file_segment->isDownloader());
+            assert(!file_segment->isDownloader());
+        }
+        catch (...)
+        {
+            tryLogCurrentException(__PRETTY_FUNCTION__);
+        }
     });
 
     bytes_to_predownload = 0;
@@ -711,9 +770,6 @@ bool CachedReadBufferFromRemoteFS::nextImplStep()
 
         if (read_type == ReadType::CACHED)
             (*current_file_segment_it)->incrementHitsCount();
-
-        if (enable_logging)
-            appendFilesystemCacheLog((*current_file_segment_it)->range(), read_type);
     }
 
     assert(!internal_buffer.empty());
@@ -747,18 +803,17 @@ bool CachedReadBufferFromRemoteFS::nextImplStep()
 
     auto download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE;
     if (download_current_segment != file_segment->isDownloader())
+    {
         throw Exception(
             ErrorCodes::LOGICAL_ERROR,
-            "Incorrect segment state. Having read type: {}, Caller id: {}, downloader id: {}, file segment state: {}",
+            "Incorrect segment state. Having read type: {}, file segment info: {}",
-            toString(read_type),
+            toString(read_type), file_segment->getInfoForLog());
-            file_segment->getCallerId(),
+    }
-            file_segment->getDownloader(),
-            file_segment->state());
 
     if (!result)
     {
 #ifndef NDEBUG
-        if (auto * cache_file_reader = typeid_cast<ReadBufferFromFile *>(implementation_buffer.get()))
+        if (auto * cache_file_reader = dynamic_cast<ReadBufferFromFileDescriptor *>(implementation_buffer.get()))
         {
             auto cache_file_size = cache_file_reader->size();
             if (cache_file_size == 0)
@@ -767,13 +822,26 @@ bool CachedReadBufferFromRemoteFS::nextImplStep()
         }
 #endif
 
+        Stopwatch watch(CLOCK_MONOTONIC);
+
         result = implementation_buffer->next();
+
+        watch.stop();
+        auto elapsed = watch.elapsedMicroseconds();
+        current_file_segment_counters.increment(ProfileEvents::FileSegmentReadMicroseconds, elapsed);
+
         size = implementation_buffer->buffer().size();
 
         if (read_type == ReadType::CACHED)
-            ProfileEvents::increment(ProfileEvents::RemoteFSCacheReadBytes, size);
+        {
+            ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromCacheBytes, size);
+            ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromCacheMicroseconds, elapsed);
+        }
         else
-            ProfileEvents::increment(ProfileEvents::RemoteFSReadBytes, size);
+        {
+            ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromSourceBytes, size);
+            ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromSourceMicroseconds, elapsed);
+        }
     }
 
     if (result)
@@ -786,12 +854,18 @@ bool CachedReadBufferFromRemoteFS::nextImplStep()
             {
                 assert(file_segment->getDownloadOffset() == static_cast<size_t>(implementation_buffer->getPosition()));
 
+                Stopwatch watch(CLOCK_MONOTONIC);
+
                 file_segment->write(
                     needed_to_predownload ? implementation_buffer->position() : implementation_buffer->buffer().begin(),
                     size,
                     file_offset_of_buffer_end);
 
-                ProfileEvents::increment(ProfileEvents::RemoteFSCacheDownloadBytes, size);
+                watch.stop();
+                auto elapsed = watch.elapsedMicroseconds();
+                current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, elapsed);
+                ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteMicroseconds, elapsed);
+                ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteBytes, size);
 
                 assert(file_segment->getDownloadOffset() <= file_segment->range().right + 1);
                 assert(
@@ -819,10 +893,13 @@ bool CachedReadBufferFromRemoteFS::nextImplStep()
         }
 
         file_offset_of_buffer_end += size;
+
     }
 
     swap(*implementation_buffer);
 
+    current_file_segment_counters.increment(ProfileEvents::FileSegmentUsedBytes, available());
+
     if (download_current_segment)
         file_segment->completeBatchAndResetDownloader();
 
@@ -851,7 +928,7 @@ bool CachedReadBufferFromRemoteFS::nextImplStep()
     if (size == 0 && file_offset_of_buffer_end < read_until_position)
     {
         std::optional<size_t> cache_file_size;
-        if (auto * cache_file_reader = dynamic_cast<ReadBufferFromFile *>(implementation_buffer.get()))
+        if (auto * cache_file_reader = dynamic_cast<ReadBufferFromFileDescriptor *>(implementation_buffer.get()))
             cache_file_size = cache_file_reader->size();
 
         throw Exception(

src/Disks/IO/CachedReadBufferFromRemoteFS.h

@@ -26,8 +26,11 @@ public:
         FileCachePtr cache_,
         RemoteFSFileReaderCreator remote_file_reader_creator_,
         const ReadSettings & settings_,
+        const String & query_id_,
         size_t read_until_position_);
 
+    ~CachedReadBufferFromRemoteFS() override;
+
     bool nextImpl() override;
 
     off_t seek(off_t off, int whence) override;
@@ -116,8 +119,10 @@ private:
 
     String query_id;
     bool enable_logging = false;
+    String current_buffer_id;
 
     CurrentMetrics::Increment metric_increment{CurrentMetrics::FilesystemCacheReadBuffers};
+    ProfileEvents::Counters current_file_segment_counters;
 };
 
 }

src/Disks/IO/ReadBufferFromRemoteFSGather.cpp

@@ -21,9 +21,11 @@
 #include <filesystem>
 #include <iostream>
 #include <Common/hex.h>
+#include <Interpreters/FilesystemCacheLog.h>
 
 namespace fs = std::filesystem;
 
+
 namespace DB
 {
 
@@ -32,17 +34,24 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
-#if USE_AWS_S3
+SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(const String & path, size_t file_size)
-SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBuffer(const String & path, size_t file_size)
+{
+    if (!current_file_path.empty() && !with_cache && enable_cache_log)
+    {
+        appendFilesystemCacheLog();
+    }
+
+    current_file_path = fs::path(common_path_prefix) / path;
+    current_file_size = file_size;
+    total_bytes_read_from_current_file = 0;
+
+    return createImplementationBufferImpl(path, file_size);
+}
+
+#if USE_AWS_S3
+SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBufferImpl(const String & path, size_t file_size)
 {
-    current_path = path;
     auto remote_path = fs::path(common_path_prefix) / path;
-
-    auto cache = settings.remote_fs_cache;
-    bool with_cache = cache
-        && settings.enable_filesystem_cache
-        && (!IFileCache::isReadOnly() || settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache);
-
     auto remote_file_reader_creator = [=, this]()
     {
         return std::make_unique<ReadBufferFromS3>(
@@ -53,7 +62,7 @@ SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBuffer(const S
     if (with_cache)
     {
         return std::make_shared<CachedReadBufferFromRemoteFS>(
-            remote_path, cache, remote_file_reader_creator, settings, read_until_position ? read_until_position : file_size);
+            remote_path, settings.remote_fs_cache, remote_file_reader_creator, settings, query_id, read_until_position ? read_until_position : file_size);
     }
 
     return remote_file_reader_creator();
@@ -62,24 +71,24 @@ SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBuffer(const S
 
 
 #if USE_AZURE_BLOB_STORAGE
-SeekableReadBufferPtr ReadBufferFromAzureBlobStorageGather::createImplementationBuffer(const String & path, size_t /* file_size */)
+SeekableReadBufferPtr ReadBufferFromAzureBlobStorageGather::createImplementationBufferImpl(const String & path, size_t /* file_size */)
 {
-    current_path = path;
+    current_file_path = path;
     return std::make_unique<ReadBufferFromAzureBlobStorage>(blob_container_client, path, max_single_read_retries,
         max_single_download_retries, settings.remote_fs_buffer_size, /* use_external_buffer */true, read_until_position);
 }
 #endif
 
 
-SeekableReadBufferPtr ReadBufferFromWebServerGather::createImplementationBuffer(const String & path, size_t /* file_size */)
+SeekableReadBufferPtr ReadBufferFromWebServerGather::createImplementationBufferImpl(const String & path, size_t /* file_size */)
 {
-    current_path = path;
+    current_file_path = path;
     return std::make_unique<ReadBufferFromWebServer>(fs::path(uri) / path, context, settings, /* use_external_buffer */true, read_until_position);
 }
 
 
 #if USE_HDFS
-SeekableReadBufferPtr ReadBufferFromHDFSGather::createImplementationBuffer(const String & path, size_t /* file_size */)
+SeekableReadBufferPtr ReadBufferFromHDFSGather::createImplementationBufferImpl(const String & path, size_t /* file_size */)
 {
     return std::make_unique<ReadBufferFromHDFS>(hdfs_uri, fs::path(hdfs_directory) / path, config, settings.remote_fs_buffer_size);
 }
@@ -94,8 +103,31 @@ ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather(
     , common_path_prefix(common_path_prefix_)
     , blobs_to_read(blobs_to_read_)
     , settings(settings_)
+    , query_id(CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() != nullptr ? CurrentThread::getQueryId() : "")
     , log(&Poco::Logger::get("ReadBufferFromRemoteFSGather"))
+    , enable_cache_log(!query_id.empty() && settings.enable_filesystem_cache_log)
 {
+    with_cache = settings.remote_fs_cache
+        && settings.enable_filesystem_cache
+        && (!IFileCache::isReadOnly() || settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache);
+}
+
+
+void ReadBufferFromRemoteFSGather::appendFilesystemCacheLog()
+{
+    FilesystemCacheLogElement elem
+    {
+        .event_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()),
+        .query_id = query_id,
+        .source_file_path = current_file_path,
+        .file_segment_range = { 0, current_file_size },
+        .read_type = FilesystemCacheLogElement::ReadType::READ_FROM_FS_BYPASSING_CACHE,
+        .file_segment_size = total_bytes_read_from_current_file,
+        .cache_attempted = false,
+    };
+
+    if (auto cache_log = Context::getGlobalContextInstance()->getFilesystemCacheLog())
+        cache_log->add(elem);
 }
 
 
@@ -199,6 +231,7 @@ bool ReadBufferFromRemoteFSGather::readImpl()
      */
     if (bytes_to_ignore)
     {
+        total_bytes_read_from_current_file += bytes_to_ignore;
         current_buf->ignore(bytes_to_ignore);
         result = current_buf->hasPendingData();
         file_offset_of_buffer_end += bytes_to_ignore;
@@ -225,6 +258,7 @@ bool ReadBufferFromRemoteFSGather::readImpl()
     {
         assert(available());
         nextimpl_working_buffer_offset = offset();
+        total_bytes_read_from_current_file += available();
     }
 
     return result;
@@ -254,7 +288,7 @@ void ReadBufferFromRemoteFSGather::reset()
 
 String ReadBufferFromRemoteFSGather::getFileName() const
 {
-    return current_path;
+    return current_file_path;
 }
 
 
@@ -282,5 +316,12 @@ size_t ReadBufferFromRemoteFSGather::getImplementationBufferOffset() const
     return current_buf->getFileOffsetOfBufferEnd();
 }
 
+ReadBufferFromRemoteFSGather::~ReadBufferFromRemoteFSGather()
+{
+    if (!with_cache && enable_cache_log)
+    {
+        appendFilesystemCacheLog();
+    }
+}
 
 }

src/Disks/IO/ReadBufferFromRemoteFSGather.h

@@ -29,6 +29,9 @@ public:
         const std::string & common_path_prefix_,
         const BlobsPathToSize & blobs_to_read_,
         const ReadSettings & settings_);
+
+    ~ReadBufferFromRemoteFSGather() override;
+
     String getFileName() const;
 
     void reset();
@@ -54,7 +57,7 @@ public:
     size_t getImplementationBufferOffset() const;
 
 protected:
-    virtual SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) = 0;
+    virtual SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) = 0;
 
     std::string common_path_prefix;
 
@@ -64,9 +67,16 @@ protected:
 
     size_t read_until_position = 0;
 
-    String current_path;
+    String current_file_path;
+    size_t current_file_size = 0;
+
+    bool with_cache;
+
+    String query_id;
 
 private:
+    SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size);
+
     bool nextImpl() override;
 
     void initialize();
@@ -75,6 +85,8 @@ private:
 
     bool moveToNextBuffer();
 
+    void appendFilesystemCacheLog();
+
     SeekableReadBufferPtr current_buf;
 
     size_t current_buf_idx = 0;
@@ -89,6 +101,10 @@ private:
     size_t bytes_to_ignore = 0;
 
     Poco::Logger * log;
+
+    size_t total_bytes_read_from_current_file = 0;
+
+    bool enable_cache_log = false;
 };
 
 
@@ -113,7 +129,7 @@ public:
     {
     }
 
-    SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) override;
+    SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) override;
 
 private:
     std::shared_ptr<Aws::S3::S3Client> client_ptr;
@@ -143,7 +159,7 @@ public:
     {
     }
 
-    SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) override;
+    SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) override;
 
 private:
     std::shared_ptr<Azure::Storage::Blobs::BlobContainerClient> blob_container_client;
@@ -168,7 +184,7 @@ public:
     {
     }
 
-    SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) override;
+    SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) override;
 
 private:
     String uri;
@@ -195,7 +211,7 @@ public:
         hdfs_uri = hdfs_uri_.substr(0, begin_of_path);
     }
 
-    SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) override;
+    SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) override;
 
 private:
     const Poco::Util::AbstractConfiguration & config;

src/Disks/IO/ThreadPoolRemoteFSReader.cpp

@@ -16,8 +16,8 @@
 
 namespace ProfileEvents
 {
-    extern const Event RemoteFSReadMicroseconds;
+    extern const Event ThreadpoolReaderTaskMicroseconds;
-    extern const Event RemoteFSReadBytes;
+    extern const Event ThreadpoolReaderReadBytes;
 }
 
 namespace CurrentMetrics
@@ -83,8 +83,8 @@ std::future<IAsynchronousReader::Result> ThreadPoolRemoteFSReader::submit(Reques
 
         watch.stop();
 
-        ProfileEvents::increment(ProfileEvents::RemoteFSReadMicroseconds, watch.elapsedMicroseconds());
+        ProfileEvents::increment(ProfileEvents::ThreadpoolReaderTaskMicroseconds, watch.elapsedMicroseconds());
-        ProfileEvents::increment(ProfileEvents::RemoteFSReadBytes, result.offset ? result.size - result.offset : result.size);
+        ProfileEvents::increment(ProfileEvents::ThreadpoolReaderReadBytes, result.offset ? result.size - result.offset : result.size);
 
         thread_status.detachQuery(/* if_not_detached */true);
 

src/Disks/IVolume.h

@@ -63,7 +63,7 @@ public:
     Disks & getDisks() { return disks; }
     const Disks & getDisks() const { return disks; }
 
-    /// Returns effective value of whether merges are allowed on this volume (true) or not (false).
+    /// Returns effective value of whether merges are allowed on this volume (false) or not (true).
     virtual bool areMergesAvoided() const { return false; }
 
     /// User setting for enabling and disabling merges on volume.

src/Disks/S3/ProxyResolverConfiguration.cpp

@@ -63,7 +63,7 @@ Aws::Client::ClientConfigurationPerRequest ProxyResolverConfiguration::getConfig
         {
             auto resolved_endpoint = endpoint;
             resolved_endpoint.setHost(resolved_hosts[i].toString());
-            session = makeHTTPSession(endpoint, timeouts, false);
+            session = makeHTTPSession(resolved_endpoint, timeouts, false);
 
             try
             {