Merge branch 'master' into fix-mutations-again

.github/workflows/master.yml

@@ -215,8 +215,8 @@ jobs:
           fetch-depth: 0 # For a proper version and performance artifacts
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -259,8 +259,8 @@ jobs:
           fetch-depth: 0 # For a proper version and performance artifacts
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -305,8 +305,8 @@ jobs:
           fetch-depth: 0 # otherwise we will have no info about contributors
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -350,8 +350,8 @@ jobs:
   #       uses: actions/checkout@v2
   #     - name: Build
   #       run: |
-  #         git -C "$GITHUB_WORKSPACE" submodule sync
+  #         git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-  #         git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+  #         git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
   #         sudo rm -fr "$TEMP_PATH"
   #         mkdir -p "$TEMP_PATH"
   #         cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -395,8 +395,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -440,8 +440,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -485,8 +485,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -530,8 +530,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -575,8 +575,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -623,8 +623,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -668,8 +668,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -715,8 +715,8 @@ jobs:
           fetch-depth: 0 # otherwise we will have no info about contributors
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -762,8 +762,8 @@ jobs:
           fetch-depth: 0 # otherwise we will have no info about contributors
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -809,8 +809,8 @@ jobs:
           fetch-depth: 0 # otherwise we will have no info about contributors
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -856,8 +856,8 @@ jobs:
           fetch-depth: 0 # otherwise we will have no info about contributors
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -903,8 +903,8 @@ jobs:
           fetch-depth: 0 # otherwise we will have no info about contributors
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

.github/workflows/nightly.yml

@@ -81,7 +81,6 @@ jobs:
           cat >> "$GITHUB_ENV" << 'EOF'
           BUILD_NAME=coverity
           CACHES_PATH=${{runner.temp}}/../ccaches
-          CHECK_NAME=ClickHouse build check (actions)
           IMAGES_PATH=${{runner.temp}}/images_path
           REPO_COPY=${{runner.temp}}/build_check/ClickHouse
           TEMP_PATH=${{runner.temp}}/build_check
@@ -99,13 +98,15 @@ jobs:
         id: coverity-checkout
         uses: actions/checkout@v2
         with:
-          submodules: 'true'
+          fetch-depth: 0 # otherwise we will have no info about contributors
       - name: Build
         run: |
+          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
       - name: Upload Coverity Analysis
         if: ${{ success() || failure() }}
         run: |

.github/workflows/pull_request.yml

@@ -277,8 +277,8 @@ jobs:
           fetch-depth: 0  # for performance artifact
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -322,8 +322,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -367,8 +367,8 @@ jobs:
   #       uses: actions/checkout@v2
   #     - name: Build
   #       run: |
-  #         git -C "$GITHUB_WORKSPACE" submodule sync
+  #         git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-  #         git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+  #         git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
   #         sudo rm -fr "$TEMP_PATH"
   #         mkdir -p "$TEMP_PATH"
   #         cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -414,8 +414,8 @@ jobs:
           fetch-depth: 0  # for performance artifact
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -459,8 +459,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -504,8 +504,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -549,8 +549,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -594,8 +594,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -639,8 +639,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -687,8 +687,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -732,8 +732,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -777,8 +777,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -822,8 +822,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -867,8 +867,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -912,8 +912,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@@ -957,8 +957,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Build
         run: |
-          git -C "$GITHUB_WORKSPACE" submodule sync
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
-          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

.gitmodules

@@ -79,10 +79,10 @@
 	url = https://github.com/ClickHouse/snappy.git
 [submodule "contrib/cppkafka"]
 	path = contrib/cppkafka
-	url = https://github.com/ClickHouse/cppkafka.git
+	url = https://github.com/mfontanini/cppkafka.git
 [submodule "contrib/brotli"]
 	path = contrib/brotli
-	url = https://github.com/ClickHouse/brotli.git
+	url = https://github.com/google/brotli.git
 [submodule "contrib/h3"]
 	path = contrib/h3
 	url = https://github.com/ClickHouse/h3
@@ -144,7 +144,7 @@
 	ignore = untracked
 [submodule "contrib/msgpack-c"]
 	path = contrib/msgpack-c
-	url = https://github.com/ClickHouse/msgpack-c
+	url = https://github.com/msgpack/msgpack-c
 [submodule "contrib/libcpuid"]
 	path = contrib/libcpuid
 	url = https://github.com/ClickHouse/libcpuid.git

CMakeLists.txt

@@ -36,7 +36,7 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON)
 
 # Check that submodules are present
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/sysroot/README.md")
-    message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive")
+    message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init")
 endif ()
 
 # Take care to add prlimit in command line before ccache, or else ccache thinks that

base/base/CMakeLists.txt

@@ -17,15 +17,12 @@ set (SRCS
     sleep.cpp
     terminalColors.cpp
     errnoToString.cpp
+    ReplxxLineReader.cpp
     StringRef.cpp
     safeExit.cpp
     throwError.cpp
 )
 
-if (ENABLE_REPLXX)
-    list (APPEND SRCS ReplxxLineReader.cpp)
-endif ()
-
 if (USE_DEBUG_HELPERS)
     get_target_property(MAGIC_ENUM_INCLUDE_DIR ch_contrib::magic_enum INTERFACE_INCLUDE_DIRECTORIES)
     # CMake generator expression will do insane quoting when it encounters special character like quotes, spaces, etc.

contrib/abseil-cpp-cmake/CMakeLists.txt

@@ -1,7 +1,4 @@
 set(ABSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp")
-if(NOT EXISTS "${ABSL_ROOT_DIR}/CMakeLists.txt")
-  message(FATAL_ERROR " submodule third_party/abseil-cpp is missing. To fix try run: \n git submodule update --init --recursive")
-endif()
 set(BUILD_TESTING OFF)
 set(ABSL_PROPAGATE_CXX_STD ON)
 add_subdirectory("${ABSL_ROOT_DIR}" "${ClickHouse_BINARY_DIR}/contrib/abseil-cpp")

contrib/amqpcpp-cmake/CMakeLists.txt

@@ -5,6 +5,7 @@ if (NOT ENABLE_AMQPCPP)
     return()
 endif()
 
+# can be removed once libuv build on MacOS with GCC is possible
 if (NOT TARGET ch_contrib::uv)
     message(STATUS "Not using AMQP-CPP because libuv is disabled")
     return()
@@ -37,21 +38,6 @@ set (SRCS
 
 add_library(_amqp-cpp ${SRCS})
 
-target_compile_options (_amqp-cpp
-    PRIVATE
-        -Wno-old-style-cast
-        -Wno-inconsistent-missing-destructor-override
-        -Wno-deprecated
-        -Wno-unused-parameter
-        -Wno-shadow
-        -Wno-tautological-type-limit-compare
-        -Wno-extra-semi
-# NOTE: disable all warnings at last because the warning:
-        #       "conversion function converting 'XXX' to itself will never be used"
-        #       doesn't have it's own diagnostic flag yet.
-        -w
-)
-
 target_include_directories (_amqp-cpp SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include" "${LIBRARY_DIR}")
 target_link_libraries (_amqp-cpp PUBLIC OpenSSL::Crypto OpenSSL::SSL ch_contrib::uv)
 add_library (ch_contrib::amqp_cpp ALIAS _amqp-cpp)

contrib/arrow

@@ -1 +1 @@
-Subproject commit 6f274b737c66a6c39bab0d3bdf6cf7d139ef06f5
+Subproject commit efdcd015cfdee1b6aa349c9ca227ca12c3d697f5

contrib/arrow-cmake/CMakeLists.txt

@@ -20,7 +20,7 @@ endif()
 option (ENABLE_PARQUET "Enable parquet" ${ENABLE_PARQUET_DEFAULT})
 
 if (NOT ENABLE_PARQUET)
-    message(STATUS "Building without Parquet support")
+    message(STATUS "Not using parquet")
     return()
 endif()
 

contrib/avro-cmake/CMakeLists.txt

@@ -60,14 +60,6 @@ target_compile_definitions (_avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE)
 target_include_directories (_avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR})
 target_link_libraries (_avrocpp PRIVATE ch_contrib::snappy)
 
-if (COMPILER_GCC)
-    set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor)
-elseif (COMPILER_CLANG)
-    set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor)
-endif ()
-
-target_compile_options(_avrocpp PRIVATE ${SUPPRESS_WARNINGS})
-
 # create a symlink to include headers with <avro/...>
 set(AVRO_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/include")
 ADD_CUSTOM_TARGET(avro_symlink_headers ALL

contrib/azure-cmake/CMakeLists.txt

@@ -52,20 +52,6 @@ include("${AZURE_DIR}/cmake-modules/AzureTransportAdapters.cmake")
 
 add_library(_azure_sdk ${AZURE_SDK_UNIFIED_SRC})
 
-if (COMPILER_CLANG)
-    target_compile_options(_azure_sdk PRIVATE
-        -Wno-deprecated-copy-dtor
-        -Wno-extra-semi
-        -Wno-suggest-destructor-override
-        -Wno-inconsistent-missing-destructor-override
-        -Wno-error=unknown-warning-option
-    )
-
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 13)
-        target_compile_options(_azure_sdk PRIVATE -Wno-reserved-identifier)
-    endif()
-endif()
-
 # Originally, on Windows azure-core is built with bcrypt and crypt32 by default
 if (TARGET OpenSSL::SSL)
     target_link_libraries(_azure_sdk PRIVATE OpenSSL::Crypto OpenSSL::SSL)

contrib/boringssl-cmake/CMakeLists.txt

@@ -1,7 +1,12 @@
 # Needed for:
 # - securely connecting to an external server, e.g. clickhouse-client --host ... --secure
 # - lots of thirdparty libraries
-option(ENABLE_SSL "Enable ssl" ${ENABLE_LIBRARIES})
+
+# Actually, so many 3rd party libraries + unit tests need SSL that we cannot disable it
+# without breaking the build ...
+option(ENABLE_SSL "Enable ssl" ON) # breaks if OFF
+# TODO: Making SSL dependent on ENABLE_LIBRARIES is desirable but needs fixing dependent libs + tests.
+# option(ENABLE_SSL "Enable ssl" ${ENABLE_LIBRARIES})
 
 if(NOT ENABLE_SSL)
   message(STATUS "Not using openssl")

contrib/brotli

@@ -1 +1 @@
-Subproject commit 5bd78768449751a78d4b4c646b0612917986f5b1
+Subproject commit 63be8a99401992075c23e99f7c84de1c653e39e2

contrib/brotli-cmake/CMakeLists.txt

@@ -45,7 +45,4 @@ add_library(ch_contrib::brotli ALIAS _brotli)
 
 target_include_directories(_brotli SYSTEM BEFORE PUBLIC "${BROTLI_SOURCE_DIR}/include")
 
-if(M_LIBRARY)
-    target_link_libraries(_brotli PRIVATE ${M_LIBRARY})
-endif()
 target_compile_definitions(_brotli PRIVATE BROTLI_BUILD_PORTABLE=1)

contrib/bzip2-cmake/CMakeLists.txt

@@ -1,6 +1,6 @@
 option(ENABLE_BZIP2 "Enable bzip2 compression support" ${ENABLE_LIBRARIES})
 if (NOT ENABLE_BZIP2)
-    message (STATUS "bzip2 compression disabled")
+    message (STATUS "Not using bzip2")
     return()
 endif()
 
@@ -26,8 +26,4 @@ configure_file (
 
 add_library(_bzip2 ${SRCS})
 add_library(ch_contrib::bzip2 ALIAS _bzip2)
-# To avoid -Wreserved-id-macro we use SYSTEM:
-#
-#     clickhouse/contrib/bzip2/bzlib.h:23:9: error: macro name is a reserved identifier [-Werror,-Wreserved-id-macro]
-#     #define _BZLIB_H
 target_include_directories(_bzip2 SYSTEM BEFORE PUBLIC "${BZIP2_SOURCE_DIR}" "${BZIP2_BINARY_DIR}")

contrib/capnproto-cmake/CMakeLists.txt

@@ -81,16 +81,12 @@ set (CAPNPC_SRCS
 add_library(_capnpc ${CAPNPC_SRCS})
 target_link_libraries(_capnpc PUBLIC _capnp)
 
-# The library has substandard code
+if (COMPILER_CLANG)
-if (COMPILER_GCC)
-    set (SUPPRESS_WARNINGS -w)
-elseif (COMPILER_CLANG)
-    set (SUPPRESS_WARNINGS -w)
     set (CAPNP_PRIVATE_CXX_FLAGS -fno-char8_t)
 endif ()
 
-target_compile_options(_kj PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS})
+target_compile_options(_kj PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS})
-target_compile_options(_capnp PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS})
+target_compile_options(_capnp PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS})
-target_compile_options(_capnpc PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS})
+target_compile_options(_capnpc PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS})
 
 add_library(ch_contrib::capnp ALIAS _capnpc)

contrib/cassandra-cmake/CMakeLists.txt

@@ -5,6 +5,7 @@ if (NOT ENABLE_CASSANDRA)
     return()
 endif()
 
+# can be removed once libuv build on MacOS with GCC is possible
 if (NOT TARGET ch_contrib::uv)
     message(STATUS "Not using cassandra because libuv is disabled")
     return()

contrib/cppkafka

@@ -1 +1 @@
-Subproject commit 64bd67db12b9c705e9127439a5b05b351d9df7da
+Subproject commit 5a119f689f8a4d90d10a9635e7ee2bee5c127de1

contrib/cppkafka-cmake/CMakeLists.txt

@@ -1,5 +1,5 @@
 if (NOT ENABLE_KAFKA)
-    message(STATUS "Not using librdkafka (skip cppkafka)")
+    message(STATUS "Not using kafka")
     return()
 endif()
 

contrib/fastops-cmake/CMakeLists.txt

@@ -5,7 +5,7 @@ elseif(ENABLE_FASTOPS)
 endif()
 
 if(NOT ENABLE_FASTOPS)
-    message(STATUS "Not using fast vectorized mathematical functions library by Mikhail Parakhin")
+    message(STATUS "Not using fastops")
     return()
 endif()
 

contrib/fmtlib-cmake/CMakeLists.txt

@@ -1,22 +1,24 @@
+set(FMT_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/fmtlib")
+
 set (SRCS
     # NOTE: do not build module for now:
     # ../fmtlib/src/fmt.cc
-    ../fmtlib/src/format.cc
+    ${FMT_SOURCE_DIR}/src/format.cc
-    ../fmtlib/src/os.cc
+    ${FMT_SOURCE_DIR}/src/os.cc
 
-    ../fmtlib/include/fmt/args.h
+    ${FMT_SOURCE_DIR}/include/fmt/args.h
-    ../fmtlib/include/fmt/chrono.h
+    ${FMT_SOURCE_DIR}/include/fmt/chrono.h
-    ../fmtlib/include/fmt/color.h
+    ${FMT_SOURCE_DIR}/include/fmt/color.h
-    ../fmtlib/include/fmt/compile.h
+    ${FMT_SOURCE_DIR}/include/fmt/compile.h
-    ../fmtlib/include/fmt/core.h
+    ${FMT_SOURCE_DIR}/include/fmt/core.h
-    ../fmtlib/include/fmt/format.h
+    ${FMT_SOURCE_DIR}/include/fmt/format.h
-    ../fmtlib/include/fmt/format-inl.h
+    ${FMT_SOURCE_DIR}/include/fmt/format-inl.h
-    ../fmtlib/include/fmt/locale.h
+    ${FMT_SOURCE_DIR}/include/fmt/locale.h
-    ../fmtlib/include/fmt/os.h
+    ${FMT_SOURCE_DIR}/include/fmt/os.h
-    ../fmtlib/include/fmt/ostream.h
+    ${FMT_SOURCE_DIR}/include/fmt/ostream.h
-    ../fmtlib/include/fmt/printf.h
+    ${FMT_SOURCE_DIR}/include/fmt/printf.h
-    ../fmtlib/include/fmt/ranges.h
+    ${FMT_SOURCE_DIR}/include/fmt/ranges.h
-    ../fmtlib/include/fmt/xchar.h
+    ${FMT_SOURCE_DIR}/include/fmt/xchar.h
 )
 
 add_library(_fmt ${SRCS})

contrib/h3-cmake/CMakeLists.txt

@@ -9,23 +9,23 @@ set(H3_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib")
 set(H3_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/h3/src/h3lib")
 
 set(SRCS
-"${H3_SOURCE_DIR}/lib/algos.c"
+    "${H3_SOURCE_DIR}/lib/algos.c"
-"${H3_SOURCE_DIR}/lib/coordijk.c"
+    "${H3_SOURCE_DIR}/lib/coordijk.c"
-"${H3_SOURCE_DIR}/lib/bbox.c"
+    "${H3_SOURCE_DIR}/lib/bbox.c"
-"${H3_SOURCE_DIR}/lib/polygon.c"
+    "${H3_SOURCE_DIR}/lib/polygon.c"
-"${H3_SOURCE_DIR}/lib/h3Index.c"
+    "${H3_SOURCE_DIR}/lib/h3Index.c"
-"${H3_SOURCE_DIR}/lib/vec2d.c"
+    "${H3_SOURCE_DIR}/lib/vec2d.c"
-"${H3_SOURCE_DIR}/lib/vec3d.c"
+    "${H3_SOURCE_DIR}/lib/vec3d.c"
-"${H3_SOURCE_DIR}/lib/vertex.c"
+    "${H3_SOURCE_DIR}/lib/vertex.c"
-"${H3_SOURCE_DIR}/lib/linkedGeo.c"
+    "${H3_SOURCE_DIR}/lib/linkedGeo.c"
-"${H3_SOURCE_DIR}/lib/localij.c"
+    "${H3_SOURCE_DIR}/lib/localij.c"
-"${H3_SOURCE_DIR}/lib/latLng.c"
+    "${H3_SOURCE_DIR}/lib/latLng.c"
-"${H3_SOURCE_DIR}/lib/directedEdge.c"
+    "${H3_SOURCE_DIR}/lib/directedEdge.c"
-"${H3_SOURCE_DIR}/lib/mathExtensions.c"
+    "${H3_SOURCE_DIR}/lib/mathExtensions.c"
-"${H3_SOURCE_DIR}/lib/iterators.c"
+    "${H3_SOURCE_DIR}/lib/iterators.c"
-"${H3_SOURCE_DIR}/lib/vertexGraph.c"
+    "${H3_SOURCE_DIR}/lib/vertexGraph.c"
-"${H3_SOURCE_DIR}/lib/faceijk.c"
+    "${H3_SOURCE_DIR}/lib/faceijk.c"
-"${H3_SOURCE_DIR}/lib/baseCells.c"
+    "${H3_SOURCE_DIR}/lib/baseCells.c"
 )
 
 configure_file("${H3_SOURCE_DIR}/include/h3api.h.in" "${H3_BINARY_DIR}/include/h3api.h")
@@ -34,8 +34,5 @@ add_library(_h3 ${SRCS})
 target_include_directories(_h3 SYSTEM PUBLIC "${H3_SOURCE_DIR}/include")
 target_include_directories(_h3 SYSTEM PUBLIC "${H3_BINARY_DIR}/include")
 target_compile_definitions(_h3 PRIVATE H3_HAVE_VLA)
-if(M_LIBRARY)
-    target_link_libraries(_h3 PRIVATE ${M_LIBRARY})
-endif()
 
 add_library(ch_contrib::h3 ALIAS _h3)

contrib/hive-metastore-cmake/CMakeLists.txt

@@ -5,7 +5,7 @@ elseif(ENABLE_HIVE)
 endif()
 
 if (NOT ENABLE_HIVE)
-    message("Hive disabled")
+    message(STATUS "Not using hive")
     return()
 endif()
 

contrib/icu-cmake/CMakeLists.txt

@@ -481,10 +481,6 @@ target_include_directories(_icui18n SYSTEM PUBLIC "${ICU_SOURCE_DIR}/i18n/")
 target_compile_definitions(_icuuc PRIVATE -DU_COMMON_IMPLEMENTATION)
 target_compile_definitions(_icui18n PRIVATE -DU_I18N_IMPLEMENTATION)
 
-if (COMPILER_CLANG)
-    target_compile_options(_icudata PRIVATE -Wno-unused-command-line-argument)
-endif ()
-
 add_library(_icu INTERFACE)
 target_link_libraries(_icu INTERFACE _icui18n _icuuc _icudata)
 add_library(ch_contrib::icu ALIAS _icu)

contrib/jemalloc-cmake/CMakeLists.txt

@@ -180,7 +180,6 @@ if (USE_UNWIND)
     target_link_libraries (_jemalloc PRIVATE unwind)
 endif ()
 
-target_compile_options(_jemalloc PRIVATE -Wno-redundant-decls)
 # for RTLD_NEXT
 target_compile_options(_jemalloc PRIVATE -D_GNU_SOURCE)
 

contrib/libcpuid-cmake/CMakeLists.txt

@@ -6,7 +6,7 @@ elseif(ENABLE_CPUID)
 endif()
 
 if (NOT ENABLE_CPUID)
-    message("Not using cpuid")
+    message(STATUS "Not using cpuid")
     return()
 endif()
 
@@ -27,8 +27,5 @@ add_library (_cpuid ${SRCS})
 
 target_include_directories (_cpuid SYSTEM PUBLIC "${LIBRARY_DIR}")
 target_compile_definitions (_cpuid PRIVATE VERSION="v0.4.1")
-if (COMPILER_CLANG)
-    target_compile_options (_cpuid PRIVATE -Wno-reserved-id-macro)
-endif ()
 
 add_library(ch_contrib::cpuid ALIAS _cpuid)

contrib/libgsasl-cmake/CMakeLists.txt

@@ -1,7 +1,7 @@
 option(ENABLE_GSASL_LIBRARY "Enable gsasl library" ${ENABLE_LIBRARIES})
 
 if (NOT ENABLE_GSASL_LIBRARY)
-    message(STATUS "Not using gsasl library")
+    message(STATUS "Not using gsasl")
     return()
 endif()
 

contrib/libuv-cmake/CMakeLists.txt

@@ -1,3 +1,4 @@
+# once fixed, please remove similar places in CMakeLists of libuv users (search "ch_contrib::uv")
 if (OS_DARWIN AND COMPILER_GCC)
     message (WARNING "libuv cannot be built with GCC in macOS due to a bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93082")
     return()

contrib/libxml2-cmake/CMakeLists.txt

@@ -53,9 +53,6 @@ set(SRCS
 add_library(_libxml2 ${SRCS})
 
 target_link_libraries(_libxml2 PRIVATE ch_contrib::zlib)
-if(M_LIBRARY)
-    target_link_libraries(_libxml2 PRIVATE ${M_LIBRARY})
-endif()
 
 target_include_directories(_libxml2 BEFORE PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include")
 target_include_directories(_libxml2 BEFORE PUBLIC "${LIBXML2_SOURCE_DIR}/include")

contrib/minizip-ng-cmake/CMakeLists.txt

@@ -1,6 +1,6 @@
 option(ENABLE_MINIZIP "Enable minizip-ng the zip manipulation library" ${ENABLE_LIBRARIES})
 if (NOT ENABLE_MINIZIP)
-    message (STATUS "minizip-ng disabled")
+    message (STATUS "Not using minizip-ng")
     return()
 endif()
 

contrib/msgpack-c

@@ -1 +1 @@
-Subproject commit 790b3fe58ebded7a8bd130782ef28bec5784c248
+Subproject commit 46684265d50b5d1b062d4c5c428ba08462844b1d

contrib/nanodbc-cmake/CMakeLists.txt

@@ -2,12 +2,12 @@ if (NOT ENABLE_ODBC)
     return ()
 endif ()
 
-set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/nanodbc")
-
 if (NOT TARGET ch_contrib::unixodbc)
     message(FATAL_ERROR "Configuration error: unixodbc is not a target")
 endif()
 
+set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/nanodbc")
+
 set (SRCS
     "${LIBRARY_DIR}/nanodbc/nanodbc.cpp"
 )

contrib/rapidjson

@@ -1 +1 @@
-Subproject commit b571bd5c1a3b1fc931d77ae36932537a3c9018c3
+Subproject commit c4ef90ccdbc21d5d5a628d08316bfd301e32d6fa

contrib/replxx-cmake/CMakeLists.txt

@@ -1,10 +1,3 @@
-option (ENABLE_REPLXX "Enable replxx support" ${ENABLE_LIBRARIES})
-
-if (NOT ENABLE_REPLXX)
-    message (STATUS "Not using replxx")
-    return()
-endif()
-
 set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx")
 
 set(SRCS
@@ -22,9 +15,4 @@ set(SRCS
 
 add_library (_replxx ${SRCS})
 target_include_directories(_replxx SYSTEM PUBLIC "${LIBRARY_DIR}/include")
-
-if (COMPILER_CLANG)
-    target_compile_options(_replxx PRIVATE -Wno-documentation)
-endif ()
-
 add_library(ch_contrib::replxx ALIAS _replxx)

contrib/s2geometry-cmake/CMakeLists.txt

@@ -149,7 +149,3 @@ target_link_libraries(_s2 PRIVATE
 
 target_include_directories(_s2 SYSTEM BEFORE PUBLIC "${S2_SOURCE_DIR}/")
 target_include_directories(_s2 SYSTEM PUBLIC "${ABSL_SOURCE_DIR}")
-
-if(M_LIBRARY)
-    target_link_libraries(_s2 PRIVATE ${M_LIBRARY})
-endif()

contrib/snappy

@@ -1 +1 @@
-Subproject commit 3786173af204d21da97180977ad6ab4321138b3d
+Subproject commit fb057edfed820212076239fd32cb2ff23e9016bf

contrib/thrift-cmake/CMakeLists.txt

@@ -1,7 +1,7 @@
 option(ENABLE_THRIFT "Enable Thrift" ${ENABLE_LIBRARIES})
 
 if (NOT ENABLE_THRIFT)
-    message (STATUS "thrift disabled")
+    message (STATUS "Not using thrift")
     return()
 endif()
 

contrib/unixodbc-cmake/CMakeLists.txt

@@ -294,14 +294,6 @@ target_include_directories (_unixodbc
         "${LIBRARY_DIR}/include"
 )
 target_compile_definitions (_unixodbc PRIVATE -DHAVE_CONFIG_H)
-target_compile_options (_unixodbc
+target_compile_options (_unixodbc PRIVATE -O2) # intended?
-    PRIVATE
-        -Wno-dangling-else
-        -Wno-parentheses
-        -Wno-misleading-indentation
-        -Wno-unknown-warning-option
-        -Wno-reserved-id-macro
-        -O2
-)
 
 add_library (ch_contrib::unixodbc ALIAS _unixodbc)

docs/en/development/tests.md

@@ -81,7 +81,7 @@ $ ./src/unit_tests_dbms --gtest_filter=LocalAddress*
 
 ## Performance Tests {#performance-tests}
 
-Performance tests allow to measure and compare performance of some isolated part of ClickHouse on synthetic queries. Tests are located at `tests/performance`. Each test is represented by `.xml` file with description of test case. Tests are run with `docker/tests/performance-comparison` tool . See the readme file for invocation.
+Performance tests allow to measure and compare performance of some isolated part of ClickHouse on synthetic queries. Tests are located at `tests/performance`. Each test is represented by `.xml` file with description of test case. Tests are run with `docker/test/performance-comparison` tool . See the readme file for invocation.
 
 Each test run one or multiple queries (possibly with combinations of parameters) in a loop.
 

docs/en/operations/server-configuration-parameters/settings.md

@@ -1745,13 +1745,3 @@ Possible values:
 -   Positive integer.
 
 Default value: `10000`.
-
-## global_memory_usage_overcommit_max_wait_microseconds {#global_memory_usage_overcommit_max_wait_microseconds}
-
-Sets maximum waiting time for global overcommit tracker.
-
-Possible values:
-
--   Positive integer.
-
-Default value: `200`.

docs/en/operations/settings/settings.md

@@ -4291,7 +4291,7 @@ Maximum time thread will wait for memory to be freed in the case of memory overc
 If the timeout is reached and memory is not freed, an exception is thrown.
 Read more about [memory overcommit](memory-overcommit.md).
 
-Default value: `200`.
+Default value: `5000000`.
 
 ## memory_overcommit_ratio_denominator_for_user
 

docs/en/operations/system-tables/part_log.md

@@ -14,6 +14,11 @@ The `system.part_log` table contains the following columns:
     -   `REMOVE_PART` — Removing or detaching a data part using [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition).
     -   `MUTATE_PART` — Mutating of a data part.
     -   `MOVE_PART` — Moving the data part from the one disk to another one.
+-   `merge_reason` ([Enum8](../../sql-reference/data-types/enum.md)) — The reason for the event with type `MERGE_PARTS`. Can have one of the following values:
+    -   `NOT_A_MERGE` — The current event has the type other than `MERGE_PARTS`.
+    -   `REGULAR_MERGE` — Some regular merge.
+    -   `TTL_DELETE_MERGE` — Cleaning up expired data.
+    -   `TTL_RECOMPRESS_MERGE` — Recompressing data part with the.
 -   `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
 -   `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
 -   `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds precision.
@@ -46,6 +51,7 @@ Row 1:
 ──────
 query_id:                      983ad9c7-28d5-4ae1-844e-603116b7de31
 event_type:                    NewPart
+merge_reason:                  NotAMerge
 event_date:                    2021-02-02
 event_time:                    2021-02-02 11:14:28
 event_time_microseconds:                    2021-02-02 11:14:28.861919

docs/zh/sql-reference/statements/insert-into.md

@@ -71,7 +71,7 @@ INSERT INTO [db.]table [(c1, c2, c3)] FORMAT format_name data_set
 INSERT INTO [db.]table [(c1, c2, c3)] FORMAT Values (v11, v12, v13), (v21, v22, v23), ...
 ```
 
-ClickHouse会清除数据前所有的空白字符与一行摘要信息（如果需要的话）。所以在进行查询时，我们建议您将数据放入到输入输出格式名称后的新的一行中去（如果数据是以空白字符开始的，这将非常重要）。
+ClickHouse会清除数据前所有的空白字符与一个换行符（如果有换行符的话）。所以在进行查询时，我们建议您将数据放入到输入输出格式名称后的新的一行中去（如果数据是以空白字符开始的，这将非常重要）。
 
 示例:
 
@@ -83,6 +83,10 @@ INSERT INTO t FORMAT TabSeparated
 
 在使用命令行客户端或HTTP客户端时，你可以将具体的查询语句与数据分开发送。更多具体信息，请参考«[客户端](../../interfaces/index.md#interfaces)»部分。
 
+### 限制 {#constraints}
+
+如果表中有一些[限制](../../sql-reference/statements/create/table.md#constraints),，数据插入时会逐行进行数据校验，如果这里面包含了不符合限制条件的数据，服务将会抛出包含限制信息的异常，这个语句也会被停止执行。
+
 ### 使用`SELECT`的结果写入 {#insert_query_insert-select}
 
 ``` sql
@@ -96,6 +100,66 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ...
 系统不支持的其他用于修改数据的查询：`UPDATE`, `DELETE`, `REPLACE`, `MERGE`, `UPSERT`, `INSERT UPDATE`。
 但是，您可以使用 `ALTER TABLE ... DROP PARTITION`查询来删除一些旧的数据。
 
+如果 `SELECT` 查询中包含了 [input()](../../sql-reference/table-functions/input.md) 函数，那么 `FORMAT` 必须出现在查询语句的最后。
+
+如果某一列限制了值不能是NULL，那么插入NULL的时候就会插入这个列类型的默认数据，可以通过设置 [insert_null_as_default](../../operations/settings/settings.md#insert_null_as_default) 插入NULL。
+
+### 从文件向表中插入数据 {#inserting-data-from-a-file}
+
+``` sql
+INSERT INTO [db.]table [(c1, c2, c3)] FROM INFILE file_name [COMPRESSION type] FORMAT format_name
+```
+使用上面的语句可以从客户端的文件上读取数据并插入表中，`file_name` 和 `type` 都是 `String` 类型，输入文件的[格式](../../interfaces/formats.md) 一定要在 `FORMAT` 语句中设置。
+
+支持读取压缩文件。默认会去读文件的拓展名作为文件的压缩方式，或者也可以在 `COMPRESSION` 语句中指明，支持的文件压缩格式如下：`'none'`， `'gzip'`， `'deflate'`， `'br'`， `'xz'`， `'zstd'`， `'lz4'`， `'bz2'`。
+
+这个功能在 [command-line client](../../interfaces/cli.md) 和 [clickhouse-local](../../operations/utilities/clickhouse-local.md) 是可用的。
+
+**样例**
+
+```bash
+echo 1,A > input.csv ; echo 2,B >> input.csv
+clickhouse-client --query="CREATE TABLE table_from_file (id UInt32, text String) ENGINE=MergeTree() ORDER BY id;"
+clickhouse-client --query="INSERT INTO table_from_file FROM INFILE 'input.csv' FORMAT CSV;"
+clickhouse-client --query="SELECT * FROM table_from_file FORMAT PrettyCompact;"
+```
+
+结果:
+
+```text
+┌─id─┬─text─┐
+│  1 │ A    │
+│  2 │ B    │
+└────┴──────┘
+```
+
+### 插入表函数 {#inserting-into-table-function}
+
+数据可以通过 [table functions](../../sql-reference/table-functions/index.md) 方法插入。
+``` sql
+INSERT INTO [TABLE] FUNCTION table_func ...
+```
+
+**例如**
+
+可以这样使用[remote](../../sql-reference/table-functions/index.md#remote) 表函数:
+
+``` sql
+CREATE TABLE simple_table (id UInt32, text String) ENGINE=MergeTree() ORDER BY id;
+INSERT INTO TABLE FUNCTION remote('localhost', default.simple_table) 
+    VALUES (100, 'inserted via remote()');
+SELECT * FROM simple_table;
+```
+
+结果:
+
+``` text
+┌──id─┬─text──────────────────┐
+│ 100 │ inserted via remote() │
+└─────┴───────────────────────┘
+```
+
+
 ### 性能的注意事项 {#xing-neng-de-zhu-yi-shi-xiang}
 
 在进行`INSERT`时将会对写入的数据进行一些处理，按照主键排序，按照月份对数据进行分区等。所以如果在您的写入数据中包含多个月份的混合数据时，将会显著的降低`INSERT`的性能。为了避免这种情况：
@@ -108,4 +172,6 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ...
 -   数据总是被实时的写入。
 -   写入的数据已经按照时间排序。
 
+也可以异步的、小规模的插入数据，这些数据会被合并成多个批次，然后安全地写入到表中，通过设置[async_insert](../../operations/settings/settings.md#async-insert)，可以使用异步插入的方式，请注意，异步插入的方式只支持HTTP协议，并且不支持数据去重。
+
 [来源文章](https://clickhouse.com/docs/en/query_language/insert_into/) <!--hide-->

programs/server/Server.cpp

@@ -1100,8 +1100,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
             total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
 
             auto * global_overcommit_tracker = global_context->getGlobalOvercommitTracker();
-            UInt64 max_overcommit_wait_time = config->getUInt64("global_memory_usage_overcommit_max_wait_microseconds", 200);
-            global_overcommit_tracker->setMaxWaitTime(max_overcommit_wait_time);
             total_memory_tracker.setOvercommitTracker(global_overcommit_tracker);
 
             // FIXME logging-related things need synchronization -- see the 'Logger * log' saved

src/Access/ContextAccess.cpp

@@ -445,10 +445,11 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
         const AccessFlags dictionary_ddl = AccessType::CREATE_DICTIONARY | AccessType::DROP_DICTIONARY;
         const AccessFlags function_ddl = AccessType::CREATE_FUNCTION | AccessType::DROP_FUNCTION;
         const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl;
+        const AccessFlags table_and_dictionary_and_function_ddl = table_ddl | dictionary_ddl | function_ddl;
         const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE;
         const AccessFlags write_dcl_access = AccessType::ACCESS_MANAGEMENT - AccessType::SHOW_ACCESS;
 
-        const AccessFlags not_readonly_flags = write_table_access | table_and_dictionary_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY;
+        const AccessFlags not_readonly_flags = write_table_access | table_and_dictionary_and_function_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY;
         const AccessFlags not_readonly_1_flags = AccessType::CREATE_TEMPORARY_TABLE;
 
         const AccessFlags ddl_flags = table_ddl | dictionary_ddl | function_ddl;

src/Common/ColumnsHashing.h

@@ -6,9 +6,11 @@
 #include <Common/Arena.h>
 #include <Common/LRUCache.h>
 #include <Common/assert_cast.h>
+#include "Columns/IColumn.h"
 #include <base/unaligned.h>
 
 #include <Columns/ColumnString.h>
+#include <Columns/ColumnConst.h>
 #include <Columns/ColumnFixedString.h>
 #include <Columns/ColumnLowCardinality.h>
 
@@ -83,8 +85,11 @@ struct HashMethodString
 
     HashMethodString(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const HashMethodContextPtr &)
     {
-        const IColumn & column = *key_columns[0];
+        const IColumn * column = key_columns[0];
-        const ColumnString & column_string = assert_cast<const ColumnString &>(column);
+        if (isColumnConst(*column))
+            column = &assert_cast<const ColumnConst &>(*column).getDataColumn();
+
+        const ColumnString & column_string = assert_cast<const ColumnString &>(*column);
         offsets = column_string.getOffsets().data();
         chars = column_string.getChars().data();
     }

src/Common/CpuId.h

@@ -221,7 +221,7 @@ bool haveAVX512F() noexcept
            && (our_xgetbv(0) & 6u) == 6u              // XMM state and YMM state are enabled by OS
            && ((our_xgetbv(0) >> 5) & 7u) == 7u       // ZMM state is enabled by OS
            && CpuInfo(0x0).registers.eax >= 0x7          // leaf 7 is present
-           && ((CpuInfo(0x7).registers.ebx >> 16) & 1u); // AVX512F bit
+           && ((CpuInfo(0x7, 0).registers.ebx >> 16) & 1u); // AVX512F bit
 #else
     return false;
 #endif

src/Common/MemoryTracker.cpp

@@ -16,6 +16,7 @@
 #include <cmath>
 #include <random>
 #include <cstdlib>
+#include <string>
 
 
 #ifdef MEMORY_TRACKER_DEBUG_CHECKS
@@ -52,11 +53,37 @@ namespace DB
     }
 }
 
+namespace
+{
+
+inline std::string_view toDescription(OvercommitResult result)
+{
+    switch (result)
+    {
+    case OvercommitResult::NONE:
+        return "Memory overcommit isn't used. OvercommitTracker isn't set.";
+    case OvercommitResult::DISABLED:
+        return "Memory overcommit isn't used. Waiting time or orvercommit denominator are set to zero.";
+    case OvercommitResult::MEMORY_FREED:
+        throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "OvercommitResult::MEMORY_FREED shouldn't be asked for description");
+    case OvercommitResult::SELECTED:
+        return "Query was selected to stop by OvercommitTracker.";
+    case OvercommitResult::TIMEOUTED:
+        return "Waiting timeout for memory to be freed is reached.";
+    case OvercommitResult::NOT_ENOUGH_FREED:
+        return "Memory overcommit has freed not enough memory.";
+    }
+}
+
+}
+
 namespace ProfileEvents
 {
     extern const Event QueryMemoryLimitExceeded;
 }
 
+using namespace std::chrono_literals;
+
 static constexpr size_t log_peak_memory_usage_every = 1ULL << 30;
 
 MemoryTracker total_memory_tracker(nullptr, VariableContext::Global);
@@ -189,11 +216,11 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryT
 
     if (unlikely(current_hard_limit && will_be > current_hard_limit) && memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded)
     {
-        bool need_to_throw = true;
+        OvercommitResult overcommit_result = OvercommitResult::NONE;
         if (auto * overcommit_tracker_ptr = overcommit_tracker.load(std::memory_order_relaxed); overcommit_tracker_ptr != nullptr && query_tracker != nullptr)
-            need_to_throw = overcommit_tracker_ptr->needToStopQuery(query_tracker, size);
+            overcommit_result = overcommit_tracker_ptr->needToStopQuery(query_tracker, size);
 
-        if (need_to_throw)
+        if (overcommit_result != OvercommitResult::MEMORY_FREED)
         {
             /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc
             MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global);
@@ -201,12 +228,13 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryT
             const auto * description = description_ptr.load(std::memory_order_relaxed);
             throw DB::Exception(
                 DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED,
-                "Memory limit{}{} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}",
+                "Memory limit{}{} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}. OvercommitTracker decision: {}.",
                 description ? " " : "",
                 description ? description : "",
                 formatReadableSizeWithBinarySuffix(will_be),
                 size,
-                formatReadableSizeWithBinarySuffix(current_hard_limit));
+                formatReadableSizeWithBinarySuffix(current_hard_limit),
+                toDescription(overcommit_result));
         }
         else
         {
@@ -337,6 +365,12 @@ OvercommitRatio MemoryTracker::getOvercommitRatio(Int64 limit)
 }
 
 
+void MemoryTracker::setOvercommitWaitingTime(UInt64 wait_time)
+{
+    max_wait_time.store(wait_time * 1us, std::memory_order_relaxed);
+}
+
+
 void MemoryTracker::resetCounters()
 {
     amount.store(0, std::memory_order_relaxed);

src/Common/MemoryTracker.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <atomic>
+#include <chrono>
 #include <base/types.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/VariableContext.h>
@@ -73,6 +74,8 @@ private:
     /// This description will be used as prefix into log messages (if isn't nullptr)
     std::atomic<const char *> description_ptr = nullptr;
 
+    std::atomic<std::chrono::microseconds> max_wait_time;
+
     std::atomic<OvercommitTracker *> overcommit_tracker = nullptr;
 
     bool updatePeak(Int64 will_be, bool log_memory_usage);
@@ -186,6 +189,13 @@ public:
     OvercommitRatio getOvercommitRatio();
     OvercommitRatio getOvercommitRatio(Int64 limit);
 
+    std::chrono::microseconds getOvercommitWaitingTime()
+    {
+        return max_wait_time.load(std::memory_order_relaxed);
+    }
+
+    void setOvercommitWaitingTime(UInt64 wait_time);
+
     void setOvercommitTracker(OvercommitTracker * tracker) noexcept
     {
         overcommit_tracker.store(tracker, std::memory_order_relaxed);

src/Common/OvercommitTracker.cpp

@@ -2,15 +2,20 @@
 
 #include <chrono>
 #include <mutex>
+#include <Common/ProfileEvents.h>
 #include <Interpreters/ProcessList.h>
 
+namespace ProfileEvents
+{
+    extern const Event MemoryOvercommitWaitTimeMicroseconds;
+}
+
 using namespace std::chrono_literals;
 
 constexpr std::chrono::microseconds ZERO_MICROSEC = 0us;
 
 OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_)
-    : max_wait_time(ZERO_MICROSEC)
+    : picked_tracker(nullptr)
-    , picked_tracker(nullptr)
     , cancellation_state(QueryCancellationState::NONE)
     , global_mutex(global_mutex_)
     , freed_memory(0)
@@ -18,13 +23,7 @@ OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_)
     , allow_release(true)
 {}
 
-void OvercommitTracker::setMaxWaitTime(UInt64 wait_time)
+OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount)
-{
-    std::lock_guard guard(overcommit_m);
-    max_wait_time = wait_time * 1us;
-}
-
-bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount)
 {
     // NOTE: Do not change the order of locks
     //
@@ -35,8 +34,10 @@ bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount)
     std::unique_lock<std::mutex> global_lock(global_mutex);
     std::unique_lock<std::mutex> lk(overcommit_m);
 
+    auto max_wait_time = tracker->getOvercommitWaitingTime();
+
     if (max_wait_time == ZERO_MICROSEC)
-        return true;
+        return OvercommitResult::DISABLED;
 
     pickQueryToExclude();
     assert(cancellation_state != QueryCancellationState::NONE);
@@ -50,7 +51,7 @@ bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount)
         // picked_tracker to be not null pointer.
         assert(cancellation_state == QueryCancellationState::SELECTED);
         cancellation_state = QueryCancellationState::NONE;
-        return true;
+        return OvercommitResult::DISABLED;
     }
     if (picked_tracker == tracker)
     {
@@ -58,17 +59,20 @@ bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount)
         // It may happen even when current state is RUNNING, because
         // ThreadStatus::~ThreadStatus may call MemoryTracker::alloc.
         cancellation_state = QueryCancellationState::RUNNING;
-        return true;
+        return OvercommitResult::SELECTED;
     }
 
     allow_release = true;
 
     required_memory += amount;
     required_per_thread[tracker] = amount;
+    auto wait_start_time = std::chrono::system_clock::now();
     bool timeout = !cv.wait_for(lk, max_wait_time, [this, tracker]()
     {
         return required_per_thread[tracker] == 0 || cancellation_state == QueryCancellationState::NONE;
     });
+    auto wait_end_time = std::chrono::system_clock::now();
+    ProfileEvents::increment(ProfileEvents::MemoryOvercommitWaitTimeMicroseconds, (wait_end_time - wait_start_time) / 1us);
     LOG_DEBUG(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : ""));
 
     required_memory -= amount;
@@ -84,7 +88,12 @@ bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount)
     // As we don't need to free memory, we can continue execution of the selected query.
     if (required_memory == 0 && cancellation_state == QueryCancellationState::SELECTED)
         reset();
-    return timeout || still_need != 0;
+    if (timeout)
+        return OvercommitResult::TIMEOUTED;
+    if (still_need != 0)
+        return OvercommitResult::NOT_ENOUGH_FREED;
+    else
+        return OvercommitResult::MEMORY_FREED;
 }
 
 void OvercommitTracker::tryContinueQueryExecutionAfterFree(Int64 amount)

src/Common/OvercommitTracker.h

@@ -36,6 +36,16 @@ struct OvercommitRatio
 
 class MemoryTracker;
 
+enum class OvercommitResult
+{
+    NONE,
+    DISABLED,
+    MEMORY_FREED,
+    SELECTED,
+    TIMEOUTED,
+    NOT_ENOUGH_FREED,
+};
+
 enum class QueryCancellationState
 {
     NONE     = 0,  // Hard limit is not reached, there is no selected query to kill.
@@ -52,9 +62,7 @@ enum class QueryCancellationState
 // is killed to free memory.
 struct OvercommitTracker : boost::noncopyable
 {
-    void setMaxWaitTime(UInt64 wait_time);
+    OvercommitResult needToStopQuery(MemoryTracker * tracker, Int64 amount);
-
-    bool needToStopQuery(MemoryTracker * tracker, Int64 amount);
 
     void tryContinueQueryExecutionAfterFree(Int64 amount);
 
@@ -72,8 +80,6 @@ protected:
     std::mutex overcommit_m;
     std::condition_variable cv;
 
-    std::chrono::microseconds max_wait_time;
-
     // Specifies memory tracker of the chosen to stop query.
     // If soft limit is not set, all the queries which reach hard limit must stop.
     // This case is represented as picked tracker pointer is set to nullptr and

src/Common/ProfileEvents.cpp

@@ -199,6 +199,7 @@
     M(RealTimeMicroseconds, "Total (wall clock) time spent in processing (queries and other tasks) threads (not that this is a sum).") \
     M(UserTimeMicroseconds, "Total time spent in processing (queries and other tasks) threads executing CPU instructions in user space. This include time CPU pipeline was stalled due to cache misses, branch mispredictions, hyper-threading, etc.") \
     M(SystemTimeMicroseconds, "Total time spent in processing (queries and other tasks) threads executing CPU instructions in OS kernel space. This include time CPU pipeline was stalled due to cache misses, branch mispredictions, hyper-threading, etc.") \
+    M(MemoryOvercommitWaitTimeMicroseconds, "Total time spent in waiting for memory to be freed in OvercommitTracker.") \
     M(SoftPageFaults, "") \
     M(HardPageFaults, "") \
     \

src/Common/TargetSpecific.cpp

@@ -16,6 +16,8 @@ UInt32 getSupportedArchs()
         result |= static_cast<UInt32>(TargetArch::AVX2);
     if (Cpu::CpuFlagsCache::have_AVX512F)
         result |= static_cast<UInt32>(TargetArch::AVX512F);
+    if (Cpu::CpuFlagsCache::have_AVX512BW)
+        result |= static_cast<UInt32>(TargetArch::AVX512BW);
     return result;
 }
 
@@ -34,6 +36,7 @@ String toString(TargetArch arch)
         case TargetArch::AVX:     return "avx";
         case TargetArch::AVX2:    return "avx2";
         case TargetArch::AVX512F: return "avx512f";
+        case TargetArch::AVX512BW: return "avx512bw";
     }
 
     __builtin_unreachable();

src/Common/TargetSpecific.h

@@ -80,6 +80,7 @@ enum class TargetArch : UInt32
     AVX      = (1 << 1),
     AVX2     = (1 << 2),
     AVX512F  = (1 << 3),
+    AVX512BW  = (1 << 4),
 };
 
 /// Runtime detection.

src/Common/tests/gtest_overcommit_tracker.cpp

@@ -40,15 +40,17 @@ static constexpr UInt64 WAIT_TIME = 4'000'000;
 template <typename T>
 void free_not_continue_test(T & overcommit_tracker)
 {
-    overcommit_tracker.setMaxWaitTime(WAIT_TIME);
-
     static constexpr size_t THREADS = 5;
     std::vector<MemoryTracker> trackers(THREADS);
+    for (auto & tracker : trackers)
+        tracker.setOvercommitWaitingTime(WAIT_TIME);
+
     std::atomic<int> need_to_stop = 0;
     std::vector<std::thread> threads;
     threads.reserve(THREADS);
 
     MemoryTracker picked;
+    picked.setOvercommitWaitingTime(WAIT_TIME);
     overcommit_tracker.setCandidate(&picked);
 
     for (size_t i = 0; i < THREADS; ++i)
@@ -56,7 +58,7 @@ void free_not_continue_test(T & overcommit_tracker)
         threads.push_back(std::thread(
             [&, i]()
             {
-                if (overcommit_tracker.needToStopQuery(&trackers[i], 100))
+                if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED)
                     ++need_to_stop;
             }
         ));
@@ -96,15 +98,16 @@ TEST(OvercommitTracker, GlobalFreeNotContinue)
 template <typename T>
 void free_continue_test(T & overcommit_tracker)
 {
-    overcommit_tracker.setMaxWaitTime(WAIT_TIME);
-
     static constexpr size_t THREADS = 5;
     std::vector<MemoryTracker> trackers(THREADS);
+    for (auto & tracker : trackers)
+        tracker.setOvercommitWaitingTime(WAIT_TIME);
     std::atomic<int> need_to_stop = 0;
     std::vector<std::thread> threads;
     threads.reserve(THREADS);
 
     MemoryTracker picked;
+    picked.setOvercommitWaitingTime(WAIT_TIME);
     overcommit_tracker.setCandidate(&picked);
 
     for (size_t i = 0; i < THREADS; ++i)
@@ -112,7 +115,7 @@ void free_continue_test(T & overcommit_tracker)
         threads.push_back(std::thread(
             [&, i]()
             {
-                if (overcommit_tracker.needToStopQuery(&trackers[i], 100))
+                if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED)
                     ++need_to_stop;
             }
         ));
@@ -152,15 +155,16 @@ TEST(OvercommitTracker, GlobalFreeContinue)
 template <typename T>
 void free_continue_and_alloc_test(T & overcommit_tracker)
 {
-    overcommit_tracker.setMaxWaitTime(WAIT_TIME);
-
     static constexpr size_t THREADS = 5;
     std::vector<MemoryTracker> trackers(THREADS);
+    for (auto & tracker : trackers)
+        tracker.setOvercommitWaitingTime(WAIT_TIME);
     std::atomic<int> need_to_stop = 0;
     std::vector<std::thread> threads;
     threads.reserve(THREADS);
 
     MemoryTracker picked;
+    picked.setOvercommitWaitingTime(WAIT_TIME);
     overcommit_tracker.setCandidate(&picked);
 
     for (size_t i = 0; i < THREADS; ++i)
@@ -168,7 +172,7 @@ void free_continue_and_alloc_test(T & overcommit_tracker)
         threads.push_back(std::thread(
             [&, i]()
             {
-                if (overcommit_tracker.needToStopQuery(&trackers[i], 100))
+                if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED)
                     ++need_to_stop;
             }
         ));
@@ -179,9 +183,10 @@ void free_continue_and_alloc_test(T & overcommit_tracker)
         [&]()
         {
             MemoryTracker failed;
+            failed.setOvercommitWaitingTime(WAIT_TIME);
             std::this_thread::sleep_for(1000ms);
             overcommit_tracker.tryContinueQueryExecutionAfterFree(5000);
-            stopped_next = overcommit_tracker.needToStopQuery(&failed, 100);
+            stopped_next = overcommit_tracker.needToStopQuery(&failed, 100) != OvercommitResult::MEMORY_FREED;
         }
     ).join();
 
@@ -212,15 +217,16 @@ TEST(OvercommitTracker, GlobalFreeContinueAndAlloc)
 template <typename T>
 void free_continue_and_alloc_2_test(T & overcommit_tracker)
 {
-    overcommit_tracker.setMaxWaitTime(WAIT_TIME);
-
     static constexpr size_t THREADS = 5;
     std::vector<MemoryTracker> trackers(THREADS);
+    for (auto & tracker : trackers)
+        tracker.setOvercommitWaitingTime(WAIT_TIME);
     std::atomic<int> need_to_stop = 0;
     std::vector<std::thread> threads;
     threads.reserve(THREADS);
 
     MemoryTracker picked;
+    picked.setOvercommitWaitingTime(WAIT_TIME);
     overcommit_tracker.setCandidate(&picked);
 
     for (size_t i = 0; i < THREADS; ++i)
@@ -228,7 +234,7 @@ void free_continue_and_alloc_2_test(T & overcommit_tracker)
         threads.push_back(std::thread(
             [&, i]()
             {
-                if (overcommit_tracker.needToStopQuery(&trackers[i], 100))
+                if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED)
                     ++need_to_stop;
             }
         ));
@@ -239,9 +245,10 @@ void free_continue_and_alloc_2_test(T & overcommit_tracker)
         [&]()
         {
             MemoryTracker failed;
+            failed.setOvercommitWaitingTime(WAIT_TIME);
             std::this_thread::sleep_for(1000ms);
             overcommit_tracker.tryContinueQueryExecutionAfterFree(5000);
-            stopped_next = overcommit_tracker.needToStopQuery(&failed, 100);
+            stopped_next = overcommit_tracker.needToStopQuery(&failed, 100) != OvercommitResult::MEMORY_FREED;
         }
     ));
 
@@ -280,15 +287,16 @@ TEST(OvercommitTracker, GlobalFreeContinueAndAlloc2)
 template <typename T>
 void free_continue_and_alloc_3_test(T & overcommit_tracker)
 {
-    overcommit_tracker.setMaxWaitTime(WAIT_TIME);
-
     static constexpr size_t THREADS = 5;
     std::vector<MemoryTracker> trackers(THREADS);
+    for (auto & tracker : trackers)
+        tracker.setOvercommitWaitingTime(WAIT_TIME);
     std::atomic<int> need_to_stop = 0;
     std::vector<std::thread> threads;
     threads.reserve(THREADS);
 
     MemoryTracker picked;
+    picked.setOvercommitWaitingTime(WAIT_TIME);
     overcommit_tracker.setCandidate(&picked);
 
     for (size_t i = 0; i < THREADS; ++i)
@@ -296,7 +304,7 @@ void free_continue_and_alloc_3_test(T & overcommit_tracker)
         threads.push_back(std::thread(
             [&, i]()
             {
-                if (overcommit_tracker.needToStopQuery(&trackers[i], 100))
+                if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED)
                     ++need_to_stop;
             }
         ));
@@ -307,9 +315,10 @@ void free_continue_and_alloc_3_test(T & overcommit_tracker)
         [&]()
         {
             MemoryTracker failed;
+            failed.setOvercommitWaitingTime(WAIT_TIME);
             std::this_thread::sleep_for(1000ms);
             overcommit_tracker.tryContinueQueryExecutionAfterFree(5000);
-            stopped_next = overcommit_tracker.needToStopQuery(&failed, 100);
+            stopped_next = overcommit_tracker.needToStopQuery(&failed, 100) != OvercommitResult::MEMORY_FREED;
         }
     ));
 
@@ -348,15 +357,16 @@ TEST(OvercommitTracker, GlobalFreeContinueAndAlloc3)
 template <typename T>
 void free_continue_2_test(T & overcommit_tracker)
 {
-    overcommit_tracker.setMaxWaitTime(WAIT_TIME);
-
     static constexpr size_t THREADS = 5;
     std::vector<MemoryTracker> trackers(THREADS);
+    for (auto & tracker : trackers)
+        tracker.setOvercommitWaitingTime(WAIT_TIME);
     std::atomic<int> need_to_stop = 0;
     std::vector<std::thread> threads;
     threads.reserve(THREADS);
 
     MemoryTracker picked;
+    picked.setOvercommitWaitingTime(WAIT_TIME);
     overcommit_tracker.setCandidate(&picked);
 
     for (size_t i = 0; i < THREADS; ++i)
@@ -364,7 +374,7 @@ void free_continue_2_test(T & overcommit_tracker)
         threads.push_back(std::thread(
             [&, i]()
             {
-                if (overcommit_tracker.needToStopQuery(&trackers[i], 100))
+                if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED)
                     ++need_to_stop;
             }
         ));
@@ -404,18 +414,18 @@ TEST(OvercommitTracker, GlobalFreeContinue2)
 template <typename T>
 void query_stop_not_continue_test(T & overcommit_tracker)
 {
-    overcommit_tracker.setMaxWaitTime(WAIT_TIME);
-
     std::atomic<int> need_to_stop = 0;
 
     MemoryTracker picked;
+    picked.setOvercommitWaitingTime(WAIT_TIME);
     overcommit_tracker.setCandidate(&picked);
 
     MemoryTracker another;
+    another.setOvercommitWaitingTime(WAIT_TIME);
     auto thread = std::thread(
         [&]()
         {
-            if (overcommit_tracker.needToStopQuery(&another, 100))
+            if (overcommit_tracker.needToStopQuery(&another, 100) != OvercommitResult::MEMORY_FREED)
                 ++need_to_stop;
         }
     );

src/Core/Settings.h

@@ -372,7 +372,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
     M(UInt64, memory_profiler_step, (4 * 1024 * 1024), "Whenever query memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stack trace. Zero means disabled memory profiler. Values lower than a few megabytes will slow down query processing.", 0) \
     M(Float, memory_profiler_sample_probability, 0., "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation. Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \
     \
-    M(UInt64, memory_usage_overcommit_max_wait_microseconds, 200, "Maximum time thread will wait for memory to be freed in the case of memory overcommit on user level. If timeout is reached and memory is not freed, exception is thrown.", 0) \
+    M(UInt64, memory_usage_overcommit_max_wait_microseconds, 5'000'000, "Maximum time thread will wait for memory to be freed in the case of memory overcommit. If timeout is reached and memory is not freed, exception is thrown.", 0) \
     \
     M(UInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \
     M(UInt64, max_network_bytes, 0, "The maximum number of bytes (compressed) to receive or transmit over the network for execution of the query.", 0) \
@@ -695,7 +695,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
     M(Bool, output_format_json_quote_denormals, false, "Enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format.", 0) \
     \
     M(Bool, output_format_json_escape_forward_slashes, true, "Controls escaping forward slashes for string outputs in JSON output format. This is intended for compatibility with JavaScript. Don't confuse with backslashes that are always escaped.", 0) \
-    M(Bool, output_format_json_named_tuples_as_objects, false, "Serialize named tuple columns as JSON objects.", 0) \
+    M(Bool, output_format_json_named_tuples_as_objects, true, "Serialize named tuple columns as JSON objects.", 0) \
     M(Bool, output_format_json_array_of_rows, false, "Output a JSON array of all rows in JSONEachRow(Compact) format.", 0) \
     \
     M(UInt64, output_format_pretty_max_rows, 10000, "Rows limit for Pretty formats.", 0) \

src/Databases/MySQL/DatabaseMySQL.cpp

@@ -26,6 +26,7 @@
 #    include <Common/setThreadName.h>
 #    include <filesystem>
 #    include <Common/filesystemHelpers.h>
+#    include <Parsers/ASTIdentifier.h>
 
 namespace fs = std::filesystem;
 
@@ -148,8 +149,16 @@ ASTPtr DatabaseMySQL::getCreateTableQueryImpl(const String & table_name, Context
         auto storage_engine_arguments = ast_storage->engine->arguments;
 
         /// Add table_name to engine arguments
-        auto mysql_table_name = std::make_shared<ASTLiteral>(table_name);
+        if (typeid_cast<ASTIdentifier *>(storage_engine_arguments->children[0].get()))
-        storage_engine_arguments->children.insert(storage_engine_arguments->children.begin() + 2, mysql_table_name);
+        {
+            storage_engine_arguments->children.push_back(
+                makeASTFunction("equals", std::make_shared<ASTIdentifier>("table"), std::make_shared<ASTLiteral>(table_name)));
+        }
+        else
+        {
+            auto mysql_table_name = std::make_shared<ASTLiteral>(table_name);
+            storage_engine_arguments->children.insert(storage_engine_arguments->children.begin() + 2, mysql_table_name);
+        }
 
         /// Unset settings
         std::erase_if(storage_children, [&](const ASTPtr & element) { return element.get() == ast_storage->settings; });

src/Dictionaries/HTTPDictionarySource.cpp

@@ -8,6 +8,7 @@
 #include <IO/WriteHelpers.h>
 #include <Processors/Formats/IInputFormat.h>
 #include <Interpreters/Context.h>
+#include <Storages/ExternalDataSourceConfiguration.h>
 #include <Poco/Net/HTTPRequest.h>
 #include <Common/logger_useful.h>
 #include "DictionarySourceFactory.h"
@@ -228,45 +229,80 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory)
         if (dict_struct.has_expressions)
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Dictionary source of type `http` does not support attribute expressions");
 
-        auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix);
+        auto settings_config_prefix = config_prefix + ".http";
-
-        const auto & settings_config_prefix = config_prefix + ".http";
-        const auto & credentials_prefix = settings_config_prefix + ".credentials";
-
         Poco::Net::HTTPBasicCredentials credentials;
-
-        if (config.has(credentials_prefix))
-        {
-            credentials.setUsername(config.getString(credentials_prefix + ".user", ""));
-            credentials.setPassword(config.getString(credentials_prefix + ".password", ""));
-        }
-
-        const auto & headers_prefix = settings_config_prefix + ".headers";
         ReadWriteBufferFromHTTP::HTTPHeaderEntries header_entries;
+        String url;
+        String endpoint;
+        String format;
 
-        if (config.has(headers_prefix))
+        auto named_collection = created_from_ddl
+                            ? getURLBasedDataSourceConfiguration(config, settings_config_prefix, global_context)
+                            : std::nullopt;
+        if (named_collection)
         {
-            Poco::Util::AbstractConfiguration::Keys config_keys;
+            url = named_collection->configuration.url;
-            config.keys(headers_prefix, config_keys);
+            endpoint = named_collection->configuration.endpoint;
+            format = named_collection->configuration.format;
 
-            header_entries.reserve(config_keys.size());
+            credentials.setUsername(named_collection->configuration.user);
-            for (const auto & key : config_keys)
+            credentials.setPassword(named_collection->configuration.password);
-            {
+
-                const auto header_key = config.getString(headers_prefix + "." + key + ".name", "");
+            header_entries.reserve(named_collection->configuration.headers.size());
-                const auto header_value = config.getString(headers_prefix + "." + key + ".value", "");
+            for (const auto & header : named_collection->configuration.headers)
-                header_entries.emplace_back(std::make_tuple(header_key, header_value));
+                header_entries.emplace_back(std::make_tuple(header.first, header.second.get<String>()));
-            }
         }
+        else
+        {
+            const auto & credentials_prefix = settings_config_prefix + ".credentials";
+
+            if (config.has(credentials_prefix))
+            {
+                credentials.setUsername(config.getString(credentials_prefix + ".user", ""));
+                credentials.setPassword(config.getString(credentials_prefix + ".password", ""));
+            }
+
+            const auto & headers_prefix = settings_config_prefix + ".headers";
+
+
+            if (config.has(headers_prefix))
+            {
+                Poco::Util::AbstractConfiguration::Keys config_keys;
+                config.keys(headers_prefix, config_keys);
+
+                header_entries.reserve(config_keys.size());
+                for (const auto & key : config_keys)
+                {
+                    const auto header_key = config.getString(headers_prefix + "." + key + ".name", "");
+                    const auto header_value = config.getString(headers_prefix + "." + key + ".value", "");
+                    header_entries.emplace_back(std::make_tuple(header_key, header_value));
+                }
+            }
+
+            url = config.getString(settings_config_prefix + ".url", "");
+            endpoint = config.getString(settings_config_prefix + ".endpoint", "");
+            format =config.getString(settings_config_prefix + ".format", "");
+        }
+
+        if (url.ends_with('/'))
+        {
+            if (endpoint.starts_with('/'))
+                url.pop_back();
+        }
+        else if (!endpoint.empty() && !endpoint.starts_with('/'))
+            url.push_back('/');
 
         auto configuration = HTTPDictionarySource::Configuration
         {
-            .url = config.getString(settings_config_prefix + ".url", ""),
+            .url = url + endpoint,
-            .format =config.getString(settings_config_prefix + ".format", ""),
+            .format = format,
             .update_field = config.getString(settings_config_prefix + ".update_field", ""),
             .update_lag = config.getUInt64(settings_config_prefix + ".update_lag", 1),
             .header_entries = std::move(header_entries) //-V1030
         };
 
+        auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix);
+
         return std::make_unique<HTTPDictionarySource>(dict_struct, configuration, credentials, sample_block, context, created_from_ddl);
     };
     factory.registerSource("http", create_table_source);

src/Functions/FunctionHashID.h

@@ -51,9 +51,11 @@ public:
 
     bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
 
+    bool useDefaultImplementationForConstants() const override { return true; }
+
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
-        if (arguments.size() < 1)
+        if (arguments.empty())
             throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Function {} expects at least one argument", getName());
 
         const auto & id_col = arguments[0];
@@ -114,18 +116,16 @@ public:
         const auto & numcolumn = arguments[0].column;
 
         if (checkAndGetColumn<ColumnUInt8>(numcolumn.get()) || checkAndGetColumn<ColumnUInt16>(numcolumn.get())
-            || checkAndGetColumn<ColumnUInt32>(numcolumn.get()) || checkAndGetColumn<ColumnUInt64>(numcolumn.get())
+            || checkAndGetColumn<ColumnUInt32>(numcolumn.get()) || checkAndGetColumn<ColumnUInt64>(numcolumn.get()))
-            || checkAndGetColumnConst<ColumnUInt8>(numcolumn.get()) || checkAndGetColumnConst<ColumnUInt16>(numcolumn.get())
-            || checkAndGetColumnConst<ColumnUInt32>(numcolumn.get()) || checkAndGetColumnConst<ColumnUInt64>(numcolumn.get()))
         {
             std::string salt;
-            UInt8 minLength = 0;
+            UInt8 min_length = 0;
             std::string alphabet;
 
             if (arguments.size() >= 4)
             {
                 const auto & alphabetcolumn = arguments[3].column;
-                if (auto alpha_col = checkAndGetColumnConst<ColumnString>(alphabetcolumn.get()))
+                if (const auto * alpha_col = checkAndGetColumnConst<ColumnString>(alphabetcolumn.get()))
                 {
                     alphabet = alpha_col->getValue<String>();
                     if (alphabet.find('\0') != std::string::npos)
@@ -138,18 +138,18 @@ public:
             if (arguments.size() >= 3)
             {
                 const auto & minlengthcolumn = arguments[2].column;
-                if (auto min_length_col = checkAndGetColumnConst<ColumnUInt8>(minlengthcolumn.get()))
+                if (const auto * min_length_col = checkAndGetColumnConst<ColumnUInt8>(minlengthcolumn.get()))
-                    minLength = min_length_col->getValue<UInt8>();
+                    min_length = min_length_col->getValue<UInt8>();
             }
 
             if (arguments.size() >= 2)
             {
                 const auto & saltcolumn = arguments[1].column;
-                if (auto salt_col = checkAndGetColumnConst<ColumnString>(saltcolumn.get()))
+                if (const auto * salt_col = checkAndGetColumnConst<ColumnString>(saltcolumn.get()))
                     salt = salt_col->getValue<String>();
             }
 
-            hashidsxx::Hashids hash(salt, minLength, alphabet);
+            hashidsxx::Hashids hash(salt, min_length, alphabet);
 
             auto col_res = ColumnString::create();
 

src/Functions/FunctionsStringArray.h

@@ -448,7 +448,7 @@ public:
 class SplitByRegexpImpl
 {
 private:
-    Regexps::Pool::Pointer re;
+    Regexps::RegexpPtr re;
     OptimizedRegularExpression::MatchVec matches;
 
     Pos pos;
@@ -477,7 +477,7 @@ public:
                             ErrorCodes::ILLEGAL_COLUMN);
 
         if (!col->getValue<String>().empty())
-            re = Regexps::get<false, false, false>(col->getValue<String>());
+            re = std::make_shared<Regexps::Regexp>(Regexps::createRegexp<false, false, false>(col->getValue<String>()));
 
     }
 
@@ -532,7 +532,7 @@ public:
 class ExtractAllImpl
 {
 private:
-    Regexps::Pool::Pointer re;
+    Regexps::RegexpPtr re;
     OptimizedRegularExpression::MatchVec matches;
     size_t capture;
 
@@ -560,7 +560,7 @@ public:
                 + " of first argument of function " + getName() + ". Must be constant string.",
                 ErrorCodes::ILLEGAL_COLUMN);
 
-        re = Regexps::get<false, false, false>(col->getValue<String>());
+        re = std::make_shared<Regexps::Regexp>(Regexps::createRegexp<false, false, false>(col->getValue<String>()));
         capture = re->getNumberOfSubpatterns() > 0 ? 1 : 0;
 
         matches.resize(capture + 1);

src/Functions/MatchImpl.h

@@ -18,8 +18,6 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int ILLEGAL_COLUMN;
-    extern const int LOGICAL_ERROR;
-    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
 }
 
 namespace impl
@@ -112,16 +110,14 @@ struct MatchImpl
         const ColumnString::Chars & haystack_data,
         const ColumnString::Offsets & haystack_offsets,
         const String & needle,
-        const ColumnPtr & start_pos_,
+        [[maybe_unused]] const ColumnPtr & start_pos_,
         PaddedPODArray<UInt8> & res)
     {
         const size_t haystack_size = haystack_offsets.size();
 
-        if (haystack_size != res.size())
+        assert(haystack_size == res.size());
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Function '{}' unexpectedly received a different number of haystacks and results", name);
 
-        if (start_pos_ != nullptr)
+        assert(start_pos_ == nullptr);
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' doesn't support start_pos argument", name);
 
         if (haystack_offsets.empty())
             return;
@@ -166,17 +162,17 @@ struct MatchImpl
         }
         else
         {
-            auto regexp = Regexps::get<is_like, true, case_insensitive>(needle);
+            const auto & regexp = Regexps::Regexp(Regexps::createRegexp<is_like, /*no_capture*/ true, case_insensitive>(needle));
 
             String required_substring;
             bool is_trivial;
             bool required_substring_is_prefix; /// for `anchored` execution of the regexp.
 
-            regexp->getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix);
+            regexp.getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix);
 
             if (required_substring.empty())
             {
-                if (!regexp->getRE2()) /// An empty regexp. Always matches.
+                if (!regexp.getRE2()) /// An empty regexp. Always matches.
                 {
                     if (haystack_size)
                         memset(res.data(), !negate, haystack_size * sizeof(res[0]));
@@ -186,7 +182,7 @@ struct MatchImpl
                     size_t prev_offset = 0;
                     for (size_t i = 0; i < haystack_size; ++i)
                     {
-                        const bool match = regexp->getRE2()->Match(
+                        const bool match = regexp.getRE2()->Match(
                                 {reinterpret_cast<const char *>(&haystack_data[prev_offset]), haystack_offsets[i] - prev_offset - 1},
                                 0,
                                 haystack_offsets[i] - prev_offset - 1,
@@ -241,7 +237,7 @@ struct MatchImpl
                             const size_t start_pos = (required_substring_is_prefix) ? (reinterpret_cast<const char *>(pos) - str_data) : 0;
                             const size_t end_pos = str_size;
 
-                            const bool match = regexp->getRE2()->Match(
+                            const bool match = regexp.getRE2()->Match(
                                     {str_data, str_size},
                                     start_pos,
                                     end_pos,
@@ -274,8 +270,7 @@ struct MatchImpl
     {
         const size_t haystack_size = haystack.size() / N;
 
-        if (haystack_size != res.size())
+        assert(haystack_size == res.size());
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Function '{}' unexpectedly received a different number of haystacks and results", name);
 
         if (haystack.empty())
             return;
@@ -325,17 +320,17 @@ struct MatchImpl
         }
         else
         {
-            auto regexp = Regexps::get<is_like, true, case_insensitive>(needle);
+            const auto & regexp = Regexps::Regexp(Regexps::createRegexp<is_like, /*no_capture*/ true, case_insensitive>(needle));
 
             String required_substring;
             bool is_trivial;
             bool required_substring_is_prefix; /// for `anchored` execution of the regexp.
 
-            regexp->getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix);
+            regexp.getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix);
 
             if (required_substring.empty())
             {
-                if (!regexp->getRE2()) /// An empty regexp. Always matches.
+                if (!regexp.getRE2()) /// An empty regexp. Always matches.
                 {
                     if (haystack_size)
                         memset(res.data(), !negate, haystack_size * sizeof(res[0]));
@@ -345,7 +340,7 @@ struct MatchImpl
                     size_t offset = 0;
                     for (size_t i = 0; i < haystack_size; ++i)
                     {
-                        const bool match = regexp->getRE2()->Match(
+                        const bool match = regexp.getRE2()->Match(
                                 {reinterpret_cast<const char *>(&haystack[offset]), N},
                                 0,
                                 N,
@@ -403,7 +398,7 @@ struct MatchImpl
                                 const size_t start_pos = (required_substring_is_prefix) ? (reinterpret_cast<const char *>(pos) - str_data) : 0;
                                 const size_t end_pos = N;
 
-                                const bool match = regexp->getRE2()->Match(
+                                const bool match = regexp.getRE2()->Match(
                                         {str_data, N},
                                         start_pos,
                                         end_pos,
@@ -433,16 +428,15 @@ struct MatchImpl
         const ColumnString::Offsets & haystack_offsets,
         const ColumnString::Chars & needle_data,
         const ColumnString::Offsets & needle_offset,
-        const ColumnPtr & start_pos_,
+        [[maybe_unused]] const ColumnPtr & start_pos_,
         PaddedPODArray<UInt8> & res)
     {
         const size_t haystack_size = haystack_offsets.size();
 
-        if (haystack_size != needle_offset.size() || haystack_size != res.size())
+        assert(haystack_size == needle_offset.size());
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Function '{}' unexpectedly received a different number of haystacks, needles and results", name);
+        assert(haystack_size == res.size());
 
-        if (start_pos_ != nullptr)
+        assert(start_pos_ == nullptr);
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' doesn't support start_pos argument", name);
 
         if (haystack_offsets.empty())
             return;
@@ -454,6 +448,9 @@ struct MatchImpl
         size_t prev_haystack_offset = 0;
         size_t prev_needle_offset = 0;
 
+        Regexps::LocalCacheTable cache;
+        Regexps::RegexpPtr regexp;
+
         for (size_t i = 0; i < haystack_size; ++i)
         {
             const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset];
@@ -479,22 +476,19 @@ struct MatchImpl
             }
             else
             {
-                // each row is expected to contain a different like/re2 pattern
+                cache.getOrSet<is_like, /*no_capture*/ true, case_insensitive>(needle, regexp);
-                // --> bypass the regexp cache, instead construct the pattern on-the-fly
-                const int flags = Regexps::buildRe2Flags</*no_capture*/ true, case_insensitive>();
-                const auto & regexp = Regexps::Regexp(Regexps::createRegexp<is_like>(needle, flags));
 
-                regexp.getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix);
+                regexp->getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix);
 
                 if (required_substr.empty())
                 {
-                    if (!regexp.getRE2()) /// An empty regexp. Always matches.
+                    if (!regexp->getRE2()) /// An empty regexp. Always matches.
                     {
                         res[i] = !negate;
                     }
                     else
                     {
-                        const bool match = regexp.getRE2()->Match(
+                        const bool match = regexp->getRE2()->Match(
                                 {reinterpret_cast<const char *>(cur_haystack_data), cur_haystack_length},
                                 0,
                                 cur_haystack_length,
@@ -524,7 +518,7 @@ struct MatchImpl
                             const size_t start_pos = (required_substring_is_prefix) ? (match - cur_haystack_data) : 0;
                             const size_t end_pos = cur_haystack_length;
 
-                            const bool match2 = regexp.getRE2()->Match(
+                            const bool match2 = regexp->getRE2()->Match(
                                     {reinterpret_cast<const char *>(cur_haystack_data), cur_haystack_length},
                                     start_pos,
                                     end_pos,
@@ -547,16 +541,15 @@ struct MatchImpl
         size_t N,
         const ColumnString::Chars & needle_data,
         const ColumnString::Offsets & needle_offset,
-        const ColumnPtr & start_pos_,
+        [[maybe_unused]] const ColumnPtr & start_pos_,
         PaddedPODArray<UInt8> & res)
     {
         const size_t haystack_size = haystack.size()/N;
 
-        if (haystack_size != needle_offset.size() || haystack_size != res.size())
+        assert(haystack_size == needle_offset.size());
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Function '{}' unexpectedly received a different number of haystacks, needles and results", name);
+        assert(haystack_size == res.size());
 
-        if (start_pos_ != nullptr)
+        assert(start_pos_ == nullptr);
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' doesn't support start_pos argument", name);
 
         if (haystack.empty())
             return;
@@ -568,6 +561,9 @@ struct MatchImpl
         size_t prev_haystack_offset = 0;
         size_t prev_needle_offset = 0;
 
+        Regexps::LocalCacheTable cache;
+        Regexps::RegexpPtr regexp;
+
         for (size_t i = 0; i < haystack_size; ++i)
         {
             const auto * const cur_haystack_data = &haystack[prev_haystack_offset];
@@ -593,22 +589,19 @@ struct MatchImpl
             }
             else
             {
-                // each row is expected to contain a different like/re2 pattern
+                cache.getOrSet<is_like, /*no_capture*/ true, case_insensitive>(needle, regexp);
-                // --> bypass the regexp cache, instead construct the pattern on-the-fly
-                const int flags = Regexps::buildRe2Flags</*no_capture*/ true, case_insensitive>();
-                const auto & regexp = Regexps::Regexp(Regexps::createRegexp<is_like>(needle, flags));
 
-                regexp.getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix);
+                regexp->getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix);
 
                 if (required_substr.empty())
                 {
-                    if (!regexp.getRE2()) /// An empty regexp. Always matches.
+                    if (!regexp->getRE2()) /// An empty regexp. Always matches.
                     {
                         res[i] = !negate;
                     }
                     else
                     {
-                        const bool match = regexp.getRE2()->Match(
+                        const bool match = regexp->getRE2()->Match(
                                 {reinterpret_cast<const char *>(cur_haystack_data), cur_haystack_length},
                                 0,
                                 cur_haystack_length,
@@ -638,7 +631,7 @@ struct MatchImpl
                             const size_t start_pos = (required_substring_is_prefix) ? (match - cur_haystack_data) : 0;
                             const size_t end_pos = cur_haystack_length;
 
-                            const bool match2 = regexp.getRE2()->Match(
+                            const bool match2 = regexp->getRE2()->Match(
                                     {reinterpret_cast<const char *>(cur_haystack_data), cur_haystack_length},
                                     start_pos,
                                     end_pos,

src/Functions/Regexps.h

@@ -9,7 +9,6 @@
 #include <vector>
 #include <Functions/likePatternToRegexp.h>
 #include <Common/Exception.h>
-#include <Common/ObjectPool.h>
 #include <Common/OptimizedRegularExpression.h>
 #include <Common/ProfileEvents.h>
 #include <Common/config.h>
@@ -38,254 +37,283 @@ namespace ErrorCodes
 
 namespace Regexps
 {
-    using Regexp = OptimizedRegularExpressionSingleThreaded;
+using Regexp = OptimizedRegularExpressionSingleThreaded;
-    using Pool = ObjectPoolMap<Regexp, String>;
+using RegexpPtr = std::shared_ptr<Regexp>;
 
-    template <bool like>
+template <bool like, bool no_capture, bool case_insensitive>
-    inline Regexp createRegexp(const std::string & pattern, int flags)
+inline Regexp createRegexp(const std::string & pattern)
+{
+    int flags = OptimizedRegularExpression::RE_DOT_NL;
+    if constexpr (no_capture)
+        flags |= OptimizedRegularExpression::RE_NO_CAPTURE;
+    if constexpr (case_insensitive)
+        flags |= OptimizedRegularExpression::RE_CASELESS;
+
+    if constexpr (like)
+        return {likePatternToRegexp(pattern), flags};
+    else
+        return {pattern, flags};
+}
+
+/// Caches compiled re2 objects for given string patterns. Intended to support the common situation of a small set of patterns which are
+/// evaluated over and over within the same query. In these situations, usage of the cache will save unnecessary pattern re-compilation.
+/// However, we must be careful that caching does not add too much static overhead to overall pattern evaluation. Therefore, the cache is
+/// intentionally very lightweight: a) no thread-safety/mutexes, b) small & fixed capacity, c) no collision list, d) but also no open
+/// addressing, instead collisions simply replace the existing element.
+class LocalCacheTable
+{
+public:
+    using RegexpPtr = std::shared_ptr<Regexp>;
+
+    LocalCacheTable()
+        : known_regexps(max_regexp_cache_size, {"", nullptr})
     {
-        if constexpr (like)
-            return {likePatternToRegexp(pattern), flags};
-        else
-            return {pattern, flags};
     }
 
-    template<bool no_capture, bool case_insensitive>
-    inline int buildRe2Flags()
-    {
-        int flags = OptimizedRegularExpression::RE_DOT_NL;
-        if constexpr (no_capture)
-            flags |= OptimizedRegularExpression::RE_NO_CAPTURE;
-        if constexpr (case_insensitive)
-            flags |= OptimizedRegularExpression::RE_CASELESS;
-        return flags;
-    }
-
-    /** Returns holder of an object from Pool.
-      * You must hold the ownership while using the object.
-      * In destructor, it returns the object back to the Pool for further reuse.
-      */
     template <bool like, bool no_capture, bool case_insensitive>
-    inline Pool::Pointer get(const std::string & pattern)
+    void getOrSet(const String & pattern, RegexpPtr & regexp)
     {
-        /// the Singleton is thread-safe in C++11
+        StringAndRegexp & bucket = known_regexps[hasher(pattern) % max_regexp_cache_size];
-        static Pool known_regexps; /// Different variables for different pattern parameters.
 
-        return known_regexps.get(pattern, [&pattern]
+        if (likely(bucket.regexp != nullptr))
         {
-            const int flags = buildRe2Flags<no_capture, case_insensitive>();
+            if (pattern == bucket.pattern)
-            ProfileEvents::increment(ProfileEvents::RegexpCreated);
+                regexp = bucket.regexp;
-            return new Regexp{createRegexp<like>(pattern, flags)};
+            else
-        });
+            {
+                regexp = std::make_shared<Regexp>(createRegexp<like, no_capture, case_insensitive>(pattern));
+                bucket = {pattern, regexp};
+            }
+        }
+        else
+        {
+            regexp = std::make_shared<Regexp>(createRegexp<like, no_capture, case_insensitive>(pattern));
+            bucket = {pattern, regexp};
+        }
     }
+
+private:
+    std::hash<std::string> hasher;
+    struct StringAndRegexp
+    {
+        std::string pattern;
+        RegexpPtr regexp;
+    };
+    using CacheTable = std::vector<StringAndRegexp>;
+    CacheTable known_regexps;
+
+    constexpr static size_t max_regexp_cache_size = 100; // collision probability
+};
+
 }
 
 #if USE_HYPERSCAN
 
 namespace MultiRegexps
 {
-    template <typename Deleter, Deleter deleter>
+template <typename Deleter, Deleter deleter>
-    struct HyperscanDeleter
+struct HyperscanDeleter
+{
+    template <typename T>
+    void operator()(T * ptr) const
     {
-        template <typename T>
+        deleter(ptr);
-        void operator()(T * ptr) const
+    }
-        {
+};
-            deleter(ptr);
-        }
-    };
 
-    /// Helper unique pointers to correctly delete the allocated space when hyperscan cannot compile something and we throw an exception.
+/// Helper unique pointers to correctly delete the allocated space when hyperscan cannot compile something and we throw an exception.
-    using CompilerError = std::unique_ptr<hs_compile_error_t, HyperscanDeleter<decltype(&hs_free_compile_error), &hs_free_compile_error>>;
+using CompilerError = std::unique_ptr<hs_compile_error_t, HyperscanDeleter<decltype(&hs_free_compile_error), &hs_free_compile_error>>;
-    using ScratchPtr = std::unique_ptr<hs_scratch_t, HyperscanDeleter<decltype(&hs_free_scratch), &hs_free_scratch>>;
+using ScratchPtr = std::unique_ptr<hs_scratch_t, HyperscanDeleter<decltype(&hs_free_scratch), &hs_free_scratch>>;
-    using DataBasePtr = std::unique_ptr<hs_database_t, HyperscanDeleter<decltype(&hs_free_database), &hs_free_database>>;
+using DataBasePtr = std::unique_ptr<hs_database_t, HyperscanDeleter<decltype(&hs_free_database), &hs_free_database>>;
 
-    /// Database is thread safe across multiple threads and Scratch is not but we can copy it whenever we use it in the searcher.
+/// Database is thread safe across multiple threads and Scratch is not but we can copy it whenever we use it in the searcher.
-    class Regexps
+class Regexps
+{
+public:
+    Regexps(hs_database_t * db_, hs_scratch_t * scratch_) : db{db_}, scratch{scratch_} { }
+
+    hs_database_t * getDB() const { return db.get(); }
+    hs_scratch_t * getScratch() const { return scratch.get(); }
+
+private:
+    DataBasePtr db;
+    ScratchPtr scratch;
+};
+
+class RegexpsConstructor
+{
+public:
+    RegexpsConstructor() = default;
+
+    void setConstructor(std::function<Regexps()> constructor_) { constructor = std::move(constructor_); }
+
+    Regexps * operator()()
     {
-    public:
+        std::unique_lock lock(mutex);
-        Regexps(hs_database_t * db_, hs_scratch_t * scratch_) : db{db_}, scratch{scratch_} { }
+        if (regexp)
-
-        hs_database_t * getDB() const { return db.get(); }
-        hs_scratch_t * getScratch() const { return scratch.get(); }
-
-    private:
-        DataBasePtr db;
-        ScratchPtr scratch;
-    };
-
-    class RegexpsConstructor
-    {
-    public:
-        RegexpsConstructor() = default;
-
-        void setConstructor(std::function<Regexps()> constructor_) { constructor = std::move(constructor_); }
-
-        Regexps * operator()()
-        {
-            std::unique_lock lock(mutex);
-            if (regexp)
-                return &*regexp;
-            regexp = constructor();
             return &*regexp;
-        }
+        regexp = constructor();
+        return &*regexp;
+    }
 
-    private:
+private:
-        std::function<Regexps()> constructor;
+    std::function<Regexps()> constructor;
-        std::optional<Regexps> regexp;
+    std::optional<Regexps> regexp;
-        std::mutex mutex;
+    std::mutex mutex;
-    };
+};
 
-    struct Pool
+struct Pool
+{
+    /// Mutex for finding in map.
+    std::mutex mutex;
+    /// Patterns + possible edit_distance to database and scratch.
+    std::map<std::pair<std::vector<String>, std::optional<UInt32>>, RegexpsConstructor> storage;
+};
+
+template <bool save_indices, bool CompileForEditDistance>
+inline Regexps constructRegexps(const std::vector<String> & str_patterns, std::optional<UInt32> edit_distance)
+{
+    (void)edit_distance;
+    /// Common pointers
+    std::vector<const char *> patterns;
+    std::vector<unsigned int> flags;
+
+    /// Pointer for external edit distance compilation
+    std::vector<hs_expr_ext> ext_exprs;
+    std::vector<const hs_expr_ext *> ext_exprs_ptrs;
+
+    patterns.reserve(str_patterns.size());
+    flags.reserve(str_patterns.size());
+
+    if constexpr (CompileForEditDistance)
     {
-        /// Mutex for finding in map.
+        ext_exprs.reserve(str_patterns.size());
-        std::mutex mutex;
+        ext_exprs_ptrs.reserve(str_patterns.size());
-        /// Patterns + possible edit_distance to database and scratch.
+    }
-        std::map<std::pair<std::vector<String>, std::optional<UInt32>>, RegexpsConstructor> storage;
-    };
 
-    template <bool save_indices, bool CompileForEditDistance>
+    for (const StringRef ref : str_patterns)
-    inline Regexps constructRegexps(const std::vector<String> & str_patterns, std::optional<UInt32> edit_distance)
     {
-        (void)edit_distance;
+        patterns.push_back(ref.data);
-        /// Common pointers
+        /* Flags below are the pattern matching flags.
-        std::vector<const char *> patterns;
+         * HS_FLAG_DOTALL is a compile flag where matching a . will not exclude newlines. This is a good
-        std::vector<unsigned int> flags;
+         * performance practice according to Hyperscan API. https://intel.github.io/hyperscan/dev-reference/performance.html#dot-all-mode
-
+         * HS_FLAG_ALLOWEMPTY is a compile flag where empty strings are allowed to match.
-        /// Pointer for external edit distance compilation
+         * HS_FLAG_UTF8 is a flag where UTF8 literals are matched.
-        std::vector<hs_expr_ext> ext_exprs;
+         * HS_FLAG_SINGLEMATCH is a compile flag where each pattern match will be returned only once. it is a good performance practice
-        std::vector<const hs_expr_ext *> ext_exprs_ptrs;
+         * as it is said in the Hyperscan documentation. https://intel.github.io/hyperscan/dev-reference/performance.html#single-match-flag
-
+         */
-        patterns.reserve(str_patterns.size());
+        flags.push_back(HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH | HS_FLAG_ALLOWEMPTY | HS_FLAG_UTF8);
-        flags.reserve(str_patterns.size());
-
         if constexpr (CompileForEditDistance)
         {
-            ext_exprs.reserve(str_patterns.size());
+            /// Hyperscan currently does not support UTF8 matching with edit distance.
-            ext_exprs_ptrs.reserve(str_patterns.size());
+            flags.back() &= ~HS_FLAG_UTF8;
+            ext_exprs.emplace_back();
+            /// HS_EXT_FLAG_EDIT_DISTANCE is a compile flag responsible for Levenstein distance.
+            ext_exprs.back().flags = HS_EXT_FLAG_EDIT_DISTANCE;
+            ext_exprs.back().edit_distance = edit_distance.value();
+            ext_exprs_ptrs.push_back(&ext_exprs.back());
         }
-
-        for (const StringRef ref : str_patterns)
-        {
-            patterns.push_back(ref.data);
-            /* Flags below are the pattern matching flags.
-             * HS_FLAG_DOTALL is a compile flag where matching a . will not exclude newlines. This is a good
-             * performance practice according to Hyperscan API. https://intel.github.io/hyperscan/dev-reference/performance.html#dot-all-mode
-             * HS_FLAG_ALLOWEMPTY is a compile flag where empty strings are allowed to match.
-             * HS_FLAG_UTF8 is a flag where UTF8 literals are matched.
-             * HS_FLAG_SINGLEMATCH is a compile flag where each pattern match will be returned only once. it is a good performance practice
-             * as it is said in the Hyperscan documentation. https://intel.github.io/hyperscan/dev-reference/performance.html#single-match-flag
-             */
-            flags.push_back(HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH | HS_FLAG_ALLOWEMPTY | HS_FLAG_UTF8);
-            if constexpr (CompileForEditDistance)
-            {
-                /// Hyperscan currently does not support UTF8 matching with edit distance.
-                flags.back() &= ~HS_FLAG_UTF8;
-                ext_exprs.emplace_back();
-                /// HS_EXT_FLAG_EDIT_DISTANCE is a compile flag responsible for Levenstein distance.
-                ext_exprs.back().flags = HS_EXT_FLAG_EDIT_DISTANCE;
-                ext_exprs.back().edit_distance = edit_distance.value();
-                ext_exprs_ptrs.push_back(&ext_exprs.back());
-            }
-        }
-        hs_database_t * db = nullptr;
-        hs_compile_error_t * compile_error;
-
-        std::unique_ptr<unsigned int[]> ids;
-
-        /// We mark the patterns to provide the callback results.
-        if constexpr (save_indices)
-        {
-            ids.reset(new unsigned int[patterns.size()]);
-            for (size_t i = 0; i < patterns.size(); ++i)
-                ids[i] = i + 1;
-        }
-
-        hs_error_t err;
-        if constexpr (!CompileForEditDistance)
-            err = hs_compile_multi(
-                patterns.data(),
-                flags.data(),
-                ids.get(),
-                patterns.size(),
-                HS_MODE_BLOCK,
-                nullptr,
-                &db,
-                &compile_error);
-        else
-            err = hs_compile_ext_multi(
-                patterns.data(),
-                flags.data(),
-                ids.get(),
-                ext_exprs_ptrs.data(),
-                patterns.size(),
-                HS_MODE_BLOCK,
-                nullptr,
-                &db,
-                &compile_error);
-
-        if (err != HS_SUCCESS)
-        {
-            /// CompilerError is a unique_ptr, so correct memory free after the exception is thrown.
-            CompilerError error(compile_error);
-
-            if (error->expression < 0)
-                throw Exception(String(error->message), ErrorCodes::LOGICAL_ERROR);
-            else
-                throw Exception(
-                    "Pattern '" + str_patterns[error->expression] + "' failed with error '" + String(error->message),
-                    ErrorCodes::BAD_ARGUMENTS);
-        }
-
-        ProfileEvents::increment(ProfileEvents::RegexpCreated);
-
-        /// We allocate the scratch space only once, then copy it across multiple threads with hs_clone_scratch
-        /// function which is faster than allocating scratch space each time in each thread.
-        hs_scratch_t * scratch = nullptr;
-        err = hs_alloc_scratch(db, &scratch);
-
-        /// If not HS_SUCCESS, it is guaranteed that the memory would not be allocated for scratch.
-        if (err != HS_SUCCESS)
-            throw Exception("Could not allocate scratch space for hyperscan", ErrorCodes::CANNOT_ALLOCATE_MEMORY);
-
-        return Regexps{db, scratch};
     }
+    hs_database_t * db = nullptr;
+    hs_compile_error_t * compile_error;
 
-    /// If CompileForEditDistance is False, edit_distance must be nullopt
+    std::unique_ptr<unsigned int[]> ids;
-    /// Also, we use templates here because each instantiation of function
+
-    /// template has its own copy of local static variables which must not be the same
+    /// We mark the patterns to provide the callback results.
-    /// for different hyperscan compilations.
+    if constexpr (save_indices)
-    template <bool save_indices, bool CompileForEditDistance>
-    inline Regexps * get(const std::vector<StringRef> & patterns, std::optional<UInt32> edit_distance)
     {
-        /// C++11 has thread-safe function-local static on most modern compilers.
+        ids.reset(new unsigned int[patterns.size()]);
-        static Pool known_regexps; /// Different variables for different pattern parameters.
+        for (size_t i = 0; i < patterns.size(); ++i)
-
+            ids[i] = i + 1;
-        std::vector<String> str_patterns;
-        str_patterns.reserve(patterns.size());
-        for (const StringRef & ref : patterns)
-            str_patterns.push_back(ref.toString());
-
-        /// Get the lock for finding database.
-        std::unique_lock lock(known_regexps.mutex);
-
-        auto it = known_regexps.storage.find({str_patterns, edit_distance});
-
-        /// If not found, compile and let other threads wait.
-        if (known_regexps.storage.end() == it)
-        {
-            it = known_regexps.storage
-                     .emplace(std::piecewise_construct, std::make_tuple(std::move(str_patterns), edit_distance), std::make_tuple())
-                     .first;
-            it->second.setConstructor([&str_patterns = it->first.first, edit_distance]()
-            {
-                return constructRegexps<save_indices, CompileForEditDistance>(str_patterns, edit_distance);
-            });
-        }
-
-        /// Unlock before possible construction.
-        lock.unlock();
-        return it->second();
     }
+
+    hs_error_t err;
+    if constexpr (!CompileForEditDistance)
+        err = hs_compile_multi(
+            patterns.data(),
+            flags.data(),
+            ids.get(),
+            patterns.size(),
+            HS_MODE_BLOCK,
+            nullptr,
+            &db,
+            &compile_error);
+    else
+        err = hs_compile_ext_multi(
+            patterns.data(),
+            flags.data(),
+            ids.get(),
+            ext_exprs_ptrs.data(),
+            patterns.size(),
+            HS_MODE_BLOCK,
+            nullptr,
+            &db,
+            &compile_error);
+
+    if (err != HS_SUCCESS)
+    {
+        /// CompilerError is a unique_ptr, so correct memory free after the exception is thrown.
+        CompilerError error(compile_error);
+
+        if (error->expression < 0)
+            throw Exception(String(error->message), ErrorCodes::LOGICAL_ERROR);
+        else
+            throw Exception(
+                "Pattern '" + str_patterns[error->expression] + "' failed with error '" + String(error->message),
+                ErrorCodes::BAD_ARGUMENTS);
+    }
+
+    ProfileEvents::increment(ProfileEvents::RegexpCreated);
+
+    /// We allocate the scratch space only once, then copy it across multiple threads with hs_clone_scratch
+    /// function which is faster than allocating scratch space each time in each thread.
+    hs_scratch_t * scratch = nullptr;
+    err = hs_alloc_scratch(db, &scratch);
+
+    /// If not HS_SUCCESS, it is guaranteed that the memory would not be allocated for scratch.
+    if (err != HS_SUCCESS)
+        throw Exception("Could not allocate scratch space for hyperscan", ErrorCodes::CANNOT_ALLOCATE_MEMORY);
+
+    return Regexps{db, scratch};
+}
+
+/// If CompileForEditDistance is False, edit_distance must be nullopt
+/// Also, we use templates here because each instantiation of function
+/// template has its own copy of local static variables which must not be the same
+/// for different hyperscan compilations.
+template <bool save_indices, bool CompileForEditDistance>
+inline Regexps * get(const std::vector<StringRef> & patterns, std::optional<UInt32> edit_distance)
+{
+    /// C++11 has thread-safe function-local static on most modern compilers.
+    static Pool known_regexps; /// Different variables for different pattern parameters.
+
+    std::vector<String> str_patterns;
+    str_patterns.reserve(patterns.size());
+    for (const StringRef & ref : patterns)
+        str_patterns.push_back(ref.toString());
+
+    /// Get the lock for finding database.
+    std::unique_lock lock(known_regexps.mutex);
+
+    auto it = known_regexps.storage.find({str_patterns, edit_distance});
+
+    /// If not found, compile and let other threads wait.
+    if (known_regexps.storage.end() == it)
+    {
+        it = known_regexps.storage
+                 .emplace(std::piecewise_construct, std::make_tuple(std::move(str_patterns), edit_distance), std::make_tuple())
+                 .first;
+        it->second.setConstructor([&str_patterns = it->first.first, edit_distance]()
+        {
+            return constructRegexps<save_indices, CompileForEditDistance>(str_patterns, edit_distance);
+        });
+    }
+
+    /// Unlock before possible construction.
+    lock.unlock();
+    return it->second();
+}
+
 }
 
 #endif // USE_HYPERSCAN

src/Functions/countMatches.h

@@ -55,7 +55,7 @@ public:
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
         const ColumnConst * column_pattern = checkAndGetColumnConstStringOrFixedString(arguments[1].column.get());
-        Regexps::Pool::Pointer re = Regexps::get<false /* like */, true /* is_no_capture */, CountMatchesBase::case_insensitive>(column_pattern->getValue<String>());
+        const Regexps::Regexp re = Regexps::createRegexp</*is_like*/ false, /*no_capture*/ true, CountMatchesBase::case_insensitive>(column_pattern->getValue<String>());
         OptimizedRegularExpression::MatchVec matches;
 
         const IColumn * column_haystack = arguments[0].column.get();
@@ -95,7 +95,7 @@ public:
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Error in FunctionCountMatches::getReturnTypeImpl()");
     }
 
-    static uint64_t countMatches(StringRef src, Regexps::Pool::Pointer & re, OptimizedRegularExpression::MatchVec & matches)
+    static uint64_t countMatches(StringRef src, const Regexps::Regexp & re, OptimizedRegularExpression::MatchVec & matches)
     {
         /// Only one match is required, no need to copy more.
         static const unsigned matches_limit = 1;
@@ -108,7 +108,7 @@ public:
         {
             if (pos >= end)
                 break;
-            if (!re->match(pos, end - pos, matches, matches_limit))
+            if (!re.match(pos, end - pos, matches, matches_limit))
                 break;
             /// Progress should be made, but with empty match the progress will not be done.
             /// Also note that simply check is pattern empty is not enough,

src/Functions/extract.cpp

@@ -21,9 +21,9 @@ struct ExtractImpl
         res_data.reserve(data.size() / 5);
         res_offsets.resize(offsets.size());
 
-        const auto & regexp = Regexps::get<false, false, false>(pattern);
+        const Regexps::Regexp regexp = Regexps::createRegexp<false, false, false>(pattern);
 
-        unsigned capture = regexp->getNumberOfSubpatterns() > 0 ? 1 : 0;
+        unsigned capture = regexp.getNumberOfSubpatterns() > 0 ? 1 : 0;
         OptimizedRegularExpression::MatchVec matches;
         matches.reserve(capture + 1);
         size_t prev_offset = 0;
@@ -34,7 +34,7 @@ struct ExtractImpl
             size_t cur_offset = offsets[i];
 
             unsigned count
-                = regexp->match(reinterpret_cast<const char *>(&data[prev_offset]), cur_offset - prev_offset - 1, matches, capture + 1);
+                = regexp.match(reinterpret_cast<const char *>(&data[prev_offset]), cur_offset - prev_offset - 1, matches, capture + 1);
             if (count > capture && matches[capture].offset != std::string::npos)
             {
                 const auto & match = matches[capture];

src/Functions/extractAllGroups.h

@@ -95,8 +95,8 @@ public:
             throw Exception("Length of 'needle' argument must be greater than 0.", ErrorCodes::BAD_ARGUMENTS);
 
         using StringPiece = typename Regexps::Regexp::StringPieceType;
-        auto holder = Regexps::get<false, false, false>(needle);
+        const Regexps::Regexp holder = Regexps::createRegexp<false, false, false>(needle);
-        const auto & regexp = holder->getRE2();
+        const auto & regexp = holder.getRE2();
 
         if (!regexp)
             throw Exception("There are no groups in regexp: " + needle, ErrorCodes::BAD_ARGUMENTS);

src/Functions/extractGroups.cpp

@@ -63,8 +63,8 @@ public:
         if (needle.empty())
             throw Exception(getName() + " length of 'needle' argument must be greater than 0.", ErrorCodes::BAD_ARGUMENTS);
 
-        auto regexp = Regexps::get<false, false, false>(needle);
+        const Regexps::Regexp regexp = Regexps::createRegexp<false, false, false>(needle);
-        const auto & re2 = regexp->getRE2();
+        const auto & re2 = regexp.getRE2();
 
         if (!re2)
             throw Exception("There are no groups in regexp: " + needle, ErrorCodes::BAD_ARGUMENTS);

src/IO/ParallelReadBuffer.cpp

@@ -49,7 +49,15 @@ ParallelReadBuffer::ParallelReadBuffer(
     , schedule(std::move(schedule_))
     , reader_factory(std::move(reader_factory_))
 {
-    addReaders();
+    try
+    {
+        addReaders();
+    }
+    catch (const Exception &)
+    {
+        finishAndWait();
+        throw;
+    }
 }
 
 bool ParallelReadBuffer::addReaderToPool()

src/IO/S3Common.cpp

@@ -765,9 +765,9 @@ namespace S3
         const String & force_region,
         const RemoteHostFilter & remote_host_filter,
         unsigned int s3_max_redirects,
-        bool enable_s3_requestrs_logging)
+        bool enable_s3_requests_logging)
     {
-        return PocoHTTPClientConfiguration(force_region, remote_host_filter, s3_max_redirects, enable_s3_requestrs_logging);
+        return PocoHTTPClientConfiguration(force_region, remote_host_filter, s3_max_redirects, enable_s3_requests_logging);
     }
 
     URI::URI(const Poco::URI & uri_)

src/IO/S3Common.h

@@ -45,7 +45,7 @@ public:
         const String & force_region,
         const RemoteHostFilter & remote_host_filter,
         unsigned int s3_max_redirects,
-        bool enable_s3_requestrs_logging);
+        bool enable_s3_requests_logging);
 
 private:
     ClientFactory();

src/Interpreters/ActionsDAG.cpp

@@ -39,7 +39,7 @@ void ActionsDAG::Node::toTree(JSONBuilder::JSONMap & map) const
         map.add("Result Type", result_type->getName());
 
     if (!result_name.empty())
-        map.add("Result Type", magic_enum::enum_name(type));
+        map.add("Result Name", result_name);
 
     if (column)
         map.add("Column", column->getName());

src/Interpreters/Cluster.cpp

@@ -418,6 +418,8 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config,
             if (address.is_local)
                 info.local_addresses.push_back(address);
 
+            info.all_addresses.push_back(address);
+
             auto pool = ConnectionPoolFactory::instance().get(
                 settings.distributed_connections_pool_size,
                 address.host_name, address.port,
@@ -485,6 +487,7 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config,
             }
 
             Addresses shard_local_addresses;
+            Addresses shard_all_addresses;
 
             ConnectionPoolPtrs all_replicas_pools;
             all_replicas_pools.reserve(replica_addresses.size());
@@ -502,6 +505,7 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config,
                 all_replicas_pools.emplace_back(replica_pool);
                 if (replica.is_local)
                     shard_local_addresses.push_back(replica);
+                shard_all_addresses.push_back(replica);
             }
 
             ConnectionPoolWithFailoverPtr shard_pool = std::make_shared<ConnectionPoolWithFailover>(
@@ -516,6 +520,7 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config,
                 current_shard_num,
                 weight,
                 std::move(shard_local_addresses),
+                std::move(shard_all_addresses),
                 std::move(shard_pool),
                 std::move(all_replicas_pools),
                 internal_replication
@@ -571,6 +576,7 @@ Cluster::Cluster(
         addresses_with_failover.emplace_back(current);
 
         Addresses shard_local_addresses;
+        Addresses all_addresses;
         ConnectionPoolPtrs all_replicas;
         all_replicas.reserve(current.size());
 
@@ -585,6 +591,7 @@ Cluster::Cluster(
             all_replicas.emplace_back(replica_pool);
             if (replica.is_local && !treat_local_as_remote)
                 shard_local_addresses.push_back(replica);
+            all_addresses.push_back(replica);
         }
 
         ConnectionPoolWithFailoverPtr shard_pool = std::make_shared<ConnectionPoolWithFailover>(
@@ -597,6 +604,7 @@ Cluster::Cluster(
             current_shard_num,
             default_weight,
             std::move(shard_local_addresses),
+            std::move(all_addresses),
             std::move(shard_pool),
             std::move(all_replicas),
             false // has_internal_replication
@@ -680,6 +688,8 @@ Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Setti
             if (address.is_local)
                 info.local_addresses.push_back(address);
 
+            info.all_addresses.push_back(address);
+
             auto pool = ConnectionPoolFactory::instance().get(
                 settings.distributed_connections_pool_size,
                 address.host_name,

src/Interpreters/Cluster.h

@@ -202,6 +202,7 @@ public:
         UInt32 shard_num = 0;
         UInt32 weight = 1;
         Addresses local_addresses;
+        Addresses all_addresses;
         /// nullptr if there are no remote addresses
         ConnectionPoolWithFailoverPtr pool;
         /// Connection pool for each replica, contains nullptr for local replicas

src/Interpreters/ClusterProxy/IStreamFactory.h

@@ -1,65 +0,0 @@
-#pragma once
-
-#include <Client/ConnectionPool.h>
-#include <Interpreters/Cluster.h>
-#include <Parsers/IAST.h>
-
-namespace DB
-{
-
-struct Settings;
-class Cluster;
-class Throttler;
-struct SelectQueryInfo;
-
-class Pipe;
-using Pipes = std::vector<Pipe>;
-
-class QueryPlan;
-using QueryPlanPtr = std::unique_ptr<QueryPlan>;
-
-struct StorageID;
-
-namespace ClusterProxy
-{
-
-/// Base class for the implementation of the details of distributed query
-/// execution that are specific to the query type.
-class IStreamFactory
-{
-public:
-    virtual ~IStreamFactory() = default;
-
-    struct Shard
-    {
-        /// Query and header may be changed depending on shard.
-        ASTPtr query;
-        Block header;
-
-        size_t shard_num = 0;
-        size_t num_replicas = 0;
-        ConnectionPoolWithFailoverPtr pool;
-        ConnectionPoolPtrs per_replica_pools;
-
-        /// If we connect to replicas lazily.
-        /// (When there is a local replica with big delay).
-        bool lazy = false;
-        UInt32 local_delay = 0;
-    };
-
-    using Shards = std::vector<Shard>;
-
-    virtual void createForShard(
-            const Cluster::ShardInfo & shard_info,
-            const ASTPtr & query_ast,
-            const StorageID & main_table,
-            const ASTPtr & table_func_ptr,
-            ContextPtr context,
-            std::vector<QueryPlanPtr> & local_plans,
-            Shards & remote_shards,
-            UInt32 shard_count) = 0;
-};
-
-}
-
-}

src/Interpreters/ClusterProxy/SelectStreamFactory.cpp

@@ -1,4 +1,5 @@
 #include <Interpreters/ClusterProxy/SelectStreamFactory.h>
+#include <Interpreters/Cluster.h>
 #include <Interpreters/InterpreterSelectQuery.h>
 #include <Storages/StorageReplicatedMergeTree.h>
 #include <Storages/VirtualColumnUtils.h>
@@ -10,14 +11,15 @@
 #include <Interpreters/RequiredSourceColumnsVisitor.h>
 #include <DataTypes/ObjectUtils.h>
 
+#include <Client/IConnections.h>
 #include <Common/logger_useful.h>
 #include <Processors/QueryPlan/QueryPlan.h>
+#include <Processors/QueryPlan/ReadFromRemote.h>
 #include <Processors/QueryPlan/ExpressionStep.h>
 #include <Processors/QueryPlan/BuildQueryPipelineSettings.h>
 #include <Processors/QueryPlan/DistributedCreateLocalPlan.h>
 #include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
 
-
 namespace ProfileEvents
 {
     extern const Event DistributedConnectionMissingTable;
@@ -63,7 +65,8 @@ void SelectStreamFactory::createForShard(
 
     auto emplace_local_stream = [&]()
     {
-        local_plans.emplace_back(createLocalPlan(query_ast, header, context, processed_stage, shard_info.shard_num, shard_count, /*coordinator=*/nullptr));
+        local_plans.emplace_back(createLocalPlan(
+            query_ast, header, context, processed_stage, shard_info.shard_num, shard_count, /*replica_num=*/0, /*replica_count=*/0, /*coordinator=*/nullptr));
     };
 
     auto emplace_remote_stream = [&](bool lazy = false, UInt32 local_delay = 0)
@@ -71,10 +74,7 @@ void SelectStreamFactory::createForShard(
         remote_shards.emplace_back(Shard{
             .query = query_ast,
             .header = header,
-            .shard_num = shard_info.shard_num,
+            .shard_info = shard_info,
-            .num_replicas = shard_info.getAllNodeCount(),
-            .pool = shard_info.pool,
-            .per_replica_pools = shard_info.per_replica_pools,
             .lazy = lazy,
             .local_delay = local_delay,
         });
@@ -173,5 +173,97 @@ void SelectStreamFactory::createForShard(
         emplace_remote_stream();
 }
 
+
+SelectStreamFactory::ShardPlans SelectStreamFactory::createForShardWithParallelReplicas(
+    const Cluster::ShardInfo & shard_info,
+    const ASTPtr & query_ast,
+    const StorageID & main_table,
+    const ASTPtr & table_function_ptr,
+    const ThrottlerPtr & throttler,
+    ContextPtr context,
+    UInt32 shard_count)
+{
+    SelectStreamFactory::ShardPlans result;
+
+    if (auto it = objects_by_shard.find(shard_info.shard_num); it != objects_by_shard.end())
+        replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, query_ast);
+
+    const auto & settings = context->getSettingsRef();
+
+    auto is_local_replica_obsolete = [&]()
+    {
+        auto resolved_id = context->resolveStorageID(main_table);
+        auto main_table_storage = DatabaseCatalog::instance().tryGetTable(resolved_id, context);
+        const auto * replicated_storage = dynamic_cast<const StorageReplicatedMergeTree *>(main_table_storage.get());
+
+        if (!replicated_storage)
+            return false;
+
+        UInt64 max_allowed_delay = settings.max_replica_delay_for_distributed_queries;
+
+        if (!max_allowed_delay)
+            return false;
+
+        UInt32 local_delay = replicated_storage->getAbsoluteDelay();
+        return local_delay >= max_allowed_delay;
+    };
+
+    size_t next_replica_number = 0;
+    size_t all_replicas_count = shard_info.getRemoteNodeCount();
+
+    auto coordinator = std::make_shared<ParallelReplicasReadingCoordinator>();
+    auto remote_plan = std::make_unique<QueryPlan>();
+
+
+    if (settings.prefer_localhost_replica && shard_info.isLocal())
+    {
+        /// We don't need more than one local replica in parallel reading
+        if (!is_local_replica_obsolete())
+        {
+            ++all_replicas_count;
+
+            result.local_plan = createLocalPlan(
+                query_ast, header, context, processed_stage, shard_info.shard_num, shard_count, next_replica_number, all_replicas_count, coordinator);
+
+            ++next_replica_number;
+        }
+    }
+
+    Scalars scalars = context->hasQueryContext() ? context->getQueryContext()->getScalars() : Scalars{};
+    scalars.emplace(
+        "_shard_count", Block{{DataTypeUInt32().createColumnConst(1, shard_count), std::make_shared<DataTypeUInt32>(), "_shard_count"}});
+    auto external_tables = context->getExternalTables();
+
+    auto shard = Shard{
+        .query = query_ast,
+        .header = header,
+        .shard_info = shard_info,
+        .lazy = false,
+        .local_delay = 0,
+    };
+
+    if (shard_info.hasRemoteConnections())
+    {
+        auto read_from_remote = std::make_unique<ReadFromParallelRemoteReplicasStep>(
+            coordinator,
+            shard,
+            header,
+            processed_stage,
+            main_table,
+            table_function_ptr,
+            context,
+            throttler,
+            std::move(scalars),
+            std::move(external_tables),
+            &Poco::Logger::get("ReadFromParallelRemoteReplicasStep"),
+            shard_count);
+
+        remote_plan->addStep(std::move(read_from_remote));
+        result.remote_plan = std::move(remote_plan);
+    }
+
+    return result;
+}
+
 }
 }

src/Interpreters/ClusterProxy/SelectStreamFactory.h

@@ -1,22 +1,56 @@
 #pragma once
 
 #include <Core/QueryProcessingStage.h>
-#include <Interpreters/ClusterProxy/IStreamFactory.h>
 #include <Interpreters/StorageID.h>
 #include <Storages/IStorage_fwd.h>
 #include <Storages/StorageSnapshot.h>
+#include <Client/ConnectionPool.h>
+#include <Interpreters/Cluster.h>
+#include <Parsers/IAST.h>
 
 namespace DB
 {
 
+
+struct Settings;
+class Cluster;
+class Throttler;
+struct SelectQueryInfo;
+
+class Pipe;
+using Pipes = std::vector<Pipe>;
+
+class QueryPlan;
+using QueryPlanPtr = std::unique_ptr<QueryPlan>;
+
+struct StorageID;
+
 namespace ClusterProxy
 {
 
+
 using ColumnsDescriptionByShardNum = std::unordered_map<UInt32, ColumnsDescription>;
 
-class SelectStreamFactory final : public IStreamFactory
+class SelectStreamFactory
 {
 public:
+
+    struct Shard
+    {
+        /// Query and header may be changed depending on shard.
+        ASTPtr query;
+        Block header;
+
+        Cluster::ShardInfo shard_info;
+
+        /// If we connect to replicas lazily.
+        /// (When there is a local replica with big delay).
+        bool lazy = false;
+        UInt32 local_delay = 0;
+    };
+
+    using Shards = std::vector<Shard>;
+
     SelectStreamFactory(
         const Block & header_,
         const ColumnsDescriptionByShardNum & objects_by_shard_,
@@ -31,7 +65,26 @@ public:
         ContextPtr context,
         std::vector<QueryPlanPtr> & local_plans,
         Shards & remote_shards,
-        UInt32 shard_count) override;
+        UInt32 shard_count);
+
+    struct ShardPlans
+    {
+        /// If a shard has local replicas this won't be nullptr
+        std::unique_ptr<QueryPlan> local_plan;
+
+        /// Contains several steps to read from all remote replicas
+        std::unique_ptr<QueryPlan> remote_plan;
+    };
+
+    ShardPlans createForShardWithParallelReplicas(
+        const Cluster::ShardInfo & shard_info,
+        const ASTPtr & query_ast,
+        const StorageID & main_table,
+        const ASTPtr & table_function_ptr,
+        const ThrottlerPtr & throttler,
+        ContextPtr context,
+        UInt32 shard_count
+    );
 
 private:
     const Block header;

src/Interpreters/ClusterProxy/executeQuery.cpp

@@ -20,6 +20,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int TOO_LARGE_DISTRIBUTED_DEPTH;
+    extern const int LOGICAL_ERROR;
 }
 
 namespace ClusterProxy
@@ -106,21 +107,19 @@ void executeQuery(
     QueryProcessingStage::Enum processed_stage,
     const StorageID & main_table,
     const ASTPtr & table_func_ptr,
-    IStreamFactory & stream_factory, Poco::Logger * log,
+    SelectStreamFactory & stream_factory, Poco::Logger * log,
     const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info,
     const ExpressionActionsPtr & sharding_key_expr,
     const std::string & sharding_key_column_name,
     const ClusterPtr & not_optimized_cluster)
 {
-    assert(log);
-
     const Settings & settings = context->getSettingsRef();
 
     if (settings.max_distributed_depth && context->getClientInfo().distributed_depth >= settings.max_distributed_depth)
         throw Exception("Maximum distributed depth exceeded", ErrorCodes::TOO_LARGE_DISTRIBUTED_DEPTH);
 
     std::vector<QueryPlanPtr> plans;
-    IStreamFactory::Shards remote_shards;
+    SelectStreamFactory::Shards remote_shards;
 
     auto new_context = updateSettingsForCluster(*query_info.getCluster(), context, settings, log);
 
@@ -213,6 +212,91 @@ void executeQuery(
     query_plan.unitePlans(std::move(union_step), std::move(plans));
 }
 
+
+void executeQueryWithParallelReplicas(
+    QueryPlan & query_plan,
+    const StorageID & main_table,
+    const ASTPtr & table_func_ptr,
+    SelectStreamFactory & stream_factory,
+    const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info,
+    const ExpressionActionsPtr & sharding_key_expr,
+    const std::string & sharding_key_column_name,
+    const ClusterPtr & not_optimized_cluster)
+{
+    const Settings & settings = context->getSettingsRef();
+
+    ThrottlerPtr user_level_throttler;
+    if (auto * process_list_element = context->getProcessListElement())
+        user_level_throttler = process_list_element->getUserNetworkThrottler();
+
+    /// Network bandwidth limit, if needed.
+    ThrottlerPtr throttler;
+    if (settings.max_network_bandwidth || settings.max_network_bytes)
+    {
+        throttler = std::make_shared<Throttler>(
+                settings.max_network_bandwidth,
+                settings.max_network_bytes,
+                "Limit for bytes to send or receive over network exceeded.",
+                user_level_throttler);
+    }
+    else
+        throttler = user_level_throttler;
+
+
+    std::vector<QueryPlanPtr> plans;
+    size_t shards = query_info.getCluster()->getShardCount();
+
+    for (const auto & shard_info : query_info.getCluster()->getShardsInfo())
+    {
+        ASTPtr query_ast_for_shard;
+        if (query_info.optimized_cluster && settings.optimize_skip_unused_shards_rewrite_in && shards > 1)
+        {
+            query_ast_for_shard = query_ast->clone();
+
+            OptimizeShardingKeyRewriteInVisitor::Data visitor_data{
+                sharding_key_expr,
+                sharding_key_expr->getSampleBlock().getByPosition(0).type,
+                sharding_key_column_name,
+                shard_info,
+                not_optimized_cluster->getSlotToShard(),
+            };
+            OptimizeShardingKeyRewriteInVisitor visitor(visitor_data);
+            visitor.visit(query_ast_for_shard);
+        }
+        else
+            query_ast_for_shard = query_ast;
+
+        auto shard_plans = stream_factory.createForShardWithParallelReplicas(shard_info,
+            query_ast_for_shard, main_table, table_func_ptr, throttler, context, shards);
+
+        if (!shard_plans.local_plan && !shard_plans.remote_plan)
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "No plans were generated for reading from shard. This is a bug");
+
+        if (shard_plans.local_plan)
+            plans.emplace_back(std::move(shard_plans.local_plan));
+
+        if (shard_plans.remote_plan)
+            plans.emplace_back(std::move(shard_plans.remote_plan));
+    }
+
+    if (plans.empty())
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "No plans were generated for reading from Distributed. This is a bug");
+
+    if (plans.size() == 1)
+    {
+        query_plan = std::move(*plans.front());
+        return;
+    }
+
+    DataStreams input_streams;
+    input_streams.reserve(plans.size());
+    for (const auto & plan : plans)
+        input_streams.emplace_back(plan->getCurrentDataStream());
+
+    auto union_step = std::make_unique<UnionStep>(std::move(input_streams));
+    query_plan.unitePlans(std::move(union_step), std::move(plans));
+}
+
 }
 
 }

src/Interpreters/ClusterProxy/executeQuery.h

@@ -23,7 +23,7 @@ struct StorageID;
 namespace ClusterProxy
 {
 
-class IStreamFactory;
+class SelectStreamFactory;
 
 /// Update settings for Distributed query.
 ///
@@ -46,7 +46,18 @@ void executeQuery(
     QueryProcessingStage::Enum processed_stage,
     const StorageID & main_table,
     const ASTPtr & table_func_ptr,
-    IStreamFactory & stream_factory, Poco::Logger * log,
+    SelectStreamFactory & stream_factory, Poco::Logger * log,
+    const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info,
+    const ExpressionActionsPtr & sharding_key_expr,
+    const std::string & sharding_key_column_name,
+    const ClusterPtr & not_optimized_cluster);
+
+
+void executeQueryWithParallelReplicas(
+    QueryPlan & query_plan,
+    const StorageID & main_table,
+    const ASTPtr & table_func_ptr,
+    SelectStreamFactory & stream_factory,
     const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info,
     const ExpressionActionsPtr & sharding_key_expr,
     const std::string & sharding_key_column_name,

src/Interpreters/DatabaseCatalog.cpp

@@ -119,7 +119,11 @@ TemporaryTableHolder & TemporaryTableHolder::operator=(TemporaryTableHolder && r
 TemporaryTableHolder::~TemporaryTableHolder()
 {
     if (id != UUIDHelpers::Nil)
+    {
+        auto table = getTable();
+        table->flushAndShutdown();
         temporary_tables->dropTable(getContext(), "_tmp_" + toString(id));
+    }
 }
 
 StorageID TemporaryTableHolder::getGlobalTableID() const

src/Interpreters/ExpressionAnalyzer.cpp

@@ -1999,7 +1999,6 @@ void ExpressionAnalysisResult::checkActions() const
         };
 
         check_actions(prewhere_info->prewhere_actions);
-        check_actions(prewhere_info->alias_actions);
     }
 }
 

src/Interpreters/InterpreterSelectQuery.cpp

@@ -1654,15 +1654,6 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan(
     {
         auto & prewhere_info = *prewhere_info_ptr;
 
-        if (prewhere_info.alias_actions)
-        {
-            pipe.addSimpleTransform([&](const Block & header)
-            {
-                return std::make_shared<ExpressionTransform>(header,
-                    std::make_shared<ExpressionActions>(prewhere_info.alias_actions));
-            });
-        }
-
         if (prewhere_info.row_level_filter)
         {
             pipe.addSimpleTransform([&](const Block & header)
@@ -1728,12 +1719,11 @@ void InterpreterSelectQuery::setMergeTreeReadTaskCallbackAndClientInfo(MergeTree
     context->setMergeTreeReadTaskCallback(std::move(callback));
 }
 
-void InterpreterSelectQuery::setProperClientInfo()
+void InterpreterSelectQuery::setProperClientInfo(size_t replica_num, size_t replica_count)
 {
     context->getClientInfo().query_kind = ClientInfo::QueryKind::SECONDARY_QUERY;
-    assert(options.shard_count.has_value() && options.shard_num.has_value());
+    context->getClientInfo().count_participating_replicas = replica_count;
-    context->getClientInfo().count_participating_replicas = *options.shard_count;
+    context->getClientInfo().number_of_current_replica = replica_num;
-    context->getClientInfo().number_of_current_replica = *options.shard_num;
 }
 
 bool InterpreterSelectQuery::shouldMoveToPrewhere()
@@ -1902,19 +1892,6 @@ void InterpreterSelectQuery::addPrewhereAliasActions()
             for (const auto & name : required_columns)
                 prewhere_info->prewhere_actions->tryRestoreColumn(name);
 
-            auto analyzed_result
-                = TreeRewriter(context).analyze(required_columns_from_prewhere_expr, metadata_snapshot->getColumns().getAllPhysical());
-            prewhere_info->alias_actions
-                = ExpressionAnalyzer(required_columns_from_prewhere_expr, analyzed_result, context).getActionsDAG(true, false);
-
-            /// Add (physical?) columns required by alias actions.
-            auto required_columns_from_alias = prewhere_info->alias_actions->getRequiredColumns();
-            Block prewhere_actions_result = prewhere_info->prewhere_actions->getResultColumns();
-            for (auto & column : required_columns_from_alias)
-                if (!prewhere_actions_result.has(column.name))
-                    if (required_columns.end() == std::find(required_columns.begin(), required_columns.end(), column.name))
-                        required_columns.push_back(column.name);
-
             /// Add physical columns required by prewhere actions.
             for (const auto & column : required_columns_from_prewhere)
                 if (!required_aliases_from_prewhere.contains(column))

src/Interpreters/InterpreterSelectQuery.h

@@ -125,7 +125,7 @@ public:
     void setMergeTreeReadTaskCallbackAndClientInfo(MergeTreeReadTaskCallback && callback);
 
     /// It will set shard_num and shard_count to the client_info
-    void setProperClientInfo();
+    void setProperClientInfo(size_t replica_num, size_t replica_count);
 
 private:
     InterpreterSelectQuery(

src/Interpreters/PartLog.cpp

@@ -16,6 +16,26 @@
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int NOT_IMPLEMENTED;
+}
+
+PartLogElement::MergeReasonType PartLogElement::getMergeReasonType(MergeType merge_type)
+{
+    switch (merge_type)
+    {
+    case MergeType::Regular:
+        return REGULAR_MERGE;
+    case MergeType::TTLDelete:
+        return TTL_DELETE_MERGE;
+    case MergeType::TTLRecompress:
+        return TTL_RECOMPRESS_MERGE;
+    }
+
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unknown MergeType {}", static_cast<UInt64>(merge_type));
+}
+
 NamesAndTypesList PartLogElement::getNamesAndTypes()
 {
     auto event_type_datatype = std::make_shared<DataTypeEnum8>(
@@ -30,11 +50,22 @@ NamesAndTypesList PartLogElement::getNamesAndTypes()
         }
     );
 
+    auto merge_reason_datatype = std::make_shared<DataTypeEnum8>(
+        DataTypeEnum8::Values
+        {
+            {"NotAMerge",           static_cast<Int8>(NOT_A_MERGE)},
+            {"RegularMerge",        static_cast<Int8>(REGULAR_MERGE)},
+            {"TTLDeleteMerge",      static_cast<Int8>(TTL_DELETE_MERGE)},
+            {"TTLRecompressMerge",  static_cast<Int8>(TTL_RECOMPRESS_MERGE)},
+        }
+    );
+
     ColumnsWithTypeAndName columns_with_type_and_name;
 
     return {
         {"query_id", std::make_shared<DataTypeString>()},
         {"event_type", std::move(event_type_datatype)},
+        {"merge_reason", std::move(merge_reason_datatype)},
         {"event_date", std::make_shared<DataTypeDate>()},
 
         {"event_time", std::make_shared<DataTypeDateTime>()},
@@ -72,6 +103,7 @@ void PartLogElement::appendToBlock(MutableColumns & columns) const
 
     columns[i++]->insert(query_id);
     columns[i++]->insert(event_type);
+    columns[i++]->insert(merge_reason);
     columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType());
     columns[i++]->insert(event_time);
     columns[i++]->insert(event_time_microseconds);

src/Interpreters/PartLog.h

@@ -4,6 +4,7 @@
 #include <Interpreters/SystemLog.h>
 #include <Core/NamesAndTypes.h>
 #include <Core/NamesAndAliases.h>
+#include <Storages/MergeTree/MergeType.h>
 
 
 namespace DB
@@ -21,9 +22,22 @@ struct PartLogElement
         MOVE_PART = 6,
     };
 
+    enum MergeReasonType
+    {
+        /// merge_reason is relevant only for event_type = 'MERGE_PARTS', in other cases it is NOT_A_MERGE
+        NOT_A_MERGE = 1,
+        /// Just regular merge
+        REGULAR_MERGE = 2,
+        /// Merge assigned to delete some data from parts (with TTLMergeSelector)
+        TTL_DELETE_MERGE = 3,
+        /// Merge with recompression
+        TTL_RECOMPRESS_MERGE = 4,
+    };
+
     String query_id;
 
     Type event_type = NEW_PART;
+    MergeReasonType merge_reason = NOT_A_MERGE;
 
     time_t event_time = 0;
     Decimal64 event_time_microseconds = 0;
@@ -57,6 +71,7 @@ struct PartLogElement
 
     static std::string name() { return "PartLog"; }
 
+    static MergeReasonType getMergeReasonType(MergeType merge_type);
     static NamesAndTypesList getNamesAndTypes();
     static NamesAndAliases getNamesAndAliases() { return {}; }
     void appendToBlock(MutableColumns & columns) const;

src/Interpreters/ProcessList.cpp

@@ -225,6 +225,8 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
             if (settings.memory_tracker_fault_probability)
                 thread_group->memory_tracker.setFaultProbability(settings.memory_tracker_fault_probability);
 
+            thread_group->memory_tracker.setOvercommitWaitingTime(settings.memory_usage_overcommit_max_wait_microseconds);
+
             /// NOTE: Do not set the limit for thread-level memory tracker since it could show unreal values
             ///  since allocation and deallocation could happen in different threads
         }
@@ -244,7 +246,6 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
         user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user);
         user_process_list.user_memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator_for_user);
         user_process_list.user_memory_tracker.setDescription("(for user)");
-        user_process_list.user_overcommit_tracker.setMaxWaitTime(settings.memory_usage_overcommit_max_wait_microseconds);
 
         if (!user_process_list.user_throttler)
         {

src/Interpreters/UserDefinedExecutableFunctionFactory.cpp

@@ -43,7 +43,7 @@ public:
     size_t getNumberOfArguments() const override { return executable_function->getConfiguration().arguments.size(); }
 
     bool useDefaultImplementationForConstants() const override { return true; }
-    bool useDefaultImplementationForNulls() const override { return true; }
+    bool useDefaultImplementationForNulls() const override { return false; }
     bool isDeterministic() const override { return false; }
     bool isDeterministicInScopeOfQuery() const override { return false; }
 

src/Interpreters/evaluateConstantExpression.cpp

@@ -31,11 +31,21 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
 }
 
+static std::pair<Field, std::shared_ptr<const IDataType>> getFieldAndDataTypeFromLiteral(ASTLiteral * literal)
+{
+    auto type = applyVisitor(FieldToDataType(), literal->value);
+    /// In case of Array field nested fields can have different types.
+    /// Example: Array [1, 2.3] will have 2 fields with types UInt64 and Float64
+    /// when result type is Array(Float64).
+    /// So, we need to convert this field to the result type.
+    Field res = convertFieldToType(literal->value, *type);
+    return {res, type};
+}
 
 std::pair<Field, std::shared_ptr<const IDataType>> evaluateConstantExpression(const ASTPtr & node, ContextPtr context)
 {
     if (ASTLiteral * literal = node->as<ASTLiteral>())
-        return std::make_pair(literal->value, applyVisitor(FieldToDataType(), literal->value));
+        return getFieldAndDataTypeFromLiteral(literal);
 
     NamesAndTypesList source_columns = {{ "_dummy", std::make_shared<DataTypeUInt8>() }};
 
@@ -63,7 +73,7 @@ std::pair<Field, std::shared_ptr<const IDataType>> evaluateConstantExpression(co
     /// AST potentially could be transformed to literal during TreeRewriter analyze.
     /// For example if we have SQL user defined function that return literal AS subquery.
     if (ASTLiteral * literal = ast->as<ASTLiteral>())
-        return std::make_pair(literal->value, applyVisitor(FieldToDataType(), literal->value));
+        return getFieldAndDataTypeFromLiteral(literal);
 
     ExpressionActionsPtr expr_for_constant_folding = ExpressionAnalyzer(ast, syntax_result, context).getConstActions();
 

src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp

@@ -41,6 +41,8 @@ std::unique_ptr<QueryPlan> createLocalPlan(
     QueryProcessingStage::Enum processed_stage,
     UInt32 shard_num,
     UInt32 shard_count,
+    size_t replica_num,
+    size_t replica_count,
     std::shared_ptr<ParallelReplicasReadingCoordinator> coordinator)
 {
     checkStackSize();
@@ -56,7 +58,7 @@ std::unique_ptr<QueryPlan> createLocalPlan(
             .setShardInfo(shard_num, shard_count)
             .ignoreASTOptimizations());
 
-    interpreter.setProperClientInfo();
+    interpreter.setProperClientInfo(replica_num, replica_count);
     if (coordinator)
     {
         interpreter.setMergeTreeReadTaskCallbackAndClientInfo([coordinator](PartitionReadRequest request) -> std::optional<PartitionReadResponse>

src/Processors/QueryPlan/DistributedCreateLocalPlan.h

@@ -15,6 +15,8 @@ std::unique_ptr<QueryPlan> createLocalPlan(
     QueryProcessingStage::Enum processed_stage,
     UInt32 shard_num,
     UInt32 shard_count,
+    size_t replica_num,
+    size_t replica_count,
     std::shared_ptr<ParallelReplicasReadingCoordinator> coordinator);
 
 }

src/Processors/QueryPlan/ReadFromRemote.cpp

@@ -16,6 +16,8 @@
 #include <Client/ConnectionPool.h>
 #include <Client/ConnectionPoolWithFailover.h>
 
+#include <boost/algorithm/string/join.hpp>
+
 namespace DB
 {
 
@@ -62,7 +64,7 @@ static String formattedAST(const ASTPtr & ast)
 }
 
 ReadFromRemote::ReadFromRemote(
-    ClusterProxy::IStreamFactory::Shards shards_,
+    ClusterProxy::SelectStreamFactory::Shards shards_,
     Block header_,
     QueryProcessingStage::Enum stage_,
     StorageID main_table_,
@@ -87,10 +89,7 @@ ReadFromRemote::ReadFromRemote(
 {
 }
 
-void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::IStreamFactory::Shard & shard,
+void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStreamFactory::Shard & shard)
-    std::shared_ptr<ParallelReplicasReadingCoordinator> coordinator,
-    std::shared_ptr<ConnectionPoolWithFailover> pool,
-    std::optional<IConnections::ReplicaInfo> replica_info)
 {
     bool add_agg_info = stage == QueryProcessingStage::WithMergeableState;
     bool add_totals = false;
@@ -103,10 +102,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::IStreamFacto
     }
 
     auto lazily_create_stream = [
-            replica_info = replica_info,
+            shard = shard, shard_count = shard_count, query = shard.query, header = shard.header,
-            pool = pool ? pool : shard.pool,
-            coordinator = coordinator,
-            shard_num = shard.shard_num, shard_count = shard_count, query = shard.query, header = shard.header,
             context = context, throttler = throttler,
             main_table = main_table, table_func_ptr = table_func_ptr,
             scalars = scalars, external_tables = external_tables,
@@ -122,15 +118,15 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::IStreamFacto
         try
         {
             if (table_func_ptr)
-                try_results = pool->getManyForTableFunction(timeouts, &current_settings, PoolMode::GET_MANY);
+                try_results = shard.shard_info.pool->getManyForTableFunction(timeouts, &current_settings, PoolMode::GET_MANY);
             else
-                try_results = pool->getManyChecked(timeouts, &current_settings, PoolMode::GET_MANY, main_table.getQualifiedName());
+                try_results = shard.shard_info.pool->getManyChecked(timeouts, &current_settings, PoolMode::GET_MANY, main_table.getQualifiedName());
         }
         catch (const Exception & ex)
         {
             if (ex.code() == ErrorCodes::ALL_CONNECTION_TRIES_FAILED)
                 LOG_WARNING(&Poco::Logger::get("ClusterProxy::SelectStreamFactory"),
-                    "Connections to remote replicas of local shard {} failed, will use stale local replica", shard_num);
+                    "Connections to remote replicas of local shard {} failed, will use stale local replica", shard.shard_info.shard_num);
             else
                 throw;
         }
@@ -144,7 +140,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::IStreamFacto
 
         if (try_results.empty() || local_delay < max_remote_delay)
         {
-            auto plan = createLocalPlan(query, header, context, stage, shard_num, shard_count, coordinator);
+            auto plan = createLocalPlan(query, header, context, stage, shard.shard_info.shard_num, shard_count, 0, 0, /*coordinator=*/nullptr);
 
             return QueryPipelineBuilder::getPipe(std::move(*plan->buildQueryPipeline(
                 QueryPlanOptimizationSettings::fromContext(context),
@@ -160,10 +156,9 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::IStreamFacto
             String query_string = formattedAST(query);
 
             scalars["_shard_num"]
-                = Block{{DataTypeUInt32().createColumnConst(1, shard_num), std::make_shared<DataTypeUInt32>(), "_shard_num"}};
+                = Block{{DataTypeUInt32().createColumnConst(1, shard.shard_info.shard_num), std::make_shared<DataTypeUInt32>(), "_shard_num"}};
             auto remote_query_executor = std::make_shared<RemoteQueryExecutor>(
-                pool, std::move(connections), query_string, header, context, throttler, scalars, external_tables, stage,
+                shard.shard_info.pool, std::move(connections), query_string, header, context, throttler, scalars, external_tables, stage);
-                RemoteQueryExecutor::Extension{.parallel_reading_coordinator = std::move(coordinator), .replica_info = replica_info});
 
             return createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read);
         }
@@ -174,10 +169,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::IStreamFacto
     addConvertingActions(pipes.back(), output_stream->header);
 }
 
-void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::IStreamFactory::Shard & shard,
+void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFactory::Shard & shard)
-    std::shared_ptr<ParallelReplicasReadingCoordinator> coordinator,
-    std::shared_ptr<ConnectionPoolWithFailover> pool,
-    std::optional<IConnections::ReplicaInfo> replica_info)
 {
     bool add_agg_info = stage == QueryProcessingStage::WithMergeableState;
     bool add_totals = false;
@@ -192,20 +184,15 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::IStreamFactory::
     String query_string = formattedAST(shard.query);
 
     scalars["_shard_num"]
-        = Block{{DataTypeUInt32().createColumnConst(1, shard.shard_num), std::make_shared<DataTypeUInt32>(), "_shard_num"}};
+        = Block{{DataTypeUInt32().createColumnConst(1, shard.shard_info.shard_num), std::make_shared<DataTypeUInt32>(), "_shard_num"}};
 
     std::shared_ptr<RemoteQueryExecutor> remote_query_executor;
 
     remote_query_executor = std::make_shared<RemoteQueryExecutor>(
-            pool ? pool : shard.pool, query_string, shard.header, context, throttler, scalars, external_tables, stage,
+            shard.shard_info.pool, query_string, shard.header, context, throttler, scalars, external_tables, stage);
-            RemoteQueryExecutor::Extension{.parallel_reading_coordinator = std::move(coordinator), .replica_info = std::move(replica_info)});
 
     remote_query_executor->setLogger(log);
-
+    remote_query_executor->setPoolMode(PoolMode::GET_MANY);
-    /// In case of parallel reading from replicas we have a connection pool per replica.
-    /// Setting PoolMode will make no sense.
-    if (!pool)
-        remote_query_executor->setPoolMode(PoolMode::GET_MANY);
 
     if (!table_func_ptr)
         remote_query_executor->setMainTable(main_table);
@@ -219,52 +206,119 @@ void ReadFromRemote::initializePipeline(QueryPipelineBuilder & pipeline, const B
 {
     Pipes pipes;
 
-    const auto & settings = context->getSettingsRef();
+    for (const auto & shard : shards)
-    const bool enable_sample_offset_parallel_processing = settings.max_parallel_replicas > 1 && !settings.allow_experimental_parallel_reading_from_replicas;
-
-    /// We have to create a pipe for each replica
-    /// FIXME: The second condition is only for tests to work, because hedged connections enabled by default.
-    if (settings.max_parallel_replicas > 1 && !enable_sample_offset_parallel_processing && !context->getSettingsRef().use_hedged_requests)
     {
-        const Settings & current_settings = context->getSettingsRef();
+        if (shard.lazy)
-        auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(current_settings);
+            addLazyPipe(pipes, shard);
-
+        else
-        for (const auto & shard : shards)
+            addPipe(pipes, shard);
-        {
-            auto coordinator = std::make_shared<ParallelReplicasReadingCoordinator>();
-
-            for (size_t replica_num = 0; replica_num < shard.num_replicas; ++replica_num)
-            {
-                IConnections::ReplicaInfo replica_info
-                {
-                    .all_replicas_count = shard.num_replicas,
-                    .number_of_current_replica = replica_num
-                };
-
-                auto pool = shard.per_replica_pools[replica_num];
-                auto pool_with_failover =  std::make_shared<ConnectionPoolWithFailover>(
-                    ConnectionPoolPtrs{pool}, current_settings.load_balancing);
-
-                if (shard.lazy)
-                    addLazyPipe(pipes, shard, coordinator, pool_with_failover, replica_info);
-                else
-                    addPipe(pipes, shard, coordinator, pool_with_failover, replica_info);
-            }
-        }
-    }
-    else
-    {
-        for (const auto & shard : shards)
-        {
-            if (shard.lazy)
-                addLazyPipe(pipes, shard, /*coordinator=*/nullptr, /*pool*/{}, /*replica_info*/std::nullopt);
-            else
-                addPipe(pipes, shard, /*coordinator=*/nullptr, /*pool*/{}, /*replica_info*/std::nullopt);
-        }
     }
 
     auto pipe = Pipe::unitePipes(std::move(pipes));
     pipeline.init(std::move(pipe));
 }
 
+
+ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep(
+    ParallelReplicasReadingCoordinatorPtr coordinator_,
+    ClusterProxy::SelectStreamFactory::Shard shard_,
+    Block header_,
+    QueryProcessingStage::Enum stage_,
+    StorageID main_table_,
+    ASTPtr table_func_ptr_,
+    ContextPtr context_,
+    ThrottlerPtr throttler_,
+    Scalars scalars_,
+    Tables external_tables_,
+    Poco::Logger * log_,
+    UInt32 shard_count_)
+    : ISourceStep(DataStream{.header = std::move(header_)})
+    , coordinator(std::move(coordinator_))
+    , shard(std::move(shard_))
+    , stage(std::move(stage_))
+    , main_table(std::move(main_table_))
+    , table_func_ptr(table_func_ptr_)
+    , context(context_)
+    , throttler(throttler_)
+    , scalars(scalars_)
+    , external_tables{external_tables_}
+    , log(log_)
+    , shard_count(shard_count_)
+{
+    std::vector<String> description;
+
+    for (const auto & address : shard.shard_info.all_addresses)
+        if (!address.is_local)
+            description.push_back(fmt::format("Replica: {}", address.host_name));
+
+    setStepDescription(boost::algorithm::join(description, ", "));
+}
+
+
+void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
+{
+    Pipes pipes;
+
+    const Settings & current_settings = context->getSettingsRef();
+    auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(current_settings);
+
+    for (size_t replica_num = 0; replica_num < shard.shard_info.getAllNodeCount(); ++replica_num)
+    {
+        if (shard.shard_info.all_addresses[replica_num].is_local)
+            continue;
+
+        IConnections::ReplicaInfo replica_info
+        {
+            .all_replicas_count = shard.shard_info.getAllNodeCount(),
+            .number_of_current_replica = replica_num
+        };
+
+        auto pool = shard.shard_info.per_replica_pools[replica_num];
+        assert(pool);
+
+        auto pool_with_failover =  std::make_shared<ConnectionPoolWithFailover>(
+            ConnectionPoolPtrs{pool}, current_settings.load_balancing);
+
+        addPipeForSingeReplica(pipes, pool_with_failover, replica_info);
+    }
+
+    auto pipe = Pipe::unitePipes(std::move(pipes));
+    pipeline.init(std::move(pipe));
+
+}
+
+
+void ReadFromParallelRemoteReplicasStep::addPipeForSingeReplica(Pipes & pipes, std::shared_ptr<ConnectionPoolWithFailover> pool, IConnections::ReplicaInfo replica_info)
+{
+    bool add_agg_info = stage == QueryProcessingStage::WithMergeableState;
+    bool add_totals = false;
+    bool add_extremes = false;
+    bool async_read = context->getSettingsRef().async_socket_for_remote;
+    if (stage == QueryProcessingStage::Complete)
+    {
+        add_totals = shard.query->as<ASTSelectQuery &>().group_by_with_totals;
+        add_extremes = context->getSettingsRef().extremes;
+    }
+
+    String query_string = formattedAST(shard.query);
+
+    scalars["_shard_num"]
+        = Block{{DataTypeUInt32().createColumnConst(1, shard.shard_info.shard_num), std::make_shared<DataTypeUInt32>(), "_shard_num"}};
+
+    std::shared_ptr<RemoteQueryExecutor> remote_query_executor;
+
+    remote_query_executor = std::make_shared<RemoteQueryExecutor>(
+            pool, query_string, shard.header, context, throttler, scalars, external_tables, stage,
+            RemoteQueryExecutor::Extension{.parallel_reading_coordinator = coordinator, .replica_info = std::move(replica_info)});
+
+    remote_query_executor->setLogger(log);
+
+    if (!table_func_ptr)
+        remote_query_executor->setMainTable(main_table);
+
+    pipes.emplace_back(createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read));
+    pipes.back().addInterpreterContext(context);
+    addConvertingActions(pipes.back(), output_stream->header);
+}
+
 }

src/Processors/QueryPlan/ReadFromRemote.h

@@ -4,7 +4,7 @@
 #include <Client/IConnections.h>
 #include <Storages/IStorage_fwd.h>
 #include <Interpreters/StorageID.h>
-#include <Interpreters/ClusterProxy/IStreamFactory.h>
+#include <Interpreters/ClusterProxy/SelectStreamFactory.h>
 #include <Storages/MergeTree/ParallelReplicasReadingCoordinator.h>
 
 namespace DB
@@ -22,7 +22,7 @@ class ReadFromRemote final : public ISourceStep
 {
 public:
     ReadFromRemote(
-        ClusterProxy::IStreamFactory::Shards shards_,
+        ClusterProxy::SelectStreamFactory::Shards shards_,
         Block header_,
         QueryProcessingStage::Enum stage_,
         StorageID main_table_,
@@ -45,7 +45,7 @@ private:
         PerShard
     };
 
-    ClusterProxy::IStreamFactory::Shards shards;
+    ClusterProxy::SelectStreamFactory::Shards shards;
     QueryProcessingStage::Enum stage;
 
     StorageID main_table;
@@ -60,16 +60,52 @@ private:
     Poco::Logger * log;
 
     UInt32 shard_count;
-    void addLazyPipe(Pipes & pipes, const ClusterProxy::IStreamFactory::Shard & shard,
+    void addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStreamFactory::Shard & shard);
-        std::shared_ptr<ParallelReplicasReadingCoordinator> coordinator,
+    void addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFactory::Shard & shard);
-        std::shared_ptr<ConnectionPoolWithFailover> pool,
+};
-        std::optional<IConnections::ReplicaInfo> replica_info);
-    void addPipe(Pipes & pipes, const ClusterProxy::IStreamFactory::Shard & shard,
-        std::shared_ptr<ParallelReplicasReadingCoordinator> coordinator,
-        std::shared_ptr<ConnectionPoolWithFailover> pool,
-        std::optional<IConnections::ReplicaInfo> replica_info);
 
-    void addPipeForReplica();
+
+class ReadFromParallelRemoteReplicasStep : public ISourceStep
+{
+public:
+    ReadFromParallelRemoteReplicasStep(
+        ParallelReplicasReadingCoordinatorPtr coordinator_,
+        ClusterProxy::SelectStreamFactory::Shard shard,
+        Block header_,
+        QueryProcessingStage::Enum stage_,
+        StorageID main_table_,
+        ASTPtr table_func_ptr_,
+        ContextPtr context_,
+        ThrottlerPtr throttler_,
+        Scalars scalars_,
+        Tables external_tables_,
+        Poco::Logger * log_,
+        UInt32 shard_count_);
+
+    String getName() const override { return "ReadFromRemoteParallelReplicas"; }
+
+    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
+
+private:
+
+    void addPipeForSingeReplica(Pipes & pipes, std::shared_ptr<ConnectionPoolWithFailover> pool, IConnections::ReplicaInfo replica_info);
+
+    ParallelReplicasReadingCoordinatorPtr coordinator;
+    ClusterProxy::SelectStreamFactory::Shard shard;
+    QueryProcessingStage::Enum stage;
+
+    StorageID main_table;
+    ASTPtr table_func_ptr;
+
+    ContextPtr context;
+
+    ThrottlerPtr throttler;
+    Scalars scalars;
+    Tables external_tables;
+
+    Poco::Logger * log;
+
+    UInt32 shard_count{0};
 };
 
 }

src/Storages/ColumnCodec.h

@@ -1,11 +0,0 @@
-#pragma once
-
-#include <string>
-#include <unordered_map>
-#include <Parsers/IAST.h>
-#include <Compression/ICompressionCodec.h>
-
-namespace DB
-{
-    using ColumnCodecs = std::unordered_map<std::string, CompressionCodecPtr>;
-}

src/Storages/ColumnsDescription.h

@@ -6,7 +6,6 @@
 #include <Core/NamesAndTypes.h>
 #include <Core/NamesAndAliases.h>
 #include <Interpreters/Context_fwd.h>
-#include <Storages/ColumnCodec.h>
 #include <Storages/ColumnDefault.h>
 #include <Common/Exception.h>
 

src/Storages/ExternalDataSourceConfiguration.cpp

@@ -248,6 +248,65 @@ std::optional<ExternalDataSourceInfo> getExternalDataSourceConfiguration(
     return std::nullopt;
 }
 
+std::optional<URLBasedDataSourceConfig> getURLBasedDataSourceConfiguration(
+    const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix, ContextPtr context)
+{
+    URLBasedDataSourceConfiguration configuration;
+    auto collection_name = dict_config.getString(dict_config_prefix + ".name", "");
+    if (!collection_name.empty())
+    {
+        const auto & config = context->getConfigRef();
+        const auto & collection_prefix = fmt::format("named_collections.{}", collection_name);
+
+        if (!config.has(collection_prefix))
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "There is no collection named `{}` in config", collection_name);
+
+        configuration.url =
+            dict_config.getString(dict_config_prefix + ".url", config.getString(collection_prefix + ".url", ""));
+        configuration.endpoint =
+            dict_config.getString(dict_config_prefix + ".endpoint", config.getString(collection_prefix + ".endpoint", ""));
+        configuration.format =
+            dict_config.getString(dict_config_prefix + ".format", config.getString(collection_prefix + ".format", ""));
+        configuration.compression_method =
+            dict_config.getString(dict_config_prefix + ".compression", config.getString(collection_prefix + ".compression_method", ""));
+        configuration.structure =
+            dict_config.getString(dict_config_prefix + ".structure", config.getString(collection_prefix + ".structure", ""));
+        configuration.user =
+            dict_config.getString(dict_config_prefix + ".credentials.user", config.getString(collection_prefix + ".credentials.user", ""));
+        configuration.password =
+            dict_config.getString(dict_config_prefix + ".credentials.password", config.getString(collection_prefix + ".credentials.password", ""));
+
+        String headers_prefix;
+        const Poco::Util::AbstractConfiguration *headers_config = nullptr;
+        if (dict_config.has(dict_config_prefix + ".headers"))
+        {
+            headers_prefix = dict_config_prefix + ".headers";
+            headers_config = &dict_config;
+        }
+        else
+        {
+            headers_prefix = collection_prefix + ".headers";
+            headers_config = &config;
+        }
+
+        if (headers_config)
+        {
+            Poco::Util::AbstractConfiguration::Keys header_keys;
+            headers_config->keys(headers_prefix, header_keys);
+            headers_prefix += ".";
+            for (const auto & header : header_keys)
+            {
+                const auto header_prefix = headers_prefix + header;
+                configuration.headers.emplace_back(
+                    std::make_pair(headers_config->getString(header_prefix + ".name"), headers_config->getString(header_prefix + ".value")));
+            }
+        }
+
+        return URLBasedDataSourceConfig{ .configuration = configuration };
+    }
+
+    return std::nullopt;
+}
 
 ExternalDataSourcesByPriority getExternalDataSourceConfigurationByPriority(
     const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix, ContextPtr context, HasConfigKeyFunc has_config_key)

src/Storages/ExternalDataSourceConfiguration.h

@@ -99,10 +99,14 @@ getExternalDataSourceConfigurationByPriority(const Poco::Util::AbstractConfigura
 struct URLBasedDataSourceConfiguration
 {
     String url;
+    String endpoint;
     String format = "auto";
     String compression_method = "auto";
     String structure = "auto";
 
+    String user;
+    String password;
+
     std::vector<std::pair<String, Field>> headers;
     String http_method;
 
@@ -129,6 +133,9 @@ struct URLBasedDataSourceConfig
 
 std::optional<URLBasedDataSourceConfig> getURLBasedDataSourceConfiguration(const ASTs & args, ContextPtr context);
 
+std::optional<URLBasedDataSourceConfig> getURLBasedDataSourceConfiguration(
+    const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix, ContextPtr context);
+
 template<typename T>
 bool getExternalDataSourceConfiguration(const ASTs & args, BaseSettings<T> & settings, ContextPtr context);
 

src/Storages/IStorage.cpp

@@ -231,11 +231,6 @@ std::string PrewhereInfo::dump() const
     WriteBufferFromOwnString ss;
     ss << "PrewhereDagInfo\n";
 
-    if (alias_actions)
-    {
-        ss << "alias_actions " << alias_actions->dumpDAG() << "\n";
-    }
-
     if (prewhere_actions)
     {
         ss << "prewhere_actions " << prewhere_actions->dumpDAG() << "\n";

src/Storages/MergeTree/BackgroundJobsAssignee.cpp

@@ -50,10 +50,11 @@ void BackgroundJobsAssignee::postpone()
 }
 
 
-void BackgroundJobsAssignee::scheduleMergeMutateTask(ExecutableTaskPtr merge_task)
+bool BackgroundJobsAssignee::scheduleMergeMutateTask(ExecutableTaskPtr merge_task)
 {
     bool res = getContext()->getMergeMutateExecutor()->trySchedule(merge_task);
     res ? trigger() : postpone();
+    return res;
 }
 
 

src/Storages/MergeTree/BackgroundJobsAssignee.h

@@ -66,7 +66,7 @@ public:
     void postpone();
     void finish();
 
-    void scheduleMergeMutateTask(ExecutableTaskPtr merge_task);
+    bool scheduleMergeMutateTask(ExecutableTaskPtr merge_task);
     void scheduleFetchTask(ExecutableTaskPtr fetch_task);
     void scheduleMoveTask(ExecutableTaskPtr move_task);
     void scheduleCommonTask(ExecutableTaskPtr common_task, bool need_trigger);

src/Storages/MergeTree/MergeList.h

@@ -197,6 +197,11 @@ public:
         ++merges_with_ttl_counter;
     }
 
+    void cancelMergeWithTTL()
+    {
+        --merges_with_ttl_counter;
+    }
+
     size_t getMergesWithTTLCount() const
     {
         return merges_with_ttl_counter;

src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp

@@ -72,8 +72,6 @@ MergeTreeBaseSelectProcessor::MergeTreeBaseSelectProcessor(
     if (prewhere_info)
     {
         prewhere_actions = std::make_unique<PrewhereExprInfo>();
-        if (prewhere_info->alias_actions)
-            prewhere_actions->alias_actions = std::make_shared<ExpressionActions>(prewhere_info->alias_actions, actions_settings);
 
         if (prewhere_info->row_level_filter)
             prewhere_actions->row_level_filter = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter, actions_settings);
@@ -556,9 +554,6 @@ Block MergeTreeBaseSelectProcessor::transformHeader(
 {
     if (prewhere_info)
     {
-        if (prewhere_info->alias_actions)
-            block = prewhere_info->alias_actions->updateHeader(std::move(block));
-
         if (prewhere_info->row_level_filter)
         {
             block = prewhere_info->row_level_filter->updateHeader(std::move(block));