Merge branch 'master' into ch_canh_fix_decrypt_with_null

.github/workflows/master.yml

@@ -887,6 +887,51 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  BuilderBinAarch64V80Compat:
+    needs: [DockerHubPush]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          BUILD_NAME=binary_aarch64_v80compat
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        run: |
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
+      - name: Upload build URLs to artifacts
+        if: ${{ success() || failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
 ############################################################################################
 ##################################### Docker images  #######################################
 ############################################################################################
@@ -972,6 +1017,7 @@ jobs:
       # - BuilderBinGCC
       - BuilderBinPPC64
       - BuilderBinAmd64SSE2
+      - BuilderBinAarch64V80Compat
       - BuilderBinClangTidy
       - BuilderDebShared
     runs-on: [self-hosted, style-checker]

.github/workflows/pull_request.yml

@@ -940,6 +940,49 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  BuilderBinAarch64V80Compat:
+    needs: [DockerHubPush, FastTest, StyleCheck]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          BUILD_NAME=binary_aarch64_v80compat
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Build
+        run: |
+          git -C "$GITHUB_WORKSPACE" submodule sync --recursive
+          git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
+      - name: Upload build URLs to artifacts
+        if: ${{ success() || failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
 ############################################################################################
 ##################################### Docker images  #######################################
 ############################################################################################
@@ -1025,6 +1068,7 @@ jobs:
       # - BuilderBinGCC
       - BuilderBinPPC64
       - BuilderBinAmd64SSE2
+      - BuilderBinAarch64V80Compat
       - BuilderBinClangTidy
       - BuilderDebShared
     runs-on: [self-hosted, style-checker]

CMakeLists.txt

@@ -18,7 +18,7 @@ include (cmake/target.cmake)
 include (cmake/tools.cmake)
 include (cmake/ccache.cmake)
 include (cmake/clang_tidy.cmake)
-include (cmake/git_status.cmake)
+include (cmake/git.cmake)
 
 # Ignore export() since we don't use it,
 # but it gets broken with a global targets via link_libraries()

base/base/CachedFn.h

@@ -1,75 +0,0 @@
-#pragma once
-
-#include <map>
-#include <tuple>
-#include <mutex>
-#include "FnTraits.h"
-
-/**
- * Caching proxy for a functor that decays to a pointer-to-function.
- * Saves pairs (func args, func result on args).
- * Cache size is unlimited. Cache items are evicted only on manual drop.
- * Invocation/update is O(log(saved cache values)).
- *
- * See Common/tests/cached_fn.cpp for examples.
-  */
-template <auto * Func>
-struct CachedFn
-{
-private:
-    using Traits = FnTraits<decltype(Func)>;
-    using DecayedArgs = TypeListMap<std::decay_t, typename Traits::Args>;
-    using Key = TypeListChangeRoot<std::tuple, DecayedArgs>;
-    using Result = typename Traits::Ret;
-
-    std::map<Key, Result> cache; // Can't use hashmap as tuples are unhashable by default
-    mutable std::mutex mutex;
-
-public:
-    template <class ...Args>
-    Result operator()(Args && ...args)
-    {
-        Key key{std::forward<Args>(args)...};
-
-        {
-            std::lock_guard lock(mutex);
-
-            if (auto it = cache.find(key); it != cache.end())
-                return it->second;
-        }
-
-        Result res = std::apply(Func, key);
-
-        {
-            std::lock_guard lock(mutex);
-            cache.emplace(std::move(key), res);
-        }
-
-        return res;
-    }
-
-    template <class ...Args>
-    void update(Args && ...args)
-    {
-        Key key{std::forward<Args>(args)...};
-        Result res = std::apply(Func, key);
-
-        {
-            std::lock_guard lock(mutex);
-            // TODO Can't use emplace(std::move(key), ..), causes test_host_ip_change errors.
-            cache[key] = std::move(res);
-        }
-    }
-
-    size_t size() const
-    {
-        std::lock_guard lock(mutex);
-        return cache.size();
-    }
-
-    void drop()
-    {
-        std::lock_guard lock(mutex);
-        cache.clear();
-    }
-};

base/base/ReplxxLineReader.cpp

@@ -1,6 +1,7 @@
 #include <base/ReplxxLineReader.h>
 #include <base/errnoToString.h>
 
+#include <stdexcept>
 #include <chrono>
 #include <cerrno>
 #include <cstring>
@@ -13,8 +14,10 @@
 #include <dlfcn.h>
 #include <fcntl.h>
 #include <fstream>
+#include <filesystem>
 #include <fmt/format.h>
-
+#include <boost/algorithm/string/split.hpp>
+#include <boost/algorithm/string/classification.hpp> /// is_any_of
 
 namespace
 {
@@ -35,6 +38,166 @@ std::string getEditor()
     return editor;
 }
 
+std::string getFuzzyFinder()
+{
+    const char * env_path = std::getenv("PATH"); // NOLINT(concurrency-mt-unsafe)
+
+    if (!env_path || !*env_path)
+        return {};
+
+    std::vector<std::string> paths;
+    boost::split(paths, env_path, boost::is_any_of(":"));
+    for (const auto & path_str : paths)
+    {
+        std::filesystem::path path(path_str);
+        std::filesystem::path sk_bin_path = path / "sk";
+        if (!access(sk_bin_path.c_str(), X_OK))
+            return sk_bin_path;
+
+        std::filesystem::path fzf_bin_path = path / "fzf";
+        if (!access(fzf_bin_path.c_str(), X_OK))
+            return fzf_bin_path;
+    }
+
+    return {};
+}
+
+/// See comments in ShellCommand::executeImpl()
+/// (for the vfork via dlsym())
+int executeCommand(char * const argv[])
+{
+#if !defined(USE_MUSL)
+    /** Here it is written that with a normal call `vfork`, there is a chance of deadlock in multithreaded programs,
+      *  because of the resolving of symbols in the shared library
+      * http://www.oracle.com/technetwork/server-storage/solaris10/subprocess-136439.html
+      * Therefore, separate the resolving of the symbol from the call.
+      */
+    static void * real_vfork = dlsym(RTLD_DEFAULT, "vfork");
+#else
+    /// If we use Musl with static linking, there is no dlsym and no issue with vfork.
+    static void * real_vfork = reinterpret_cast<void *>(&vfork);
+#endif
+    if (!real_vfork)
+        throw std::runtime_error("Cannot find vfork symbol");
+
+    pid_t pid = reinterpret_cast<pid_t (*)()>(real_vfork)();
+
+    if (-1 == pid)
+        throw std::runtime_error(fmt::format("Cannot vfork {}: {}", argv[0], errnoToString()));
+
+    /// Child
+    if (0 == pid)
+    {
+        sigset_t mask;
+        sigemptyset(&mask);
+        sigprocmask(0, nullptr, &mask); // NOLINT(concurrency-mt-unsafe) // ok in newly created process
+        sigprocmask(SIG_UNBLOCK, &mask, nullptr); // NOLINT(concurrency-mt-unsafe) // ok in newly created process
+
+        execvp(argv[0], argv);
+        _exit(-1);
+    }
+
+    int status = 0;
+    do
+    {
+        int exited_pid = waitpid(pid, &status, 0);
+        if (exited_pid != -1)
+            break;
+
+        if (errno == EINTR)
+            continue;
+
+        throw std::runtime_error(fmt::format("Cannot waitpid {}: {}", pid, errnoToString()));
+    } while (true);
+
+    return status;
+}
+
+void writeRetry(int fd, const std::string & data)
+{
+    size_t bytes_written = 0;
+    const char * begin = data.c_str();
+    size_t offset = data.size();
+
+    while (bytes_written != offset)
+    {
+        ssize_t res = ::write(fd, begin + bytes_written, offset - bytes_written);
+        if ((-1 == res || 0 == res) && errno != EINTR)
+            throw std::runtime_error(fmt::format("Cannot write to {}: {}", fd, errnoToString()));
+        bytes_written += res;
+    }
+}
+std::string readFile(const std::string & path)
+{
+    std::ifstream t(path);
+    std::string str;
+    t.seekg(0, std::ios::end);
+    str.reserve(t.tellg());
+    t.seekg(0, std::ios::beg);
+    str.assign((std::istreambuf_iterator<char>(t)), std::istreambuf_iterator<char>());
+    return str;
+}
+
+/// Simple wrapper for temporary files.
+class TemporaryFile
+{
+private:
+    std::string path;
+    int fd = -1;
+
+public:
+    explicit TemporaryFile(const char * pattern)
+        : path(pattern)
+    {
+        size_t dot_pos = path.rfind('.');
+        if (dot_pos != std::string::npos)
+            fd = ::mkstemps(path.data(), path.size() - dot_pos);
+        else
+            fd = ::mkstemp(path.data());
+
+        if (-1 == fd)
+            throw std::runtime_error(fmt::format("Cannot create temporary file {}: {}", path, errnoToString()));
+    }
+    ~TemporaryFile()
+    {
+        try
+        {
+            close();
+            unlink();
+        }
+        catch (const std::runtime_error & e)
+        {
+            fmt::print(stderr, "{}", e.what());
+        }
+    }
+
+    void close()
+    {
+        if (fd == -1)
+            return;
+
+        if (0 != ::close(fd))
+            throw std::runtime_error(fmt::format("Cannot close temporary file {}: {}", path, errnoToString()));
+        fd = -1;
+    }
+
+    void write(const std::string & data)
+    {
+        if (fd == -1)
+            throw std::runtime_error(fmt::format("Cannot write to uninitialized file {}", path));
+
+        writeRetry(fd, data);
+    }
+
+    void unlink()
+    {
+        if (0 != ::unlink(path.c_str()))
+            throw std::runtime_error(fmt::format("Cannot remove temporary file {}: {}", path, errnoToString()));
+    }
+
+    std::string & getPath() { return path; }
+};
+
 /// Copied from replxx::src/util.cxx::now_ms_str() under the terms of 3-clause BSD license of Replxx.
 /// Copyright (c) 2017-2018, Marcin Konarski (amok at codestation.org)
 /// Copyright (c) 2010, Salvatore Sanfilippo (antirez at gmail dot com)
@@ -142,6 +305,7 @@ ReplxxLineReader::ReplxxLineReader(
     replxx::Replxx::highlighter_callback_t highlighter_)
     : LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_)), highlighter(std::move(highlighter_))
     , editor(getEditor())
+    , fuzzy_finder(getFuzzyFinder())
 {
     using namespace std::placeholders;
     using Replxx = replxx::Replxx;
@@ -249,6 +413,17 @@ ReplxxLineReader::ReplxxLineReader(
         return rx.invoke(Replxx::ACTION::COMMIT_LINE, code);
     };
     rx.bind_key(Replxx::KEY::meta('#'), insert_comment_action);
+
+    /// interactive search in history (requires fzf/sk)
+    if (!fuzzy_finder.empty())
+    {
+        auto interactive_history_search = [this](char32_t code)
+        {
+            openInteractiveHistorySearch();
+            return rx.invoke(Replxx::ACTION::REPAINT, code);
+        };
+        rx.bind_key(Replxx::KEY::control('R'), interactive_history_search);
+    }
 }
 
 ReplxxLineReader::~ReplxxLineReader()
@@ -293,116 +468,70 @@ void ReplxxLineReader::addToHistory(const String & line)
         rx.print("Unlock of history file failed: %s\n", errnoToString().c_str());
 }
 
-/// See comments in ShellCommand::executeImpl()
-/// (for the vfork via dlsym())
-int ReplxxLineReader::executeEditor(const std::string & path)
-{
-    std::vector<char> argv0(editor.data(), editor.data() + editor.size() + 1);
-    std::vector<char> argv1(path.data(), path.data() + path.size() + 1);
-    char * const argv[] = {argv0.data(), argv1.data(), nullptr};
-
-    static void * real_vfork = dlsym(RTLD_DEFAULT, "vfork");
-    if (!real_vfork)
-    {
-        rx.print("Cannot find symbol vfork in myself: %s\n", errnoToString().c_str());
-        return -1;
-    }
-
-    pid_t pid = reinterpret_cast<pid_t (*)()>(real_vfork)();
-
-    if (-1 == pid)
-    {
-        rx.print("Cannot vfork: %s\n", errnoToString().c_str());
-        return -1;
-    }
-
-    /// Child
-    if (0 == pid)
-    {
-        sigset_t mask;
-        sigemptyset(&mask);
-        sigprocmask(0, nullptr, &mask); // NOLINT(concurrency-mt-unsafe) // ok in newly created process
-        sigprocmask(SIG_UNBLOCK, &mask, nullptr); // NOLINT(concurrency-mt-unsafe) // ok in newly created process
-
-        execvp(editor.c_str(), argv);
-        rx.print("Cannot execute %s: %s\n", editor.c_str(), errnoToString().c_str());
-        _exit(-1);
-    }
-
-    int status = 0;
-    do
-    {
-        int exited_pid = waitpid(pid, &status, 0);
-        if (exited_pid == -1)
-        {
-            if (errno == EINTR)
-                continue;
-
-            rx.print("Cannot waitpid: %s\n", errnoToString().c_str());
-            return -1;
-        }
-        else
-            break;
-    } while (true);
-    return status;
-}
-
 void ReplxxLineReader::openEditor()
 {
-    char filename[] = "clickhouse_replxx_XXXXXX.sql";
+    TemporaryFile editor_file("clickhouse_client_editor_XXXXXX.sql");
-    int fd = ::mkstemps(filename, 4);
+    editor_file.write(rx.get_state().text());
-    if (-1 == fd)
+    editor_file.close();
-    {
-        rx.print("Cannot create temporary file to edit query: %s\n", errnoToString().c_str());
-        return;
-    }
 
-    replxx::Replxx::State state(rx.get_state());
+    char * const argv[] = {editor.data(), editor_file.getPath().data(), nullptr};
-
-    size_t bytes_written = 0;
-    const char * begin = state.text();
-    size_t offset = strlen(state.text());
-    while (bytes_written != offset)
-    {
-        ssize_t res = ::write(fd, begin + bytes_written, offset - bytes_written);
-        if ((-1 == res || 0 == res) && errno != EINTR)
-        {
-            rx.print("Cannot write to temporary query file %s: %s\n", filename, errnoToString().c_str());
-            break;
-        }
-        bytes_written += res;
-    }
-
-    if (0 != ::close(fd))
-    {
-        rx.print("Cannot close temporary query file %s: %s\n", filename, errnoToString().c_str());
-        return;
-    }
-
-    if (0 == executeEditor(filename))
-    {
     try
     {
-            std::ifstream t(filename);
+        if (executeCommand(argv) == 0)
-            std::string str;
-            t.seekg(0, std::ios::end);
-            str.reserve(t.tellg());
-            t.seekg(0, std::ios::beg);
-            str.assign((std::istreambuf_iterator<char>(t)), std::istreambuf_iterator<char>());
-            rx.set_state(replxx::Replxx::State(str.c_str(), str.size()));
-        }
-        catch (...)
         {
-            rx.print("Cannot read from temporary query file %s: %s\n", filename, errnoToString().c_str());
+            const std::string & new_query = readFile(editor_file.getPath());
-            return;
+            rx.set_state(replxx::Replxx::State(new_query.c_str(), new_query.size()));
         }
     }
+    catch (const std::runtime_error & e)
+    {
+        rx.print(e.what());
+    }
 
     if (bracketed_paste_enabled)
         enableBracketedPaste();
+}
 
-    if (0 != ::unlink(filename))
+void ReplxxLineReader::openInteractiveHistorySearch()
-        rx.print("Cannot remove temporary query file %s: %s\n", filename, errnoToString().c_str());
+{
+    assert(!fuzzy_finder.empty());
+    TemporaryFile history_file("clickhouse_client_history_in_XXXXXX.bin");
+    auto hs(rx.history_scan());
+    while (hs.next())
+    {
+        history_file.write(hs.get().text());
+        history_file.write(std::string(1, '\0'));
+    }
+    history_file.close();
+
+    TemporaryFile output_file("clickhouse_client_history_out_XXXXXX.sql");
+    output_file.close();
+
+    char sh[] = "sh";
+    char sh_c[] = "-c";
+    /// NOTE: You can use one of the following to configure the behaviour additionally:
+    /// - SKIM_DEFAULT_OPTIONS
+    /// - FZF_DEFAULT_OPTS
+    std::string fuzzy_finder_command = fmt::format(
+        "{} --read0 --tac --no-sort --tiebreak=index --bind=ctrl-r:toggle-sort --height=30% < {} > {}",
+        fuzzy_finder, history_file.getPath(), output_file.getPath());
+    char * const argv[] = {sh, sh_c, fuzzy_finder_command.data(), nullptr};
+
+    try
+    {
+        if (executeCommand(argv) == 0)
+        {
+            const std::string & new_query = readFile(output_file.getPath());
+            rx.set_state(replxx::Replxx::State(new_query.c_str(), new_query.size()));
+        }
+    }
+    catch (const std::runtime_error & e)
+    {
+        rx.print(e.what());
+    }
+
+    if (bracketed_paste_enabled)
+        enableBracketedPaste();
 }
 
 void ReplxxLineReader::enableBracketedPaste()

base/base/ReplxxLineReader.h

@@ -27,6 +27,7 @@ private:
     void addToHistory(const String & line) override;
     int executeEditor(const std::string & path);
     void openEditor();
+    void openInteractiveHistorySearch();
 
     replxx::Replxx rx;
     replxx::Replxx::highlighter_callback_t highlighter;
@@ -36,4 +37,5 @@ private:
     bool bracketed_paste_enabled = false;
 
     std::string editor;
+    std::string fuzzy_finder;
 };

cmake/cpu_features.cmake

@@ -11,22 +11,72 @@ cmake_push_check_state ()
 # All of them are unrelated to the instruction set at the host machine
 # (you can compile for newer instruction set on old machines and vice versa).
 
-option (ENABLE_SSSE3 "Use SSSE3 instructions on x86_64" 1)
+option (ARCH_NATIVE "Add -march=native compiler flag. This makes your binaries non-portable but more performant code may be generated. This option overrides ENABLE_* options for specific instruction set. Highly not recommended to use." 0)
-option (ENABLE_SSE41 "Use SSE4.1 instructions on x86_64" 1)
-option (ENABLE_SSE42 "Use SSE4.2 instructions on x86_64" 1)
-option (ENABLE_PCLMULQDQ "Use pclmulqdq instructions on x86_64" 1)
-option (ENABLE_POPCNT "Use popcnt instructions on x86_64" 1)
-option (ENABLE_AVX "Use AVX instructions on x86_64" 0)
-option (ENABLE_AVX2 "Use AVX2 instructions on x86_64" 0)
-option (ENABLE_AVX512 "Use AVX512 instructions on x86_64" 0)
-option (ENABLE_AVX512_VBMI "Use AVX512_VBMI instruction on x86_64 (depends on ENABLE_AVX512)" 0)
-option (ENABLE_BMI "Use BMI instructions on x86_64" 0)
-option (ENABLE_AVX2_FOR_SPEC_OP "Use avx2 instructions for specific operations on x86_64" 0)
-option (ENABLE_AVX512_FOR_SPEC_OP "Use avx512 instructions for specific operations on x86_64" 0)
 
-# X86: Allow compilation for a SSE2-only target machine. Done by a special build in CI for embedded or very old hardware.
+if (ARCH_NATIVE)
-option (NO_SSE3_OR_HIGHER "Disable SSE3 or higher on x86_64" 0)
+    set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
-if (NO_SSE3_OR_HIGHER)
+
+elseif (ARCH_AARCH64)
+    # ARM publishes almost every year a new revision of it's ISA [1]. Each version comes with new mandatory and optional features from
+    # which CPU vendors can pick and choose. This creates a lot of variability ... We provide two build "profiles", one for maximum
+    # compatibility intended to run on all 64-bit ARM hardware released after 2013 (e.g. Raspberry Pi 4), and one for modern ARM server
+    # CPUs, (e.g. Graviton).
+    #
+    # [1] https://en.wikipedia.org/wiki/AArch64
+    option (NO_ARMV81_OR_HIGHER "Disable ARMv8.1 or higher on Aarch64 for maximum compatibility with older/embedded hardware." 0)
+
+    if (NO_ARMV81_OR_HIGHER)
+        # crc32 is optional in v8.0 and mandatory in v8.1. Enable it as __crc32()* is used in lot's of places and even very old ARM CPUs
+        # support it.
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=armv8+crc")
+    else ()
+        # ARMv8.2 is quite ancient but the lowest common denominator supported by both Graviton 2 and 3 processors [1]. In particular, it
+        # includes LSE (made mandatory with ARMv8.1) which provides nice speedups without having to fall back to compat flag
+        # "-moutline-atomics" for v8.0 [2, 3, 4] that requires a recent glibc with runtime dispatch helper, limiting our ability to run on
+        # old OSs.
+        #
+        # simd:    NEON, introduced as optional in v8.0, A few extensions were added with v8.1 but it's still not mandatory. Enables the
+        #          compiler to auto-vectorize.
+        # sve:     Scalable Vector Extensions, introduced as optional in v8.2. Available in Graviton 3 but not in Graviton 2, and most likely
+        #          also not in CI machines. Compiler support for autovectorization is rudimentary at the time of writing, see [5]. Can be
+        #          enabled one-fine-day (TM) but not now.
+        # ssbs:    "Speculative Store Bypass Safe". Optional in v8.0, mandatory in v8.5. Meltdown/spectre countermeasure.
+        # crypto:  SHA1, SHA256, AES. Optional in v8.0. In v8.4, further algorithms were added but it's still optional, see [6].
+        # dotprod: Scalar vector product (SDOT and UDOT instructions). Probably the most obscure extra flag with doubtful performance benefits
+        #          but it has been activated since always, so why not enable it. It's not 100% clear in which revision this flag was
+        #          introduced as optional, either in v8.2 [7] or in v8.4 [8].
+        #
+        # [1] https://github.com/aws/aws-graviton-getting-started/blob/main/c-c%2B%2B.md
+        # [2] https://community.arm.com/arm-community-blogs/b/tools-software-ides-blog/posts/making-the-most-of-the-arm-architecture-in-gcc-10
+        # [3] https://mysqlonarm.github.io/ARM-LSE-and-MySQL/
+        # [4] https://dev.to/aws-builders/large-system-extensions-for-aws-graviton-processors-3eci
+        # [5] https://developer.arm.com/tools-and-software/open-source-software/developer-tools/llvm-toolchain/sve-support
+        # [6] https://developer.arm.com/documentation/100067/0612/armclang-Command-line-Options/-mcpu?lang=en
+        # [7] https://gcc.gnu.org/onlinedocs/gcc/ARM-Options.html
+        # [8] https://developer.arm.com/documentation/102651/a/What-are-dot-product-intructions-
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=armv8.2-a+simd+crypto+dotprod+ssbs")
+    endif ()
+
+elseif (ARCH_PPC64LE)
+    # Note that gcc and clang have support for x86 SSE2 intrinsics when building for PowerPC
+    set (COMPILER_FLAGS "${COMPILER_FLAGS} -maltivec -mcpu=power8 -D__SSE2__=1 -DNO_WARN_X86_INTRINSICS")
+
+elseif (ARCH_AMD64)
+    option (ENABLE_SSSE3 "Use SSSE3 instructions on x86_64" 1)
+    option (ENABLE_SSE41 "Use SSE4.1 instructions on x86_64" 1)
+    option (ENABLE_SSE42 "Use SSE4.2 instructions on x86_64" 1)
+    option (ENABLE_PCLMULQDQ "Use pclmulqdq instructions on x86_64" 1)
+    option (ENABLE_POPCNT "Use popcnt instructions on x86_64" 1)
+    option (ENABLE_AVX "Use AVX instructions on x86_64" 0)
+    option (ENABLE_AVX2 "Use AVX2 instructions on x86_64" 0)
+    option (ENABLE_AVX512 "Use AVX512 instructions on x86_64" 0)
+    option (ENABLE_AVX512_VBMI "Use AVX512_VBMI instruction on x86_64 (depends on ENABLE_AVX512)" 0)
+    option (ENABLE_BMI "Use BMI instructions on x86_64" 0)
+    option (ENABLE_AVX2_FOR_SPEC_OP "Use avx2 instructions for specific operations on x86_64" 0)
+    option (ENABLE_AVX512_FOR_SPEC_OP "Use avx512 instructions for specific operations on x86_64" 0)
+
+    option (NO_SSE3_OR_HIGHER "Disable SSE3 or higher on x86_64 for maximum compatibility with older/embedded hardware." 0)
+    if (NO_SSE3_OR_HIGHER)
         SET(ENABLE_SSSE3 0)
         SET(ENABLE_SSE41 0)
         SET(ENABLE_SSE42 0)
@@ -39,21 +89,8 @@ if (NO_SSE3_OR_HIGHER)
         SET(ENABLE_BMI 0)
         SET(ENABLE_AVX2_FOR_SPEC_OP 0)
         SET(ENABLE_AVX512_FOR_SPEC_OP 0)
-endif()
+    endif()
 
-option (ARCH_NATIVE "Add -march=native compiler flag. This makes your binaries non-portable but more performant code may be generated. This option overrides ENABLE_* options for specific instruction set. Highly not recommended to use." 0)
-
-if (ARCH_NATIVE)
-    set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
-
-elseif (ARCH_AARCH64)
-    set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=armv8-a+crc+simd+crypto+dotprod+ssbs")
-
-elseif (ARCH_PPC64LE)
-    # Note that gcc and clang have support for x86 SSE2 intrinsics when building for PowerPC
-    set (COMPILER_FLAGS "${COMPILER_FLAGS} -maltivec -mcpu=power8 -D__SSE2__=1 -DNO_WARN_X86_INTRINSICS")
-
-elseif (ARCH_AMD64)
     set (TEST_FLAG "-mssse3")
     set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
     check_cxx_source_compiles("

cmake/git.cmake

@@ -0,0 +1,42 @@
+find_package(Git)
+
+# Make basic Git information available as variables. Such data will later be embedded into the build, e.g. for view SYSTEM.BUILD_OPTIONS.
+if (Git_FOUND)
+  # Commit hash + whether the building workspace was dirty or not
+  execute_process(COMMAND
+    "${GIT_EXECUTABLE}" rev-parse HEAD
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_HASH
+    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+  # Branch name
+  execute_process(COMMAND
+    "${GIT_EXECUTABLE}" rev-parse --abbrev-ref HEAD
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_BRANCH
+    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+  # Date of the commit
+  SET(ENV{TZ} "UTC")
+  execute_process(COMMAND
+    "${GIT_EXECUTABLE}" log -1 --format=%ad --date=iso-local
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_DATE
+    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+  # Subject of the commit
+  execute_process(COMMAND
+    "${GIT_EXECUTABLE}" log -1 --format=%s
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_COMMIT_SUBJECT
+    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+  message(STATUS "Git HEAD commit hash: ${GIT_HASH}")
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} status
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_STRIP_TRAILING_WHITESPACE)
+else()
+  message(STATUS "Git could not be found.")
+endif()
+

cmake/git_status.cmake

@@ -1,22 +0,0 @@
-# Print the status of the git repository (if git is available).
-# This is useful for troubleshooting build failure reports
-
-find_package(Git)
-
-if (Git_FOUND)
-
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} rev-parse HEAD
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    OUTPUT_VARIABLE GIT_COMMIT_ID
-    OUTPUT_STRIP_TRAILING_WHITESPACE)
-
-  message(STATUS "HEAD's commit hash ${GIT_COMMIT_ID}")
-
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} status
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_STRIP_TRAILING_WHITESPACE)
-
-else()
-  message(STATUS "Git could not be found.")
-endif()

docker/packager/packager

@@ -128,6 +128,7 @@ def parse_env_variables(
     DARWIN_SUFFIX = "-darwin"
     DARWIN_ARM_SUFFIX = "-darwin-aarch64"
     ARM_SUFFIX = "-aarch64"
+    ARM_V80COMPAT_SUFFIX = "-aarch64-v80compat"
     FREEBSD_SUFFIX = "-freebsd"
     PPC_SUFFIX = "-ppc64le"
     AMD64_SSE2_SUFFIX = "-amd64sse2"
@@ -140,6 +141,7 @@ def parse_env_variables(
     is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
     is_cross_darwin_arm = compiler.endswith(DARWIN_ARM_SUFFIX)
     is_cross_arm = compiler.endswith(ARM_SUFFIX)
+    is_cross_arm_v80compat = compiler.endswith(ARM_V80COMPAT_SUFFIX)
     is_cross_ppc = compiler.endswith(PPC_SUFFIX)
     is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
     is_amd64_sse2 = compiler.endswith(AMD64_SSE2_SUFFIX)
@@ -178,6 +180,13 @@ def parse_env_variables(
             "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake"
         )
         result.append("DEB_ARCH=arm64")
+    elif is_cross_arm_v80compat:
+        cc = compiler[: -len(ARM_V80COMPAT_SUFFIX)]
+        cmake_flags.append(
+            "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake"
+        )
+        cmake_flags.append("-DNO_ARMV81_OR_HIGHER=1")
+        result.append("DEB_ARCH=arm64")
     elif is_cross_freebsd:
         cc = compiler[: -len(FREEBSD_SUFFIX)]
         cmake_flags.append(
@@ -343,6 +352,7 @@ if __name__ == "__main__":
             "clang-15-darwin",
             "clang-15-darwin-aarch64",
             "clang-15-aarch64",
+            "clang-15-aarch64-v80compat",
             "clang-15-ppc64le",
             "clang-15-amd64sse2",
             "clang-15-freebsd",

docker/test/stress/run.sh

@@ -338,6 +338,12 @@ echo $previous_release_tag | download_release_packets && echo -e 'Download scrip
     || echo -e 'Download script failed\tFAIL' >> /test_output/test_results.tsv
 
 mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.clean.log
+for table in query_log trace_log
+do
+    clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.tsv.gz ||:
+done
+
+tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
 
 # Check if we cloned previous release repository successfully
 if ! [ "$(ls -A previous_release_repository/tests/queries)" ]
@@ -454,6 +460,7 @@ else
                -e "This engine is deprecated and is not supported in transactions" \
                -e "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part" \
                -e "The set of parts restored in place of" \
+               -e "(ReplicatedMergeTreeAttachThread): Initialization failed. Error" \
         /var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
         && echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
         || echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
@@ -496,6 +503,12 @@ else
 
     # Remove file bc_check_fatal_messages.txt if it's empty
     [ -s /test_output/bc_check_fatal_messages.txt ] || rm /test_output/bc_check_fatal_messages.txt
+
+    tar -chf /test_output/coordination.backward.tar /var/lib/clickhouse/coordination ||:
+    for table in query_log trace_log
+    do
+        clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.backward.tsv.gz ||:
+    done
 fi
 
 dmesg -T > /test_output/dmesg.log
@@ -505,17 +518,8 @@ grep -q -F -e 'Out of memory: Killed process' -e 'oom_reaper: reaped process' -e
     && echo -e 'OOM in dmesg\tFAIL' >> /test_output/test_results.tsv \
     || echo -e 'No OOM in dmesg\tOK' >> /test_output/test_results.tsv
 
-tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
 mv /var/log/clickhouse-server/stderr.log /test_output/
 
-# Replace the engine with Ordinary to avoid extra symlinks stuff in artifacts.
-# (so that clickhouse-local --path can read it w/o extra care).
-sed -i -e "s/ATTACH DATABASE _ UUID '[^']*'/ATTACH DATABASE system/" -e "s/Atomic/Ordinary/" /var/lib/clickhouse/metadata/system.sql
-for table in query_log trace_log; do
-    sed -i "s/ATTACH TABLE _ UUID '[^']*'/ATTACH TABLE $table/" /var/lib/clickhouse/metadata/system/${table}.sql
-    tar -chf /test_output/${table}_dump.tar /var/lib/clickhouse/metadata/system.sql /var/lib/clickhouse/metadata/system/${table}.sql /var/lib/clickhouse/data/system/${table} ||:
-done
-
 # Write check result into check_status.tsv
 clickhouse-local --structure "test String, res String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' order by (lower(test) like '%hung%'), rowNumberInAllBlocks() LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv
 [ -s /test_output/check_status.tsv ] || echo -e "success\tNo errors found" > /test_output/check_status.tsv

docs/_includes/install/universal.sh

@@ -11,8 +11,17 @@ then
     then
         DIR="amd64"
     elif [ "${ARCH}" = "aarch64" -o "${ARCH}" = "arm64" ]
+    then
+        # If the system has >=ARMv8.2 (https://en.wikipedia.org/wiki/AArch64), choose the corresponding build, else fall back to a v8.0
+        # compat build. Unfortunately, the ARM ISA level cannot be read directly, we need to guess from the "features" in /proc/cpuinfo.
+        # Also, the flags in /proc/cpuinfo are named differently than the flags passed to the compiler (cmake/cpu_features.cmake).
+        ARMV82=$(grep -m 1 'Features' /proc/cpuinfo | awk '/asimd/ && /sha1/ && /aes/ && /atomics/')
+        if [ "${ARMV82}" ]
         then
             DIR="aarch64"
+        else
+            DIR="aarch64v80compat"
+        fi
     elif [ "${ARCH}" = "powerpc64le" -o "${ARCH}" = "ppc64le" ]
     then
         DIR="powerpc64le"
@@ -22,12 +31,6 @@ then
     if [ "${ARCH}" = "x86_64" -o "${ARCH}" = "amd64" ]
     then
         DIR="freebsd"
-    elif [ "${ARCH}" = "aarch64" -o "${ARCH}" = "arm64" ]
-    then
-        DIR="freebsd-aarch64"
-    elif [ "${ARCH}" = "powerpc64le" -o "${ARCH}" = "ppc64le" ]
-    then
-        DIR="freebsd-powerpc64le"
     fi
 elif [ "${OS}" = "Darwin" ]
 then
@@ -42,7 +45,7 @@ fi
 
 if [ -z "${DIR}" ]
 then
-    echo "The '${OS}' operating system with the '${ARCH}' architecture is not supported."
+    echo "Operating system '${OS}' / architecture '${ARCH}' is unsupported."
     exit 1
 fi
 

docs/en/sql-reference/statements/create/view.md

@@ -6,7 +6,7 @@ sidebar_label: VIEW
 
 # CREATE VIEW
 
-Creates a new view. Views can be [normal](#normal), [materialized](#materialized), [live](#live-view), and [window](#window-view) (live view and window view are experimental features).
+Creates a new view. Views can be [normal](#normal-view), [materialized](#materialized-view), [live](#live-view-experimental), and [window](#window-view-experimental) (live view and window view are experimental features).
 
 ## Normal View
 

docs/ru/engines/table-engines/special/external-data.md

@@ -22,17 +22,17 @@ ClickHouse позволяет отправить на сервер данные,
 
 Таких секций может быть несколько - по числу передаваемых таблиц.
 
-**–external** - маркер начала секции.
+- **--external** - маркер начала секции. 
-**–file** - путь к файлу с дампом таблицы, или -, что обозначает stdin.
+- **--file** - путь к файлу с дампом таблицы, или `-`, что обозначает `stdin`. 
-Из stdin может быть считана только одна таблица.
+Из `stdin` может быть считана только одна таблица.
 
 Следующие параметры не обязательные:
-**–name** - имя таблицы. Если не указано - используется _data.
+- **--name** - имя таблицы. Если не указано - используется _data. 
-**–format** - формат данных в файле. Если не указано - используется TabSeparated.
+- **--format** - формат данных в файле. Если не указано - используется TabSeparated. 
 
 Должен быть указан один из следующих параметров:
-**–types** - список типов столбцов через запятую. Например, `UInt64,String`. Столбцы будут названы _1, _2, …
+- **--types** - список типов столбцов через запятую. Например, `UInt64,String`. Столбцы будут названы _1, _2, … 
-**–structure** - структура таблицы, в форме `UserID UInt64`, `URL String`. Определяет имена и типы столбцов.
+- **--structure** - структура таблицы, в форме `UserID UInt64`, `URL String`. Определяет имена и типы столбцов. 
 
 Файлы, указанные в file, будут разобраны форматом, указанным в format, с использованием типов данных, указанных в types или structure. Таблица будет загружена на сервер, и доступна там в качестве временной таблицы с именем name.
 

programs/keeper/CMakeLists.txt

@@ -33,7 +33,7 @@ install(FILES keeper_config.xml DESTINATION "${CLICKHOUSE_ETC_DIR}/clickhouse-ke
 add_dependencies(clickhouse-keeper-lib clickhouse_keeper_configs)
 
 if (BUILD_STANDALONE_KEEPER)
-    # Sraight list of all required sources
+    # Straight list of all required sources
     set(CLICKHOUSE_KEEPER_STANDALONE_SOURCES
         ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/ACLMap.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/Changelog.cpp
@@ -92,6 +92,7 @@ if (BUILD_STANDALONE_KEEPER)
         ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Daemon/BaseDaemon.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Daemon/SentryWriter.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Daemon/GraphiteWriter.cpp
+        ${CMAKE_CURRENT_BINARY_DIR}/../../src/Daemon/GitHash.generated.cpp
 
         Keeper.cpp
         TinyContext.cpp

programs/keeper/Keeper.cpp

@@ -490,8 +490,9 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
 void Keeper::logRevision() const
 {
     Poco::Logger::root().information("Starting ClickHouse Keeper " + std::string{VERSION_STRING}
-        + " with revision " + std::to_string(ClickHouseRevision::getVersionRevision())
+        + "(revision : " + std::to_string(ClickHouseRevision::getVersionRevision())
-        + ", " + build_id_info
+        + ", git hash: " + (git_hash.empty() ? "<unknown>" : git_hash)
+        + ", build id: " + (build_id.empty() ? "<unknown>" : build_id) + ")"
         + ", PID " + std::to_string(getpid()));
 }
 

programs/server/config.xml

@@ -1114,10 +1114,6 @@
     <asynchronous_metric_log>
         <database>system</database>
         <table>asynchronous_metric_log</table>
-        <!--
-            Asynchronous metrics are updated once a minute, so there is
-            no need to flush more often.
-        -->
         <flush_interval_milliseconds>7000</flush_interval_milliseconds>
     </asynchronous_metric_log>
 

programs/server/dashboard.html

@@ -820,7 +820,7 @@ async function draw(idx, chart, url_params, query) {
     sync.sub(plots[idx]);
 
     /// Set title
-    const title = queries[idx].title.replaceAll(/\{(\w+)\}/g, (_, name) => params[name] );
+    const title = queries[idx].title ? queries[idx].title.replaceAll(/\{(\w+)\}/g, (_, name) => params[name] ) : '';
     chart.querySelector('.title').firstChild.data = title;
 }
 

programs/static-files-disk-uploader/static-files-disk-uploader.cpp

@@ -58,7 +58,9 @@ void processFile(const fs::path & file_path, const fs::path & dst_path, bool tes
     }
     else
     {
-        auto src_buf = createReadBufferFromFileBase(file_path, {}, fs::file_size(file_path));
+        ReadSettings read_settings{};
+        read_settings.local_fs_method = LocalFSReadMethod::pread;
+        auto src_buf = createReadBufferFromFileBase(file_path, read_settings, fs::file_size(file_path));
         std::shared_ptr<WriteBuffer> dst_buf;
 
         /// test mode for integration tests.

src/Client/ClientBase.cpp

@@ -70,7 +70,7 @@
 #include <Client/InternalTextLogs.h>
 #include <boost/algorithm/string/replace.hpp>
 #include <IO/ForkWriteBuffer.h>
-
+#include <Parsers/Kusto/ParserKQLStatement.h>
 
 namespace fs = std::filesystem;
 using namespace std::literals;
@@ -292,7 +292,7 @@ void ClientBase::setupSignalHandler()
 
 ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, bool allow_multi_statements) const
 {
-    ParserQuery parser(end, global_context->getSettings().allow_settings_after_format_in_insert);
+    std::unique_ptr<IParserBase> parser;
     ASTPtr res;
 
     const auto & settings = global_context->getSettingsRef();
@@ -301,10 +301,17 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, bool allow_mu
     if (!allow_multi_statements)
         max_length = settings.max_query_size;
 
+    const Dialect & dialect = settings.dialect;
+
+    if (dialect == Dialect::kusto)
+        parser = std::make_unique<ParserKQLStatement>(end, global_context->getSettings().allow_settings_after_format_in_insert);
+    else
+        parser = std::make_unique<ParserQuery>(end, global_context->getSettings().allow_settings_after_format_in_insert);
+
     if (is_interactive || ignore_error)
     {
         String message;
-        res = tryParseQuery(parser, pos, end, message, true, "", allow_multi_statements, max_length, settings.max_parser_depth);
+        res = tryParseQuery(*parser, pos, end, message, true, "", allow_multi_statements, max_length, settings.max_parser_depth);
 
         if (!res)
         {
@@ -314,7 +321,7 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, bool allow_mu
     }
     else
     {
-        res = parseQueryAndMovePosition(parser, pos, end, "", allow_multi_statements, max_length, settings.max_parser_depth);
+        res = parseQueryAndMovePosition(*parser, pos, end, "", allow_multi_statements, max_length, settings.max_parser_depth);
     }
 
     if (is_interactive)

src/Client/ClientBase.h

@@ -198,6 +198,7 @@ protected:
     SharedContextHolder shared_context;
     ContextMutablePtr global_context;
 
+    /// thread status should be destructed before shared context because it relies on process list.
     std::optional<ThreadStatus> thread_status;
 
     ServerConnectionPtr connection;

src/Client/Connection.cpp

@@ -179,6 +179,9 @@ void Connection::connect(const ConnectionTimeouts & timeouts)
     {
         disconnect();
 
+        /// Remove this possible stale entry from cache
+        DNSResolver::instance().removeHostFromCache(host);
+
         /// Add server address to exception. Also Exception will remember stack trace. It's a pity that more precise exception type is lost.
         throw NetException(e.displayText() + " (" + getDescription() + ")", ErrorCodes::NETWORK_ERROR);
     }
@@ -186,6 +189,9 @@ void Connection::connect(const ConnectionTimeouts & timeouts)
     {
         disconnect();
 
+        /// Remove this possible stale entry from cache
+        DNSResolver::instance().removeHostFromCache(host);
+
         /// Add server address to exception. Also Exception will remember stack trace. It's a pity that more precise exception type is lost.
         /// This exception can only be thrown from socket->connect(), so add information about connection timeout.
         const auto & connection_timeout = static_cast<bool>(secure) ? timeouts.secure_connection_timeout : timeouts.connection_timeout;

src/Columns/ColumnVector.cpp

@@ -12,14 +12,12 @@
 #include <Common/RadixSort.h>
 #include <Common/SipHash.h>
 #include <Common/WeakHash.h>
-#include <Common/TargetSpecific.h>
 #include <Common/assert_cast.h>
 #include <base/sort.h>
 #include <base/unaligned.h>
 #include <base/bit_cast.h>
 #include <base/scope_guard.h>
 
-#include <bit>
 #include <cmath>
 #include <cstring>
 
@@ -27,10 +25,6 @@
 #    include <emmintrin.h>
 #endif
 
-#if USE_MULTITARGET_CODE
-#    include <immintrin.h>
-#endif
-
 #if USE_EMBEDDED_COMPILER
 #include <DataTypes/Native.h>
 #include <llvm/IR/IRBuilder.h>
@@ -477,128 +471,6 @@ void ColumnVector<T>::insertRangeFrom(const IColumn & src, size_t start, size_t
     memcpy(data.data() + old_size, &src_vec.data[start], length * sizeof(data[0]));
 }
 
-static inline UInt64 blsr(UInt64 mask)
-{
-#ifdef __BMI__
-    return _blsr_u64(mask);
-#else
-    return mask & (mask-1);
-#endif
-}
-
-DECLARE_DEFAULT_CODE(
-template <typename T, typename Container, size_t SIMD_BYTES>
-inline void doFilterAligned(const UInt8 *& filt_pos, const UInt8 *& filt_end_aligned, const T *& data_pos, Container & res_data)
-{
-    while (filt_pos < filt_end_aligned)
-    {
-        UInt64 mask = bytes64MaskToBits64Mask(filt_pos);
-
-        if (0xffffffffffffffff == mask)
-        {
-            res_data.insert(data_pos, data_pos + SIMD_BYTES);
-        }
-        else
-        {
-            while (mask)
-            {
-                size_t index = std::countr_zero(mask);
-                res_data.push_back(data_pos[index]);
-                mask = blsr(mask);
-            }
-        }
-
-        filt_pos += SIMD_BYTES;
-        data_pos += SIMD_BYTES;
-    }
-}
-)
-
-namespace
-{
-template <typename T, typename Container>
-void resize(Container & res_data, size_t reserve_size)
-{
-#if defined(MEMORY_SANITIZER)
-    res_data.resize_fill(reserve_size, static_cast<T>(0)); // MSan doesn't recognize that all allocated memory is written by AVX-512 intrinsics.
-#else
-    res_data.resize(reserve_size);
-#endif
-}
-}
-
-DECLARE_AVX512VBMI2_SPECIFIC_CODE(
-template <size_t ELEMENT_WIDTH>
-inline void compressStoreAVX512(const void *src, void *dst, const UInt64 mask)
-{
-    __m512i vsrc = _mm512_loadu_si512(src);
-    if constexpr (ELEMENT_WIDTH == 1)
-        _mm512_mask_compressstoreu_epi8(dst, static_cast<__mmask64>(mask), vsrc);
-    else if constexpr (ELEMENT_WIDTH == 2)
-        _mm512_mask_compressstoreu_epi16(dst, static_cast<__mmask32>(mask), vsrc);
-    else if constexpr (ELEMENT_WIDTH == 4)
-        _mm512_mask_compressstoreu_epi32(dst, static_cast<__mmask16>(mask), vsrc);
-    else if constexpr (ELEMENT_WIDTH == 8)
-        _mm512_mask_compressstoreu_epi64(dst, static_cast<__mmask8>(mask), vsrc);
-}
-
-template <typename T, typename Container, size_t SIMD_BYTES>
-inline void doFilterAligned(const UInt8 *& filt_pos, const UInt8 *& filt_end_aligned, const T *& data_pos, Container & res_data)
-{
-    static constexpr size_t VEC_LEN = 64;   /// AVX512 vector length - 64 bytes
-    static constexpr size_t ELEMENT_WIDTH = sizeof(T);
-    static constexpr size_t ELEMENTS_PER_VEC = VEC_LEN / ELEMENT_WIDTH;
-    static constexpr UInt64 KMASK = 0xffffffffffffffff >> (64 - ELEMENTS_PER_VEC);
-
-    size_t current_offset = res_data.size();
-    size_t reserve_size = res_data.size();
-    size_t alloc_size = SIMD_BYTES * 2;
-
-    while (filt_pos < filt_end_aligned)
-    {
-        /// to avoid calling resize too frequently, resize to reserve buffer.
-        if (reserve_size - current_offset < SIMD_BYTES)
-        {
-            reserve_size += alloc_size;
-            resize<T>(res_data, reserve_size);
-            alloc_size *= 2;
-        }
-
-        UInt64 mask = bytes64MaskToBits64Mask(filt_pos);
-
-        if (0xffffffffffffffff == mask)
-        {
-            for (size_t i = 0; i < SIMD_BYTES; i += ELEMENTS_PER_VEC)
-                _mm512_storeu_si512(reinterpret_cast<void *>(&res_data[current_offset + i]),
-                        _mm512_loadu_si512(reinterpret_cast<const void *>(data_pos + i)));
-            current_offset += SIMD_BYTES;
-        }
-        else
-        {
-            if (mask)
-            {
-                for (size_t i = 0; i < SIMD_BYTES; i += ELEMENTS_PER_VEC)
-                {
-                    compressStoreAVX512<ELEMENT_WIDTH>(reinterpret_cast<const void *>(data_pos + i),
-                            reinterpret_cast<void *>(&res_data[current_offset]), mask & KMASK);
-                    current_offset += std::popcount(mask & KMASK);
-                    /// prepare mask for next iter, if ELEMENTS_PER_VEC = 64, no next iter
-                    if (ELEMENTS_PER_VEC < 64)
-                    {
-                        mask >>= ELEMENTS_PER_VEC;
-                    }
-                }
-            }
-        }
-
-        filt_pos += SIMD_BYTES;
-        data_pos += SIMD_BYTES;
-    }
-    /// resize to the real size.
-    res_data.resize(current_offset);
-}
-)
-
 template <typename T>
 ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
 {
@@ -624,13 +496,31 @@ ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_s
     static constexpr size_t SIMD_BYTES = 64;
     const UInt8 * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
 
-#if USE_MULTITARGET_CODE
+    while (filt_pos < filt_end_aligned)
-    static constexpr bool VBMI2_CAPABLE = sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4 || sizeof(T) == 8;
+    {
-    if (VBMI2_CAPABLE && isArchSupported(TargetArch::AVX512VBMI2))
+        UInt64 mask = bytes64MaskToBits64Mask(filt_pos);
-        TargetSpecific::AVX512VBMI2::doFilterAligned<T, Container, SIMD_BYTES>(filt_pos, filt_end_aligned, data_pos, res_data);
+
+        if (0xffffffffffffffff == mask)
+        {
+            res_data.insert(data_pos, data_pos + SIMD_BYTES);
+        }
         else
-#endif
+        {
-        TargetSpecific::Default::doFilterAligned<T, Container, SIMD_BYTES>(filt_pos, filt_end_aligned, data_pos, res_data);
+            while (mask)
+            {
+                size_t index = std::countr_zero(mask);
+                res_data.push_back(data_pos[index]);
+            #ifdef __BMI__
+                mask = _blsr_u64(mask);
+            #else
+                mask = mask & (mask-1);
+            #endif
+            }
+        }
+
+        filt_pos += SIMD_BYTES;
+        data_pos += SIMD_BYTES;
+    }
 
     while (filt_pos < filt_end)
     {

src/Columns/ColumnVector.h

@@ -7,15 +7,11 @@
 #include <base/unaligned.h>
 #include <Core/Field.h>
 #include <Common/assert_cast.h>
-#include <Common/TargetSpecific.h>
 #include <Core/TypeId.h>
 #include <base/TypeName.h>
 
 #include "config_core.h"
 
-#if USE_MULTITARGET_CODE
-#    include <immintrin.h>
-#endif
 
 namespace DB
 {
@@ -395,124 +391,6 @@ protected:
     Container data;
 };
 
-DECLARE_DEFAULT_CODE(
-template <typename Container, typename Type>
-inline void vectorIndexImpl(const Container & data, const PaddedPODArray<Type> & indexes, size_t limit, Container & res_data)
-{
-    for (size_t i = 0; i < limit; ++i)
-        res_data[i] = data[indexes[i]];
-}
-);
-
-DECLARE_AVX512VBMI_SPECIFIC_CODE(
-template <typename Container, typename Type>
-inline void vectorIndexImpl(const Container & data, const PaddedPODArray<Type> & indexes, size_t limit, Container & res_data)
-{
-    static constexpr UInt64 MASK64 = 0xffffffffffffffff;
-    const size_t limit64 = limit & ~63;
-    size_t pos = 0;
-    size_t data_size = data.size();
-
-    auto data_pos = reinterpret_cast<const UInt8 *>(data.data());
-    auto indexes_pos = reinterpret_cast<const UInt8 *>(indexes.data());
-    auto res_pos = reinterpret_cast<UInt8 *>(res_data.data());
-
-    if (data_size <= 64)
-    {
-        /// one single mask load for table size <= 64
-        __mmask64 last_mask = MASK64 >> (64 - data_size);
-        __m512i table1 = _mm512_maskz_loadu_epi8(last_mask, data_pos);
-
-        /// 64 bytes table lookup using one single permutexvar_epi8
-        while (pos < limit64)
-        {
-            __m512i vidx = _mm512_loadu_epi8(indexes_pos + pos);
-            __m512i out = _mm512_permutexvar_epi8(vidx, table1);
-            _mm512_storeu_epi8(res_pos + pos, out);
-            pos += 64;
-        }
-        /// tail handling
-        if (limit > limit64)
-        {
-            __mmask64 tail_mask = MASK64 >> (limit64 + 64 - limit);
-            __m512i vidx = _mm512_maskz_loadu_epi8(tail_mask, indexes_pos + pos);
-            __m512i out = _mm512_permutexvar_epi8(vidx, table1);
-            _mm512_mask_storeu_epi8(res_pos + pos, tail_mask, out);
-        }
-    }
-    else if (data_size <= 128)
-    {
-        /// table size (64, 128] requires 2 zmm load
-        __mmask64 last_mask = MASK64 >> (128 - data_size);
-        __m512i table1 = _mm512_loadu_epi8(data_pos);
-        __m512i table2 = _mm512_maskz_loadu_epi8(last_mask, data_pos + 64);
-
-        /// 128 bytes table lookup using one single permute2xvar_epi8
-        while (pos < limit64)
-        {
-            __m512i vidx = _mm512_loadu_epi8(indexes_pos + pos);
-            __m512i out = _mm512_permutex2var_epi8(table1, vidx, table2);
-            _mm512_storeu_epi8(res_pos + pos, out);
-            pos += 64;
-        }
-        if (limit > limit64)
-        {
-            __mmask64 tail_mask = MASK64 >> (limit64 + 64 - limit);
-            __m512i vidx = _mm512_maskz_loadu_epi8(tail_mask, indexes_pos + pos);
-            __m512i out = _mm512_permutex2var_epi8(table1, vidx, table2);
-            _mm512_mask_storeu_epi8(res_pos + pos, tail_mask, out);
-        }
-    }
-    else
-    {
-        if (data_size > 256)
-        {
-            /// byte index will not exceed 256 boundary.
-            data_size = 256;
-        }
-
-        __m512i table1 = _mm512_loadu_epi8(data_pos);
-        __m512i table2 = _mm512_loadu_epi8(data_pos + 64);
-        __m512i table3, table4;
-        if (data_size <= 192)
-        {
-            /// only 3 tables need to load if size <= 192
-            __mmask64 last_mask = MASK64 >> (192 - data_size);
-            table3 = _mm512_maskz_loadu_epi8(last_mask, data_pos + 128);
-            table4 = _mm512_setzero_si512();
-        }
-        else
-        {
-            __mmask64 last_mask = MASK64 >> (256 - data_size);
-            table3 = _mm512_loadu_epi8(data_pos + 128);
-            table4 = _mm512_maskz_loadu_epi8(last_mask, data_pos + 192);
-        }
-
-        /// 256 bytes table lookup can use: 2 permute2xvar_epi8 plus 1 blender with MSB
-        while (pos < limit64)
-        {
-            __m512i vidx = _mm512_loadu_epi8(indexes_pos + pos);
-            __m512i tmp1 = _mm512_permutex2var_epi8(table1, vidx, table2);
-            __m512i tmp2 = _mm512_permutex2var_epi8(table3, vidx, table4);
-            __mmask64 msb = _mm512_movepi8_mask(vidx);
-            __m512i out = _mm512_mask_blend_epi8(msb, tmp1, tmp2);
-            _mm512_storeu_epi8(res_pos + pos, out);
-            pos += 64;
-        }
-        if (limit > limit64)
-        {
-            __mmask64 tail_mask = MASK64 >> (limit64 + 64 - limit);
-            __m512i vidx = _mm512_maskz_loadu_epi8(tail_mask, indexes_pos + pos);
-            __m512i tmp1 = _mm512_permutex2var_epi8(table1, vidx, table2);
-            __m512i tmp2 = _mm512_permutex2var_epi8(table3, vidx, table4);
-            __mmask64 msb = _mm512_movepi8_mask(vidx);
-            __m512i out = _mm512_mask_blend_epi8(msb, tmp1, tmp2);
-            _mm512_mask_storeu_epi8(res_pos + pos, tail_mask, out);
-        }
-    }
-}
-);
-
 template <typename T>
 template <typename Type>
 ColumnPtr ColumnVector<T>::indexImpl(const PaddedPODArray<Type> & indexes, size_t limit) const
@@ -521,18 +399,8 @@ ColumnPtr ColumnVector<T>::indexImpl(const PaddedPODArray<Type> & indexes, size_
 
     auto res = this->create(limit);
     typename Self::Container & res_data = res->getData();
-#if USE_MULTITARGET_CODE
+    for (size_t i = 0; i < limit; ++i)
-    if constexpr (sizeof(T) == 1 && sizeof(Type) == 1)
+        res_data[i] = data[indexes[i]];
-    {
-        /// VBMI optimization only applicable for (U)Int8 types
-        if (isArchSupported(TargetArch::AVX512VBMI))
-        {
-            TargetSpecific::AVX512VBMI::vectorIndexImpl<Container, Type>(data, indexes, limit, res_data);
-            return res;
-        }
-    }
-#endif
-    TargetSpecific::Default::vectorIndexImpl<Container, Type>(data, indexes, limit, res_data);
 
     return res;
 }

src/Columns/tests/gtest_column_vector.cpp

@@ -1,158 +0,0 @@
-#include <limits>
-#include <typeinfo>
-#include <vector>
-#include <Columns/ColumnsNumber.h>
-#include <Common/randomSeed.h>
-#include <gtest/gtest.h>
-
-using namespace DB;
-
-static pcg64 rng(randomSeed());
-static constexpr int error_code = 12345;
-static constexpr size_t TEST_RUNS = 500;
-static constexpr size_t MAX_ROWS = 10000;
-static const std::vector<size_t> filter_ratios = {1, 2, 5, 11, 32, 64, 100, 1000};
-static const size_t K = filter_ratios.size();
-
-template <typename T>
-static MutableColumnPtr createColumn(size_t n)
-{
-    auto column = ColumnVector<T>::create();
-    auto & values = column->getData();
-
-    for (size_t i = 0; i < n; ++i)
-    {
-        values.push_back(i);
-    }
-
-    return column;
-}
-
-bool checkFilter(const PaddedPODArray<UInt8> &flit, const IColumn & src, const IColumn & dst)
-{
-    size_t n = flit.size();
-    size_t dst_size = dst.size();
-    size_t j = 0;   /// index of dest
-    for (size_t i = 0; i < n; ++i)
-    {
-        if (flit[i] != 0)
-        {
-            if ((dst_size <= j) || (src.compareAt(i, j, dst, 0) != 0))
-                return false;
-            j++;
-        }
-    }
-    return dst_size == j;   /// filtered size check
-}
-
-template <typename T>
-static void testFilter()
-{
-    auto test_case = [&](size_t rows, size_t filter_ratio)
-    {
-        auto vector_column = createColumn<T>(rows);
-        PaddedPODArray<UInt8> flit(rows);
-        for (size_t i = 0; i < rows; ++i)
-            flit[i] = rng() % filter_ratio == 0;
-        auto res_column = vector_column->filter(flit, -1);
-
-        if (!checkFilter(flit, *vector_column, *res_column))
-            throw Exception(error_code, "VectorColumn filter failure, type: {}", typeid(T).name());
-    };
-
-    try
-    {
-        for (size_t i = 0; i < TEST_RUNS; ++i)
-        {
-            size_t rows = rng() % MAX_ROWS + 1;
-            size_t filter_ratio = filter_ratios[rng() % K];
-
-            test_case(rows, filter_ratio);
-        }
-    }
-    catch (const Exception & e)
-    {
-        FAIL() << e.displayText();
-    }
-}
-
-TEST(ColumnVector, Filter)
-{
-    testFilter<UInt8>();
-    testFilter<Int16>();
-    testFilter<UInt32>();
-    testFilter<Int64>();
-    testFilter<UInt128>();
-    testFilter<Int256>();
-    testFilter<Float32>();
-    testFilter<Float64>();
-    testFilter<UUID>();
-}
-
-template <typename T>
-static MutableColumnPtr createIndexColumn(size_t limit, size_t rows)
-{
-    auto column = ColumnVector<T>::create();
-    auto & values = column->getData();
-    auto max = std::numeric_limits<T>::max();
-    limit = limit > max ? max : limit;
-
-    for (size_t i = 0; i < rows; ++i)
-    {
-        T val = rng() % limit;
-        values.push_back(val);
-    }
-
-    return column;
-}
-
-template <typename T, typename IndexType>
-static void testIndex()
-{
-    static const std::vector<size_t> column_sizes = {64, 128, 196, 256, 512};
-
-    auto test_case = [&](size_t rows, size_t index_rows, size_t limit)
-    {
-        auto vector_column = createColumn<T>(rows);
-        auto index_column = createIndexColumn<IndexType>(rows, index_rows);
-        auto res_column = vector_column->index(*index_column, limit);
-        if (limit == 0)
-            limit = index_column->size();
-
-        /// check results
-        if (limit != res_column->size())
-            throw Exception(error_code, "ColumnVector index size not match to limit: {} {}", typeid(T).name(), typeid(IndexType).name());
-        for (size_t i = 0; i < limit; ++i)
-        {
-            /// vector_column data is the same as index, so indexed column's value will equals to index_column.
-            if (res_column->get64(i) != index_column->get64(i))
-                throw Exception(error_code, "ColumnVector index fail: {} {}", typeid(T).name(), typeid(IndexType).name());
-        }
-    };
-
-    try
-    {
-        for (size_t i = 0; i < TEST_RUNS; ++i)
-        {
-            /// make sure rows distribute in (column_sizes[r-1], colulmn_sizes[r]]
-            size_t row_idx = rng() % column_sizes.size();
-            size_t row_base = row_idx > 0 ? column_sizes[row_idx - 1] : 0;
-            size_t rows = row_base + (rng() % (column_sizes[row_idx] - row_base) + 1);
-            size_t index_rows = rng() % MAX_ROWS + 1;
-
-            test_case(rows, index_rows, 0);
-            test_case(rows, index_rows, static_cast<size_t>(0.5 * index_rows));
-        }
-    }
-    catch (const Exception & e)
-    {
-        FAIL() << e.displayText();
-    }
-}
-
-TEST(ColumnVector, Index)
-{
-    testIndex<UInt8, UInt8>();
-    testIndex<UInt16, UInt8>();
-    testIndex<UInt16, UInt16>();
-}

src/Common/CpuId.h

@@ -82,7 +82,6 @@ inline bool cpuid(UInt32 op, UInt32 * res) noexcept /// NOLINT
     OP(AVX512BW)             \
     OP(AVX512VL)             \
     OP(AVX512VBMI)           \
-    OP(AVX512VBMI2)          \
     OP(PREFETCHWT1)          \
     OP(SHA)                  \
     OP(ADX)                  \
@@ -303,11 +302,6 @@ bool haveAVX512VBMI() noexcept
     return haveAVX512F() && ((CpuInfo(0x7, 0).registers.ecx >> 1) & 1u);
 }
 
-bool haveAVX512VBMI2() noexcept
-{
-    return haveAVX512F() && ((CpuInfo(0x7, 0).registers.ecx >> 6) & 1u);
-}
-
 bool haveRDRAND() noexcept
 {
     return CpuInfo(0x0).registers.eax >= 0x7 && ((CpuInfo(0x1).registers.ecx >> 30) & 1u);

src/Common/DNSResolver.cpp

@@ -1,7 +1,8 @@
 #include "DNSResolver.h"
-#include <base/CachedFn.h>
+#include <Common/CacheBase.h>
 #include <Common/Exception.h>
 #include <Common/ProfileEvents.h>
+#include <Common/thread_local_rng.h>
 #include <Core/Names.h>
 #include <base/types.h>
 #include <Poco/Net/IPAddress.h>
@@ -12,6 +13,7 @@
 #include <atomic>
 #include <optional>
 #include <string_view>
+#include <unordered_set>
 #include "DNSPTRResolverProvider.h"
 
 namespace ProfileEvents
@@ -41,9 +43,11 @@ namespace ErrorCodes
     extern const int DNS_ERROR;
 }
 
+namespace
+{
 
 /// Slightly altered implementation from https://github.com/pocoproject/poco/blob/poco-1.6.1/Net/src/SocketAddress.cpp#L86
-static void splitHostAndPort(const std::string & host_and_port, std::string & out_host, UInt16 & out_port)
+void splitHostAndPort(const std::string & host_and_port, std::string & out_host, UInt16 & out_port)
 {
     String port_str;
     out_host.clear();
@@ -84,7 +88,7 @@ static void splitHostAndPort(const std::string & host_and_port, std::string & ou
         throw Exception("Port must be numeric", ErrorCodes::BAD_ARGUMENTS);
 }
 
-static DNSResolver::IPAddresses hostByName(const std::string & host)
+DNSResolver::IPAddresses hostByName(const std::string & host)
 {
     /// Do not resolve IPv6 (or IPv4) if no local IPv6 (or IPv4) addresses are configured.
     /// It should not affect client address checking, since client cannot connect from IPv6 address
@@ -112,7 +116,7 @@ static DNSResolver::IPAddresses hostByName(const std::string & host)
     return addresses;
 }
 
-static DNSResolver::IPAddresses resolveIPAddressImpl(const std::string & host)
+DNSResolver::IPAddresses resolveIPAddressImpl(const std::string & host)
 {
     Poco::Net::IPAddress ip;
 
@@ -136,7 +140,13 @@ static DNSResolver::IPAddresses resolveIPAddressImpl(const std::string & host)
     return addresses;
 }
 
-static std::unordered_set<String> reverseResolveImpl(const Poco::Net::IPAddress & address)
+DNSResolver::IPAddresses resolveIPAddressWithCache(CacheBase<std::string, DNSResolver::IPAddresses> & cache, const std::string & host)
+{
+    auto [result, _ ] = cache.getOrSet(host, [&host]() { return std::make_shared<DNSResolver::IPAddresses>(resolveIPAddressImpl(host)); });
+    return *result;
+}
+
+std::unordered_set<String> reverseResolveImpl(const Poco::Net::IPAddress & address)
 {
     auto ptr_resolver = DB::DNSPTRResolverProvider::get();
 
@@ -149,13 +159,27 @@ static std::unordered_set<String> reverseResolveImpl(const Poco::Net::IPAddress
     }
 }
 
+std::unordered_set<String> reverseResolveWithCache(
+    CacheBase<Poco::Net::IPAddress, std::unordered_set<std::string>> & cache, const Poco::Net::IPAddress & address)
+{
+    auto [result, _ ] = cache.getOrSet(address, [&address]() { return std::make_shared<std::unordered_set<String>>(reverseResolveImpl(address)); });
+    return *result;
+}
+
+Poco::Net::IPAddress pickAddress(const DNSResolver::IPAddresses & addresses)
+{
+    return addresses.front();
+}
+
+}
+
 struct DNSResolver::Impl
 {
     using HostWithConsecutiveFailures = std::unordered_map<String, UInt32>;
     using AddressWithConsecutiveFailures = std::unordered_map<Poco::Net::IPAddress, UInt32>;
 
-    CachedFn<&resolveIPAddressImpl> cache_host;
+    CacheBase<std::string, DNSResolver::IPAddresses> cache_host{100};
-    CachedFn<&reverseResolveImpl> cache_address;
+    CacheBase<Poco::Net::IPAddress, std::unordered_set<std::string>> cache_address{100};
 
     std::mutex drop_mutex;
     std::mutex update_mutex;
@@ -180,7 +204,7 @@ DNSResolver::DNSResolver() : impl(std::make_unique<DNSResolver::Impl>()), log(&P
 
 Poco::Net::IPAddress DNSResolver::resolveHost(const std::string & host)
 {
-    return resolveHostAll(host).front();
+    return pickAddress(resolveHostAll(host));
 }
 
 DNSResolver::IPAddresses DNSResolver::resolveHostAll(const std::string & host)
@@ -189,7 +213,7 @@ DNSResolver::IPAddresses DNSResolver::resolveHostAll(const std::string & host)
         return resolveIPAddressImpl(host);
 
     addToNewHosts(host);
-    return impl->cache_host(host);
+    return resolveIPAddressWithCache(impl->cache_host, host);
 }
 
 Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host_and_port)
@@ -202,7 +226,7 @@ Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host_an
     splitHostAndPort(host_and_port, host, port);
 
     addToNewHosts(host);
-    return Poco::Net::SocketAddress(impl->cache_host(host).front(), port);
+    return Poco::Net::SocketAddress(pickAddress(resolveIPAddressWithCache(impl->cache_host, host)), port);
 }
 
 Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host, UInt16 port)
@@ -211,7 +235,7 @@ Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host, U
         return Poco::Net::SocketAddress(host, port);
 
     addToNewHosts(host);
-    return  Poco::Net::SocketAddress(impl->cache_host(host).front(), port);
+    return Poco::Net::SocketAddress(pickAddress(resolveIPAddressWithCache(impl->cache_host, host)), port);
 }
 
 std::vector<Poco::Net::SocketAddress> DNSResolver::resolveAddressList(const std::string & host, UInt16 port)
@@ -224,7 +248,7 @@ std::vector<Poco::Net::SocketAddress> DNSResolver::resolveAddressList(const std:
     if (!impl->disable_cache)
         addToNewHosts(host);
 
-    std::vector<Poco::Net::IPAddress> ips = impl->disable_cache ? hostByName(host) : impl->cache_host(host);
+    std::vector<Poco::Net::IPAddress> ips = impl->disable_cache ? hostByName(host) : resolveIPAddressWithCache(impl->cache_host, host);
     auto ips_end = std::unique(ips.begin(), ips.end());
 
     addresses.reserve(ips_end - ips.begin());
@@ -240,13 +264,13 @@ std::unordered_set<String> DNSResolver::reverseResolve(const Poco::Net::IPAddres
         return reverseResolveImpl(address);
 
     addToNewAddresses(address);
-    return impl->cache_address(address);
+    return reverseResolveWithCache(impl->cache_address, address);
 }
 
 void DNSResolver::dropCache()
 {
-    impl->cache_host.drop();
+    impl->cache_host.reset();
-    impl->cache_address.drop();
+    impl->cache_address.reset();
 
     std::scoped_lock lock(impl->update_mutex, impl->drop_mutex);
 
@@ -257,6 +281,11 @@ void DNSResolver::dropCache()
     impl->host_name.reset();
 }
 
+void DNSResolver::removeHostFromCache(const std::string & host)
+{
+    impl->cache_host.remove(host);
+}
+
 void DNSResolver::setDisableCacheFlag(bool is_disabled)
 {
     impl->disable_cache = is_disabled;
@@ -378,17 +407,20 @@ bool DNSResolver::updateCache(UInt32 max_consecutive_failures)
 
 bool DNSResolver::updateHost(const String & host)
 {
-    /// Usage of updateHost implies that host is already in cache and there is no extra computations
+    const auto old_value = resolveIPAddressWithCache(impl->cache_host, host);
-    auto old_value = impl->cache_host(host);
+    auto new_value = resolveIPAddressImpl(host);
-    impl->cache_host.update(host);
+    const bool result = old_value != new_value;
-    return old_value != impl->cache_host(host);
+    impl->cache_host.set(host, std::make_shared<DNSResolver::IPAddresses>(std::move(new_value)));
+    return result;
 }
 
 bool DNSResolver::updateAddress(const Poco::Net::IPAddress & address)
 {
-    auto old_value = impl->cache_address(address);
+    const auto old_value = reverseResolveWithCache(impl->cache_address, address);
-    impl->cache_address.update(address);
+    auto new_value = reverseResolveImpl(address);
-    return old_value == impl->cache_address(address);
+    const bool result = old_value != new_value;
+    impl->cache_address.set(address, std::make_shared<std::unordered_set<String>>(std::move(new_value)));
+    return result;
 }
 
 void DNSResolver::addToNewHosts(const String & host)

src/Common/DNSResolver.h

@@ -18,6 +18,7 @@ class DNSResolver : private boost::noncopyable
 {
 public:
     using IPAddresses = std::vector<Poco::Net::IPAddress>;
+    using IPAddressesPtr = std::shared_ptr<IPAddresses>;
 
     static DNSResolver & instance();
 
@@ -48,6 +49,9 @@ public:
     /// Drops all caches
     void dropCache();
 
+    /// Removes an entry from cache or does nothing
+    void removeHostFromCache(const std::string & host);
+
     /// Updates all known hosts in cache.
     /// Returns true if IP of any host has been changed or an element was dropped (too many failures)
     bool updateCache(UInt32 max_consecutive_failures);

src/Common/OpenTelemetryTraceContext.cpp

@@ -5,7 +5,7 @@
 #include <Common/Exception.h>
 #include <Common/hex.h>
 #include <Core/Settings.h>
-#include <IO/WriteHelpers.h>
+#include <IO/Operators.h>
 
 namespace DB
 {
@@ -226,6 +226,30 @@ String TracingContext::composeTraceparentHeader() const
         static_cast<uint8_t>(trace_flags));
 }
 
+void TracingContext::deserialize(ReadBuffer & buf)
+{
+    buf >> this->trace_id
+        >> "\n"
+        >> this->span_id
+        >> "\n"
+        >> this->tracestate
+        >> "\n"
+        >> this->trace_flags
+        >> "\n";
+}
+
+void TracingContext::serialize(WriteBuffer & buf) const
+{
+    buf << this->trace_id
+        << "\n"
+        << this->span_id
+        << "\n"
+        << this->tracestate
+        << "\n"
+        << this->trace_flags
+        << "\n";
+}
+
 const TracingContextOnThread & CurrentContext()
 {
     return current_thread_trace_context;

src/Common/OpenTelemetryTraceContext.h

@@ -7,6 +7,8 @@ namespace DB
 
 struct Settings;
 class OpenTelemetrySpanLog;
+class WriteBuffer;
+class ReadBuffer;
 
 namespace OpenTelemetry
 {
@@ -63,6 +65,9 @@ struct TracingContext
     {
         return trace_id != UUID();
     }
+
+    void deserialize(ReadBuffer & buf);
+    void serialize(WriteBuffer & buf) const;
 };
 
 /// Tracing context kept on each thread
@@ -157,5 +162,16 @@ struct SpanHolder : public Span
 
 }
 
+inline WriteBuffer & operator<<(WriteBuffer & buf, const OpenTelemetry::TracingContext & context)
+{
+    context.serialize(buf);
+    return buf;
 }
 
+inline ReadBuffer & operator>> (ReadBuffer & buf, OpenTelemetry::TracingContext & context)
+{
+    context.deserialize(buf);
+    return buf;
+}
+
+}

src/Common/SipHash.h

@@ -32,6 +32,13 @@
         v2 += v1; v1 = ROTL(v1, 17); v1 ^= v2; v2 = ROTL(v2, 32); \
     } while(0)
 
+/// Define macro CURRENT_BYTES_IDX for building index used in current_bytes array
+/// to ensure correct byte order on different endian machines
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#define CURRENT_BYTES_IDX(i) (7 - i)
+#else
+#define CURRENT_BYTES_IDX(i) (i)
+#endif
 
 class SipHash
 {
@@ -55,7 +62,7 @@ private:
     ALWAYS_INLINE void finalize()
     {
         /// In the last free byte, we write the remainder of the division by 256.
-        current_bytes[7] = static_cast<UInt8>(cnt);
+        current_bytes[CURRENT_BYTES_IDX(7)] = static_cast<UInt8>(cnt);
 
         v3 ^= current_word;
         SIPROUND;
@@ -92,7 +99,7 @@ public:
         {
             while (cnt & 7 && data < end)
             {
-                current_bytes[cnt & 7] = *data;
+                current_bytes[CURRENT_BYTES_IDX(cnt & 7)] = *data;
                 ++data;
                 ++cnt;
             }
@@ -125,13 +132,13 @@ public:
         current_word = 0;
         switch (end - data)
         {
-            case 7: current_bytes[6] = data[6]; [[fallthrough]];
+            case 7: current_bytes[CURRENT_BYTES_IDX(6)] = data[6]; [[fallthrough]];
-            case 6: current_bytes[5] = data[5]; [[fallthrough]];
+            case 6: current_bytes[CURRENT_BYTES_IDX(5)] = data[5]; [[fallthrough]];
-            case 5: current_bytes[4] = data[4]; [[fallthrough]];
+            case 5: current_bytes[CURRENT_BYTES_IDX(4)] = data[4]; [[fallthrough]];
-            case 4: current_bytes[3] = data[3]; [[fallthrough]];
+            case 4: current_bytes[CURRENT_BYTES_IDX(3)] = data[3]; [[fallthrough]];
-            case 3: current_bytes[2] = data[2]; [[fallthrough]];
+            case 3: current_bytes[CURRENT_BYTES_IDX(2)] = data[2]; [[fallthrough]];
-            case 2: current_bytes[1] = data[1]; [[fallthrough]];
+            case 2: current_bytes[CURRENT_BYTES_IDX(1)] = data[1]; [[fallthrough]];
-            case 1: current_bytes[0] = data[0]; [[fallthrough]];
+            case 1: current_bytes[CURRENT_BYTES_IDX(0)] = data[0]; [[fallthrough]];
             case 0: break;
         }
     }
@@ -157,8 +164,8 @@ public:
     void get128(char * out)
     {
         finalize();
-        unalignedStoreLE<UInt64>(out, v0 ^ v1);
+        unalignedStore<UInt64>(out, v0 ^ v1);
-        unalignedStoreLE<UInt64>(out + 8, v2 ^ v3);
+        unalignedStore<UInt64>(out + 8, v2 ^ v3);
     }
 
     template <typename T>
@@ -225,3 +232,5 @@ inline UInt64 sipHash64(const std::string & s)
 {
     return sipHash64(s.data(), s.size());
 }
+
+#undef CURRENT_BYTES_IDX

src/Common/StackTrace.cpp

@@ -4,14 +4,15 @@
 #include <Common/Elf.h>
 #include <Common/SymbolIndex.h>
 #include <Common/MemorySanitizer.h>
-#include <base/CachedFn.h>
 #include <base/demangle.h>
 
 #include <atomic>
 #include <cstring>
 #include <filesystem>
+#include <mutex>
 #include <sstream>
 #include <unordered_map>
+#include <map>
 
 #include <Common/config.h>
 
@@ -462,20 +463,36 @@ std::string StackTrace::toString(void ** frame_pointers_, size_t offset, size_t
     return toStringStatic(frame_pointers_copy, offset, size);
 }
 
-static CachedFn<&toStringImpl> & cacheInstance()
+using StackTraceRepresentation = std::tuple<StackTrace::FramePointers, size_t, size_t>;
+using StackTraceCache = std::map<StackTraceRepresentation, std::string>;
+
+static StackTraceCache & cacheInstance()
 {
-    static CachedFn<&toStringImpl> cache;
+    static StackTraceCache cache;
     return cache;
 }
 
+static std::mutex stacktrace_cache_mutex;
+
 std::string StackTrace::toStringStatic(const StackTrace::FramePointers & frame_pointers, size_t offset, size_t size)
 {
     /// Calculation of stack trace text is extremely slow.
     /// We use simple cache because otherwise the server could be overloaded by trash queries.
-    return cacheInstance()(frame_pointers, offset, size);
+    /// Note that this cache can grow unconditionally, but practically it should be small.
+    std::lock_guard lock{stacktrace_cache_mutex};
+
+    StackTraceRepresentation key{frame_pointers, offset, size};
+    auto & cache = cacheInstance();
+    if (cache.contains(key))
+        return cache[key];
+
+    auto result = toStringImpl(frame_pointers, offset, size);
+    cache[key] = result;
+    return result;
 }
 
 void StackTrace::dropCache()
 {
-    cacheInstance().drop();
+    std::lock_guard lock{stacktrace_cache_mutex};
+    cacheInstance().clear();
 }

src/Common/TargetSpecific.cpp

@@ -20,8 +20,6 @@ UInt32 getSupportedArchs()
         result |= static_cast<UInt32>(TargetArch::AVX512BW);
     if (Cpu::CpuFlagsCache::have_AVX512VBMI)
         result |= static_cast<UInt32>(TargetArch::AVX512VBMI);
-    if (Cpu::CpuFlagsCache::have_AVX512VBMI2)
-        result |= static_cast<UInt32>(TargetArch::AVX512VBMI2);
     return result;
 }
 
@@ -42,7 +40,6 @@ String toString(TargetArch arch)
         case TargetArch::AVX512F: return "avx512f";
         case TargetArch::AVX512BW: return "avx512bw";
         case TargetArch::AVX512VBMI: return "avx512vbmi";
-        case TargetArch::AVX512VBMI2: return "avx512vbmi";
     }
 
     __builtin_unreachable();

src/Common/TargetSpecific.h

@@ -31,7 +31,7 @@
  * int funcImpl() {
  *     return 2;
  * }
- * ) // DECLARE_AVX2_SPECIFIC_CODE
+ * ) // DECLARE_DEFAULT_CODE
  *
  * int func() {
  * #if USE_MULTITARGET_CODE
@@ -82,7 +82,6 @@ enum class TargetArch : UInt32
     AVX512F  = (1 << 3),
     AVX512BW  = (1 << 4),
     AVX512VBMI = (1 << 5),
-    AVX512VBMI2 = (1 << 6),
 };
 
 /// Runtime detection.
@@ -101,7 +100,6 @@ String toString(TargetArch arch);
 
 #if defined(__clang__)
 
-#define AVX512VBMI2_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,avx512vbmi2")))
 #define AVX512VBMI_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi")))
 #define AVX512BW_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw")))
 #define AVX512_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f")))
@@ -110,8 +108,6 @@ String toString(TargetArch arch);
 #define SSE42_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt")))
 #define DEFAULT_FUNCTION_SPECIFIC_ATTRIBUTE
 
-#   define BEGIN_AVX512VBMI2_SPECIFIC_CODE \
-        _Pragma("clang attribute push(__attribute__((target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,avx512vbmi2\"))),apply_to=function)")
 #   define BEGIN_AVX512VBMI_SPECIFIC_CODE \
         _Pragma("clang attribute push(__attribute__((target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi\"))),apply_to=function)")
 #   define BEGIN_AVX512BW_SPECIFIC_CODE \
@@ -133,7 +129,6 @@ String toString(TargetArch arch);
 #   define DUMMY_FUNCTION_DEFINITION [[maybe_unused]] void _dummy_function_definition();
 #else
 
-#define AVX512VBMI2_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,avx512vbmi2,tune=native")))
 #define AVX512VBMI_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,tune=native")))
 #define AVX512BW_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,tune=native")))
 #define AVX512_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,tune=native")))
@@ -142,9 +137,6 @@ String toString(TargetArch arch);
 #define SSE42_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt",tune=native)))
 #define DEFAULT_FUNCTION_SPECIFIC_ATTRIBUTE
 
-#   define BEGIN_AVX512VBMI2_SPECIFIC_CODE \
-        _Pragma("GCC push_options") \
-        _Pragma("GCC target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,avx512vbmi2,tune=native\")")
 #   define BEGIN_AVX512VBMI_SPECIFIC_CODE \
         _Pragma("GCC push_options") \
         _Pragma("GCC target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,tune=native\")")
@@ -225,16 +217,6 @@ namespace TargetSpecific::AVX512VBMI { \
 } \
 END_TARGET_SPECIFIC_CODE
 
-#define DECLARE_AVX512VBMI2_SPECIFIC_CODE(...) \
-BEGIN_AVX512VBMI2_SPECIFIC_CODE \
-namespace TargetSpecific::AVX512VBMI2 { \
-    DUMMY_FUNCTION_DEFINITION \
-    using namespace DB::TargetSpecific::AVX512VBMI2; \
-    __VA_ARGS__ \
-} \
-END_TARGET_SPECIFIC_CODE
-
-
 #else
 
 #define USE_MULTITARGET_CODE 0
@@ -247,7 +229,6 @@ END_TARGET_SPECIFIC_CODE
 #define DECLARE_AVX512F_SPECIFIC_CODE(...)
 #define DECLARE_AVX512BW_SPECIFIC_CODE(...)
 #define DECLARE_AVX512VBMI_SPECIFIC_CODE(...)
-#define DECLARE_AVX512VBMI2_SPECIFIC_CODE(...)
 
 #endif
 
@@ -264,9 +245,8 @@ DECLARE_SSE42_SPECIFIC_CODE  (__VA_ARGS__) \
 DECLARE_AVX_SPECIFIC_CODE    (__VA_ARGS__) \
 DECLARE_AVX2_SPECIFIC_CODE   (__VA_ARGS__) \
 DECLARE_AVX512F_SPECIFIC_CODE(__VA_ARGS__) \
-DECLARE_AVX512BW_SPECIFIC_CODE    (__VA_ARGS__) \
+DECLARE_AVX512BW_SPECIFIC_CODE(__VA_ARGS__) \
-DECLARE_AVX512VBMI_SPECIFIC_CODE  (__VA_ARGS__) \
+DECLARE_AVX512VBMI_SPECIFIC_CODE(__VA_ARGS__)
-DECLARE_AVX512VBMI2_SPECIFIC_CODE (__VA_ARGS__)
 
 DECLARE_DEFAULT_CODE(
     constexpr auto BuildArch = TargetArch::Default; /// NOLINT
@@ -296,9 +276,6 @@ DECLARE_AVX512VBMI_SPECIFIC_CODE(
     constexpr auto BuildArch = TargetArch::AVX512VBMI; /// NOLINT
 ) // DECLARE_AVX512VBMI_SPECIFIC_CODE
 
-DECLARE_AVX512VBMI2_SPECIFIC_CODE(
-    constexpr auto BuildArch = TargetArch::AVX512VBMI2; /// NOLINT
-) // DECLARE_AVX512VBMI2_SPECIFIC_CODE
 
 /** Runtime Dispatch helpers for class members.
   *

src/Common/tests/gtest_cached_fn.cpp

@@ -1,54 +0,0 @@
-#include <gtest/gtest.h>
-#include <thread>
-#include <chrono>
-#include <base/CachedFn.h>
-
-using namespace std::chrono_literals;
-
-constexpr int add(int x, int y)
-{
-    return x + y;
-}
-
-int longFunction(int x, int y)
-{
-    std::this_thread::sleep_for(1s);
-    return x + y;
-}
-
-auto f = [](int x, int y) { return x - y; };
-
-TEST(CachedFn, Basic)
-{
-    CachedFn<&add> fn;
-
-    const int res = fn(1, 2);
-    EXPECT_EQ(fn(1, 2), res);
-
-    /// In GCC, lambda can't be placed in TEST, producing "<labmda> has no linkage".
-    /// Assuming http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4268.html,
-    /// this is a GCC bug.
-    CachedFn<+f> fn2;
-
-    const int res2 = fn2(1, 2);
-    EXPECT_EQ(fn2(1, 2), res2);
-}
-
-TEST(CachedFn, CachingResults)
-{
-    CachedFn<&longFunction> fn;
-
-    for (int x = 0; x < 2; ++x)
-    {
-        for (int y = 0; y < 2; ++y)
-        {
-            const int res = fn(x, y);
-            const time_t start = time(nullptr);
-
-            for (int count = 0; count < 1000; ++count)
-                EXPECT_EQ(fn(x, y), res);
-
-            EXPECT_LT(time(nullptr) - start, 10);
-        }
-    }
-}

src/Coordination/KeeperServer.cpp

@@ -705,7 +705,7 @@ void KeeperServer::waitInit()
 
     int64_t timeout = coordination_settings->startup_timeout.totalMilliseconds();
     if (!initialized_cv.wait_for(lock, std::chrono::milliseconds(timeout), [&] { return initialized_flag.load(); }))
-        throw Exception(ErrorCodes::RAFT_ERROR, "Failed to wait RAFT initialization");
+        LOG_WARNING(log, "Failed to wait for RAFT initialization in {}ms, will continue in background", timeout);
 }
 
 std::vector<int64_t> KeeperServer::getDeadSessions()

src/Core/Settings.h

@@ -42,6 +42,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
   */
 
 #define COMMON_SETTINGS(M) \
+    M(Dialect, dialect, Dialect::clickhouse, "Which SQL dialect will be used to parse query", 0)\
     M(UInt64, min_compress_block_size, 65536, "The actual size of the block to compress, if the uncompressed data less than max_compress_block_size is no less than this value and no less than the volume of data for one mark.", 0) \
     M(UInt64, max_compress_block_size, 1048576, "The maximum size of blocks of uncompressed data before compressing for writing to a table.", 0) \
     M(UInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size for reading", 0) \

src/Core/SettingsEnums.cpp

@@ -158,5 +158,7 @@ IMPLEMENT_SETTING_ENUM(MsgPackUUIDRepresentation , ErrorCodes::BAD_ARGUMENTS,
                         {"str", FormatSettings::MsgPackUUIDRepresentation::STR},
                         {"ext", FormatSettings::MsgPackUUIDRepresentation::EXT}})
 
-
+IMPLEMENT_SETTING_ENUM(Dialect, ErrorCodes::BAD_ARGUMENTS,
+    {{"clickhouse", Dialect::clickhouse},
+     {"kusto", Dialect::kusto}})
 }

src/Core/SettingsEnums.h

@@ -183,4 +183,12 @@ DECLARE_SETTING_ENUM_WITH_RENAME(EscapingRule, FormatSettings::EscapingRule)
 
 DECLARE_SETTING_ENUM_WITH_RENAME(MsgPackUUIDRepresentation, FormatSettings::MsgPackUUIDRepresentation)
 
+enum class Dialect
+{
+    clickhouse,
+    kusto,
+    kusto_auto,
+};
+
+DECLARE_SETTING_ENUM(Dialect)
 }

src/Daemon/BaseDaemon.cpp

@@ -266,8 +266,8 @@ private:
     {
         size_t pos = message.find('\n');
 
-        LOG_FATAL(log, "(version {}{}, {}) (from thread {}) {}",
+        LOG_FATAL(log, "(version {}{}, build id: {}) (from thread {}) {}",
-            VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info, thread_num, message.substr(0, pos));
+            VERSION_STRING, VERSION_OFFICIAL, daemon.build_id, thread_num, message.substr(0, pos));
 
         /// Print trace from std::terminate exception line-by-line to make it easy for grep.
         while (pos != std::string_view::npos)
@@ -315,14 +315,14 @@ private:
 
         if (query_id.empty())
         {
-            LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (no query) Received signal {} ({})",
+            LOG_FATAL(log, "(version {}{}, build id: {}) (from thread {}) (no query) Received signal {} ({})",
-                VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info,
+                VERSION_STRING, VERSION_OFFICIAL, daemon.build_id,
                 thread_num, strsignal(sig), sig); // NOLINT(concurrency-mt-unsafe) // it is not thread-safe but ok in this context
         }
         else
         {
-            LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (query_id: {}) (query: {}) Received signal {} ({})",
+            LOG_FATAL(log, "(version {}{}, build id: {}) (from thread {}) (query_id: {}) (query: {}) Received signal {} ({})",
-                VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info,
+                VERSION_STRING, VERSION_OFFICIAL, daemon.build_id,
                 thread_num, query_id, query, strsignal(sig), sig); // NOLINT(concurrency-mt-unsafe) // it is not thread-safe but ok in this context)
         }
 
@@ -838,6 +838,7 @@ static void blockSignals(const std::vector<int> & signals)
         throw Poco::Exception("Cannot block signal.");
 }
 
+extern String getGitHash();
 
 void BaseDaemon::initializeTerminationAndSignalProcessing()
 {
@@ -870,13 +871,15 @@ void BaseDaemon::initializeTerminationAndSignalProcessing()
 #if defined(__ELF__) && !defined(OS_FREEBSD)
     String build_id_hex = DB::SymbolIndex::instance()->getBuildIDHex();
     if (build_id_hex.empty())
-        build_id_info = "no build id";
+        build_id = "";
     else
-        build_id_info = "build id: " + build_id_hex;
+        build_id = build_id_hex;
 #else
-    build_id_info = "no build id";
+    build_id = "";
 #endif
 
+    git_hash = getGitHash();
+
 #if defined(OS_LINUX)
     std::string executable_path = getExecutablePath();
 
@@ -888,8 +891,9 @@ void BaseDaemon::initializeTerminationAndSignalProcessing()
 void BaseDaemon::logRevision() const
 {
     Poco::Logger::root().information("Starting " + std::string{VERSION_FULL}
-        + " with revision " + std::to_string(ClickHouseRevision::getVersionRevision())
+        + " (revision: " + std::to_string(ClickHouseRevision::getVersionRevision())
-        + ", " + build_id_info
+        + ", git hash: " + (git_hash.empty() ? "<unknown>" : git_hash)
+        + ", build id: " + (build_id.empty() ? "<unknown>" : build_id) + ")"
         + ", PID " + std::to_string(getpid()));
 }
 

src/Daemon/BaseDaemon.h

@@ -172,7 +172,8 @@ protected:
     DB::ConfigProcessor::LoadedConfig loaded_config;
     Poco::Util::AbstractConfiguration * last_configuration = nullptr;
 
-    String build_id_info;
+    String build_id;
+    String git_hash;
     String stored_binary_hash;
 
     std::vector<int> handled_signals;

src/Daemon/CMakeLists.txt

@@ -1,7 +1,10 @@
+configure_file(GitHash.cpp.in GitHash.generated.cpp)
+
 add_library (daemon
     BaseDaemon.cpp
     GraphiteWriter.cpp
     SentryWriter.cpp
+    GitHash.generated.cpp
 )
 
 if (OS_DARWIN AND NOT USE_STATIC_LIBRARIES)

src/Daemon/GitHash.cpp.in

@@ -0,0 +1,8 @@
+// File was generated by CMake
+
+#include <base/types.h>
+
+String getGitHash()
+{
+    return "@GIT_HASH@";
+}

src/Databases/DatabaseReplicated.cpp

@@ -8,6 +8,7 @@
 #include <Interpreters/executeQuery.h>
 #include <Parsers/queryToString.h>
 #include <Common/Exception.h>
+#include <Common/OpenTelemetryTraceContext.h>
 #include <Common/ZooKeeper/KeeperException.h>
 #include <Common/ZooKeeper/Types.h>
 #include <Common/ZooKeeper/ZooKeeper.h>
@@ -642,6 +643,7 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, Contex
     entry.query = queryToString(query);
     entry.initiator = ddl_worker->getCommonHostID();
     entry.setSettingsIfRequired(query_context);
+    entry.tracing_context = OpenTelemetry::CurrentContext();
     String node_path = ddl_worker->tryEnqueueAndExecuteEntry(entry, query_context);
 
     Strings hosts_to_wait = getZooKeeper()->getChildren(zookeeper_path + "/replicas");

src/Databases/DatabaseReplicatedWorker.cpp

@@ -221,6 +221,10 @@ String DatabaseReplicatedDDLWorker::tryEnqueueAndExecuteEntry(DDLLogEntry & entr
     /// NOTE Possibly it would be better to execute initial query on the most up-to-date node,
     /// but it requires more complex logic around /try node.
 
+    OpenTelemetry::SpanHolder span(__FUNCTION__);
+    span.addAttribute("clickhouse.cluster", database->getDatabaseName());
+    entry.tracing_context = OpenTelemetry::CurrentContext();
+
     auto zookeeper = getAndSetZooKeeper();
     UInt32 our_log_ptr = getLogPointer();
     UInt32 max_log_ptr = parse<UInt32>(zookeeper->get(database->zookeeper_path + "/max_log_ptr"));

src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp

@@ -37,7 +37,7 @@ namespace ErrorCodes
 
 
 AsynchronousReadIndirectBufferFromRemoteFS::AsynchronousReadIndirectBufferFromRemoteFS(
-        AsynchronousReaderPtr reader_,
+        IAsynchronousReader & reader_,
         const ReadSettings & settings_,
         std::shared_ptr<ReadBufferFromRemoteFSGather> impl_,
         size_t min_bytes_for_seek_)
@@ -111,7 +111,7 @@ std::future<IAsynchronousReader::Result> AsynchronousReadIndirectBufferFromRemot
         request.ignore = bytes_to_ignore;
         bytes_to_ignore = 0;
     }
-    return reader->submit(request);
+    return reader.submit(request);
 }
 
 

src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h

@@ -31,7 +31,7 @@ class AsynchronousReadIndirectBufferFromRemoteFS : public ReadBufferFromFileBase
 {
 public:
     explicit AsynchronousReadIndirectBufferFromRemoteFS(
-        AsynchronousReaderPtr reader_, const ReadSettings & settings_,
+        IAsynchronousReader & reader_, const ReadSettings & settings_,
         std::shared_ptr<ReadBufferFromRemoteFSGather> impl_,
         size_t min_bytes_for_seek = DBMS_DEFAULT_BUFFER_SIZE);
 
@@ -64,7 +64,7 @@ private:
 
     std::future<IAsynchronousReader::Result> asyncReadInto(char * data, size_t size);
 
-    AsynchronousReaderPtr reader;
+    IAsynchronousReader & reader;
 
     Int32 priority;
 

src/Disks/IO/ThreadPoolReader.cpp

@@ -198,31 +198,10 @@ std::future<IAsynchronousReader::Result> ThreadPoolReader::submit(Request reques
 
     ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheMiss);
 
-    ThreadGroupStatusPtr running_group;
+    auto schedule = threadPoolCallbackRunner<Result>(pool, "ThreadPoolRead");
-    if (CurrentThread::isInitialized() && CurrentThread::get().getThreadGroup())
-        running_group = CurrentThread::get().getThreadGroup();
 
-    ContextPtr query_context;
+    return schedule([request, fd]() -> Result
-    if (CurrentThread::isInitialized())
-        query_context = CurrentThread::get().getQueryContext();
-
-    auto task = std::make_shared<std::packaged_task<Result()>>([request, fd, running_group, query_context]
     {
-        ThreadStatus thread_status;
-
-        SCOPE_EXIT({
-            if (running_group)
-                thread_status.detachQuery();
-        });
-
-        if (running_group)
-            thread_status.attachQuery(running_group);
-
-        if (query_context)
-            thread_status.attachQueryContext(query_context);
-
-        setThreadName("ThreadPoolRead");
-
         Stopwatch watch(CLOCK_MONOTONIC);
         SCOPE_EXIT({
             watch.stop();
@@ -260,14 +239,7 @@ std::future<IAsynchronousReader::Result> ThreadPoolReader::submit(Request reques
         ProfileEvents::increment(ProfileEvents::ReadBufferFromFileDescriptorReadBytes, bytes_read);
 
         return Result{ .size = bytes_read, .offset = request.ignore };
-    });
+    }, request.priority);
-
-    auto future = task->get_future();
-
-    /// ThreadPool is using "bigger is higher priority" instead of "smaller is more priority".
-    pool.scheduleOrThrow([task]{ (*task)(); }, -request.priority);
-
-    return future;
 }
 
 }

src/Disks/IO/ThreadPoolReader.h

@@ -2,6 +2,7 @@
 
 #include <IO/AsynchronousReader.h>
 #include <Common/ThreadPool.h>
+#include <Interpreters/threadPoolCallbackRunner.h>
 
 
 namespace DB
@@ -31,8 +32,11 @@ private:
 
 public:
     ThreadPoolReader(size_t pool_size, size_t queue_size_);
+
     std::future<Result> submit(Request request) override;
 
+    void wait() override { pool.wait(); }
+
     /// pool automatically waits for all tasks in destructor.
 };
 

src/Disks/IO/ThreadPoolRemoteFSReader.cpp

@@ -5,7 +5,6 @@
 #include <Common/CurrentMetrics.h>
 #include <Common/Stopwatch.h>
 #include <Common/assert_cast.h>
-#include <Common/setThreadName.h>
 #include <Common/CurrentThread.h>
 #include <Common/config.h>
 #include <IO/SeekableReadBuffer.h>
@@ -40,33 +39,10 @@ ThreadPoolRemoteFSReader::ThreadPoolRemoteFSReader(size_t pool_size, size_t queu
 
 std::future<IAsynchronousReader::Result> ThreadPoolRemoteFSReader::submit(Request request)
 {
-    ThreadGroupStatusPtr running_group;
+    auto schedule = threadPoolCallbackRunner<Result>(pool, "VFSRead");
-    if (CurrentThread::isInitialized() && CurrentThread::get().getThreadGroup())
-        running_group = CurrentThread::get().getThreadGroup();
 
-    ContextPtr query_context;
+    return schedule([request]() -> Result
-    if (CurrentThread::isInitialized())
-        query_context = CurrentThread::get().getQueryContext();
-
-    auto task = std::make_shared<std::packaged_task<Result()>>([request, running_group, query_context]
     {
-        ThreadStatus thread_status;
-
-        SCOPE_EXIT({
-            if (running_group)
-                thread_status.detachQuery();
-        });
-
-        /// To be able to pass ProfileEvents.
-        if (running_group)
-            thread_status.attachQuery(running_group);
-
-        /// Save query context if any, because cache implementation needs it.
-        if (query_context)
-            thread_status.attachQueryContext(query_context);
-
-        setThreadName("VFSRead");
-
         CurrentMetrics::Increment metric_increment{CurrentMetrics::Read};
         auto * remote_fs_fd = assert_cast<RemoteFSFileDescriptor *>(request.descriptor.get());
 
@@ -80,14 +56,7 @@ std::future<IAsynchronousReader::Result> ThreadPoolRemoteFSReader::submit(Reques
         ProfileEvents::increment(ProfileEvents::ThreadpoolReaderReadBytes, result.offset ? result.size - result.offset : result.size);
 
         return Result{ .size = result.size, .offset = result.offset };
-    });
+    }, request.priority);
-
-    auto future = task->get_future();
-
-    /// ThreadPool is using "bigger is higher priority" instead of "smaller is more priority".
-    pool.scheduleOrThrow([task]{ (*task)(); }, -request.priority);
-
-    return future;
 }
 
 }

src/Disks/IO/ThreadPoolRemoteFSReader.h

@@ -3,6 +3,7 @@
 #include <IO/AsynchronousReader.h>
 #include <IO/ReadBuffer.h>
 #include <Common/ThreadPool.h>
+#include <Interpreters/threadPoolCallbackRunner.h>
 
 namespace DB
 {
@@ -14,6 +15,8 @@ public:
 
     std::future<IAsynchronousReader::Result> submit(Request request) override;
 
+    void wait() override { pool.wait(); }
+
 private:
     ThreadPool pool;
 };

src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp

@@ -11,19 +11,20 @@
 namespace DB
 {
 
+static constexpr auto DEFAULT_RETRY_NUM = 3;
+
 WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage(
     std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> blob_container_client_,
     const String & blob_path_,
     size_t max_single_part_upload_size_,
     size_t buf_size_,
-    const WriteSettings & write_settings_,
+    const WriteSettings & write_settings_)
-    std::optional<std::map<std::string, std::string>> attributes_)
     : BufferWithOwnMemory<WriteBuffer>(buf_size_, nullptr, 0)
-    , blob_container_client(blob_container_client_)
+    , log(&Poco::Logger::get("WriteBufferFromAzureBlobStorage"))
     , max_single_part_upload_size(max_single_part_upload_size_)
     , blob_path(blob_path_)
     , write_settings(write_settings_)
-    , attributes(attributes_)
+    , blob_container_client(blob_container_client_)
 {
 }
 
@@ -33,63 +34,69 @@ WriteBufferFromAzureBlobStorage::~WriteBufferFromAzureBlobStorage()
     finalize();
 }
 
-void WriteBufferFromAzureBlobStorage::finalizeImpl()
+void WriteBufferFromAzureBlobStorage::execWithRetry(std::function<void()> func, size_t num_tries)
 {
-    if (attributes.has_value())
+    auto handle_exception = [&](const auto & e, size_t i)
     {
-        auto blob_client = blob_container_client->GetBlobClient(blob_path);
+        if (i == num_tries - 1)
-        Azure::Storage::Metadata metadata;
+            throw;
-        for (const auto & [key, value] : *attributes)
-            metadata[key] = value;
-        blob_client.SetMetadata(metadata);
-    }
 
-    const size_t max_tries = 3;
+        LOG_DEBUG(log, "Write at attempt {} for blob `{}` failed: {}", i + 1, blob_path, e.Message);
-    for (size_t i = 0; i < max_tries; ++i)
+    };
+
+    for (size_t i = 0; i < num_tries; ++i)
     {
         try
         {
-            next();
+            func();
             break;
         }
+        catch (const Azure::Core::Http::TransportException & e)
+        {
+            handle_exception(e, i);
+        }
         catch (const Azure::Core::RequestFailedException & e)
         {
-            if (i == max_tries - 1)
+            handle_exception(e, i);
-                throw;
-            LOG_INFO(&Poco::Logger::get("WriteBufferFromAzureBlobStorage"),
-                     "Exception caught during finalizing azure storage write at attempt {}: {}", i + 1, e.Message);
         }
     }
 }
 
+void WriteBufferFromAzureBlobStorage::finalizeImpl()
+{
+    execWithRetry([this](){ next(); }, DEFAULT_RETRY_NUM);
+}
+
 void WriteBufferFromAzureBlobStorage::nextImpl()
 {
     if (!offset())
         return;
 
-    auto * buffer_begin = working_buffer.begin();
+    char * buffer_begin = working_buffer.begin();
-    auto len = offset();
+    size_t total_size = offset();
+
     auto block_blob_client = blob_container_client->GetBlockBlobClient(blob_path);
 
-    size_t read = 0;
+    size_t current_size = 0;
     std::vector<std::string> block_ids;
-    while (read < len)
+
+    while (current_size < total_size)
     {
-        auto part_len = std::min(len - read, max_single_part_upload_size);
+        size_t part_len = std::min(total_size - current_size, max_single_part_upload_size);
+        const std::string & block_id = block_ids.emplace_back(getRandomASCIIString(64));
 
-        auto block_id = getRandomASCIIString(64);
+        Azure::Core::IO::MemoryBodyStream tmp_buffer(reinterpret_cast<uint8_t *>(buffer_begin + current_size), part_len);
-        block_ids.push_back(block_id);
+        execWithRetry([&](){ block_blob_client.StageBlock(block_id, tmp_buffer); }, DEFAULT_RETRY_NUM);
 
-        Azure::Core::IO::MemoryBodyStream tmp_buffer(reinterpret_cast<uint8_t *>(buffer_begin + read), part_len);
+        current_size += part_len;
-        block_blob_client.StageBlock(block_id, tmp_buffer);
+        LOG_TRACE(log, "Staged block (id: {}) of size {} (written {}/{}, blob path: {}).", block_id, part_len, current_size, total_size, blob_path);
-
-        read += part_len;
     }
 
-    block_blob_client.CommitBlockList(block_ids);
+    execWithRetry([&](){ block_blob_client.CommitBlockList(block_ids); }, DEFAULT_RETRY_NUM);
+    LOG_TRACE(log, "Committed {} blocks for blob `{}`", block_ids.size(), blob_path);
 
     if (write_settings.remote_throttler)
-        write_settings.remote_throttler->add(read);
+        write_settings.remote_throttler->add(total_size);
 }
 
 }

src/Disks/IO/WriteBufferFromAzureBlobStorage.h

@@ -13,20 +13,25 @@
 #include <azure/core/io/body_stream.hpp>
 
 
+namespace Poco
+{
+class Logger;
+}
+
 namespace DB
 {
 
 class WriteBufferFromAzureBlobStorage : public BufferWithOwnMemory<WriteBuffer>
 {
 public:
+    using AzureClientPtr = std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient>;
 
     WriteBufferFromAzureBlobStorage(
-        std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> blob_container_client_,
+        AzureClientPtr blob_container_client_,
         const String & blob_path_,
         size_t max_single_part_upload_size_,
         size_t buf_size_,
-        const WriteSettings & write_settings_,
+        const WriteSettings & write_settings_);
-        std::optional<std::map<std::string, std::string>> attributes_ = {});
 
     ~WriteBufferFromAzureBlobStorage() override;
 
@@ -34,12 +39,15 @@ public:
 
 private:
     void finalizeImpl() override;
+    void execWithRetry(std::function<void()> func, size_t num_tries);
 
-    std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> blob_container_client;
+    Poco::Logger * log;
-    size_t max_single_part_upload_size;
+
-    const String blob_path;
+    const size_t max_single_part_upload_size;
-    WriteSettings write_settings;
+    const std::string blob_path;
-    std::optional<std::map<std::string, std::string>> attributes;
+    const WriteSettings write_settings;
+
+    AzureClientPtr blob_container_client;
 };
 
 }

src/Disks/IO/createReadBufferFromFileBase.cpp

@@ -77,13 +77,21 @@ std::unique_ptr<ReadBufferFromFileBase> createReadBufferFromFileBase(
         }
         else if (settings.local_fs_method == LocalFSReadMethod::pread_fake_async)
         {
-            static AsynchronousReaderPtr reader = std::make_shared<SynchronousReader>();
+            auto context = Context::getGlobalContextInstance();
+            if (!context)
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context not initialized");
+
+            auto & reader = context->getThreadPoolReader(Context::FilesystemReaderType::SYNCHRONOUS_LOCAL_FS_READER);
             res = std::make_unique<AsynchronousReadBufferFromFileWithDescriptorsCache>(
                 reader, settings.priority, filename, buffer_size, actual_flags, existing_memory, alignment, file_size);
         }
         else if (settings.local_fs_method == LocalFSReadMethod::pread_threadpool)
         {
-            static AsynchronousReaderPtr reader = std::make_shared<ThreadPoolReader>(16, 1000000);
+            auto context = Context::getGlobalContextInstance();
+            if (!context)
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context not initialized");
+
+            auto & reader = context->getThreadPoolReader(Context::FilesystemReaderType::ASYNCHRONOUS_LOCAL_FS_READER);
             res = std::make_unique<AsynchronousReadBufferFromFileWithDescriptorsCache>(
                 reader, settings.priority, filename, buffer_size, actual_flags, existing_memory, alignment, file_size);
         }

src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp

@@ -107,7 +107,7 @@ std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObjects( /// NOL
 
     if (disk_read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
     {
-        auto reader = getThreadPoolReader();
+        auto & reader = getThreadPoolReader();
         return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, disk_read_settings, std::move(reader_impl));
     }
     else

src/Disks/ObjectStorages/IObjectStorage.cpp

@@ -11,22 +11,25 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int NOT_IMPLEMENTED;
+    extern const int LOGICAL_ERROR;
 }
 
-AsynchronousReaderPtr IObjectStorage::getThreadPoolReader()
+IAsynchronousReader & IObjectStorage::getThreadPoolReader()
 {
-    constexpr size_t pool_size = 50;
+    auto context = Context::getGlobalContextInstance();
-    constexpr size_t queue_size = 1000000;
+    if (!context)
-    static AsynchronousReaderPtr reader = std::make_shared<ThreadPoolRemoteFSReader>(pool_size, queue_size);
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context not initialized");
-    return reader;
+
+    return context->getThreadPoolReader(Context::FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER);
 }
 
 ThreadPool & IObjectStorage::getThreadPoolWriter()
 {
-    constexpr size_t pool_size = 100;
+    auto context = Context::getGlobalContextInstance();
-    constexpr size_t queue_size = 1000000;
+    if (!context)
-    static ThreadPool writer(pool_size, pool_size, queue_size);
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context not initialized");
-    return writer;
+
+    return context->getThreadPoolWriter();
 }
 
 void IObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT

src/Disks/ObjectStorages/IObjectStorage.h

@@ -130,7 +130,7 @@ public:
     /// Path to directory with objects cache
     virtual const std::string & getCacheBasePath() const;
 
-    static AsynchronousReaderPtr getThreadPoolReader();
+    static IAsynchronousReader & getThreadPoolReader();
 
     static ThreadPool & getThreadPoolWriter();
 

src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp

@@ -190,7 +190,7 @@ std::unique_ptr<ReadBufferFromFileBase> S3ObjectStorage::readObjects( /// NOLINT
 
     if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
     {
-        auto reader = getThreadPoolReader();
+        auto & reader = getThreadPoolReader();
         return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, disk_read_settings, std::move(s3_impl));
     }
     else
@@ -230,6 +230,8 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "S3 doesn't support append to files");
 
     auto settings_ptr = s3_settings.get();
+    auto scheduler = threadPoolCallbackRunner<void>(getThreadPoolWriter(), "VFSWrite");
+
     auto s3_buffer = std::make_unique<WriteBufferFromS3>(
         client.get(),
         bucket,
@@ -237,7 +239,7 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
         settings_ptr->s3_settings,
         attributes,
         buf_size,
-        threadPoolCallbackRunner(getThreadPoolWriter()),
+        std::move(scheduler),
         disk_write_settings);
 
     return std::make_unique<WriteIndirectBufferFromRemoteFS>(

src/Disks/ObjectStorages/Web/WebObjectStorage.cpp

@@ -168,7 +168,7 @@ std::unique_ptr<ReadBufferFromFileBase> WebObjectStorage::readObject( /// NOLINT
 
     if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
     {
-        auto reader = IObjectStorage::getThreadPoolReader();
+        auto & reader = IObjectStorage::getThreadPoolReader();
         return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, read_settings, std::move(web_impl), min_bytes_for_seek);
     }
     else

src/IO/AsynchronousReadBufferFromFile.cpp

@@ -24,7 +24,7 @@ namespace ErrorCodes
 
 
 AsynchronousReadBufferFromFile::AsynchronousReadBufferFromFile(
-    AsynchronousReaderPtr reader_,
+    IAsynchronousReader & reader_,
     Int32 priority_,
     const std::string & file_name_,
     size_t buf_size,
@@ -32,7 +32,7 @@ AsynchronousReadBufferFromFile::AsynchronousReadBufferFromFile(
     char * existing_memory,
     size_t alignment,
     std::optional<size_t> file_size_)
-    : AsynchronousReadBufferFromFileDescriptor(std::move(reader_), priority_, -1, buf_size, existing_memory, alignment, file_size_)
+    : AsynchronousReadBufferFromFileDescriptor(reader_, priority_, -1, buf_size, existing_memory, alignment, file_size_)
     , file_name(file_name_)
 {
     ProfileEvents::increment(ProfileEvents::FileOpen);
@@ -58,7 +58,7 @@ AsynchronousReadBufferFromFile::AsynchronousReadBufferFromFile(
 
 
 AsynchronousReadBufferFromFile::AsynchronousReadBufferFromFile(
-    AsynchronousReaderPtr reader_,
+    IAsynchronousReader & reader_,
     Int32 priority_,
     int & fd_,
     const std::string & original_file_name,
@@ -66,7 +66,7 @@ AsynchronousReadBufferFromFile::AsynchronousReadBufferFromFile(
     char * existing_memory,
     size_t alignment,
     std::optional<size_t> file_size_)
-    : AsynchronousReadBufferFromFileDescriptor(std::move(reader_), priority_, fd_, buf_size, existing_memory, alignment, file_size_)
+    : AsynchronousReadBufferFromFileDescriptor(reader_, priority_, fd_, buf_size, existing_memory, alignment, file_size_)
     , file_name(original_file_name.empty() ? "(fd = " + toString(fd_) + ")" : original_file_name)
 {
     fd_ = -1;
@@ -105,4 +105,3 @@ AsynchronousReadBufferFromFileWithDescriptorsCache::~AsynchronousReadBufferFromF
 
 
 }
-

src/IO/AsynchronousReadBufferFromFile.h

@@ -14,7 +14,7 @@ protected:
 
 public:
     explicit AsynchronousReadBufferFromFile(
-        AsynchronousReaderPtr reader_,
+        IAsynchronousReader & reader_,
         Int32 priority_,
         const std::string & file_name_,
         size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
@@ -25,7 +25,7 @@ public:
 
     /// Use pre-opened file descriptor.
     explicit AsynchronousReadBufferFromFile(
-        AsynchronousReaderPtr reader_,
+        IAsynchronousReader & reader_,
         Int32 priority_,
         int & fd, /// Will be set to -1 if constructor didn't throw and ownership of file descriptor is passed to the object.
         const std::string & original_file_name = {},
@@ -45,7 +45,6 @@ public:
     }
 };
 
-
 /** Similar to AsynchronousReadBufferFromFile but also transparently shares open file descriptors.
   */
 class AsynchronousReadBufferFromFileWithDescriptorsCache : public AsynchronousReadBufferFromFileDescriptor
@@ -56,7 +55,7 @@ private:
 
 public:
     AsynchronousReadBufferFromFileWithDescriptorsCache(
-        AsynchronousReaderPtr reader_,
+        IAsynchronousReader & reader_,
         Int32 priority_,
         const std::string & file_name_,
         size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
@@ -64,7 +63,7 @@ public:
         char * existing_memory = nullptr,
         size_t alignment = 0,
         std::optional<size_t> file_size_ = std::nullopt)
-        : AsynchronousReadBufferFromFileDescriptor(std::move(reader_), priority_, -1, buf_size, existing_memory, alignment, file_size_)
+        : AsynchronousReadBufferFromFileDescriptor(reader_, priority_, -1, buf_size, existing_memory, alignment, file_size_)
         , file_name(file_name_)
     {
         file = OpenedFileCache::instance().get(file_name, flags);
@@ -80,4 +79,3 @@ public:
 };
 
 }
-

src/IO/AsynchronousReadBufferFromFileDescriptor.cpp

@@ -54,7 +54,7 @@ std::future<IAsynchronousReader::Result> AsynchronousReadBufferFromFileDescripto
         return std::async(std::launch::deferred, [] { return IAsynchronousReader::Result{.size = 0, .offset = 0}; });
     }
 
-    return reader->submit(request);
+    return reader.submit(request);
 }
 
 
@@ -140,7 +140,7 @@ void AsynchronousReadBufferFromFileDescriptor::finalize()
 
 
 AsynchronousReadBufferFromFileDescriptor::AsynchronousReadBufferFromFileDescriptor(
-    AsynchronousReaderPtr reader_,
+    IAsynchronousReader & reader_,
     Int32 priority_,
     int fd_,
     size_t buf_size,
@@ -148,7 +148,7 @@ AsynchronousReadBufferFromFileDescriptor::AsynchronousReadBufferFromFileDescript
     size_t alignment,
     std::optional<size_t> file_size_)
     : ReadBufferFromFileBase(buf_size, existing_memory, alignment, file_size_)
-    , reader(std::move(reader_))
+    , reader(reader_)
     , priority(priority_)
     , required_alignment(alignment)
     , fd(fd_)

src/IO/AsynchronousReadBufferFromFileDescriptor.h

@@ -16,7 +16,7 @@ namespace DB
 class AsynchronousReadBufferFromFileDescriptor : public ReadBufferFromFileBase
 {
 protected:
-    AsynchronousReaderPtr reader;
+    IAsynchronousReader & reader;
     Int32 priority;
 
     Memory<> prefetch_buffer;
@@ -36,7 +36,7 @@ protected:
 
 public:
     AsynchronousReadBufferFromFileDescriptor(
-        AsynchronousReaderPtr reader_,
+        IAsynchronousReader & reader_,
         Int32 priority_,
         int fd_,
         size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,

src/IO/AsynchronousReader.h

@@ -4,6 +4,7 @@
 #include <optional>
 #include <memory>
 #include <future>
+#include <boost/noncopyable.hpp>
 
 
 namespace DB
@@ -18,7 +19,7 @@ namespace DB
   * For example, this interface may not suffice if you want to serve 10 000 000 of 4 KiB requests per second.
   * This interface is fairly limited.
   */
-class IAsynchronousReader
+class IAsynchronousReader : private boost::noncopyable
 {
 public:
     /// For local filesystems, the file descriptor is simply integer
@@ -68,6 +69,8 @@ public:
     /// The method can be called concurrently from multiple threads.
     virtual std::future<Result> submit(Request request) = 0;
 
+    virtual void wait() = 0;
+
     /// Destructor must wait for all not completed request and ignore the results.
     /// It may also cancel the requests.
     virtual ~IAsynchronousReader() = default;

src/IO/ParallelReadBuffer.cpp

@@ -43,7 +43,7 @@ struct ParallelReadBuffer::ReadWorker
 };
 
 ParallelReadBuffer::ParallelReadBuffer(
-    std::unique_ptr<ReadBufferFactory> reader_factory_, CallbackRunner schedule_, size_t max_working_readers_)
+    std::unique_ptr<ReadBufferFactory> reader_factory_, ThreadPoolCallbackRunner<void> schedule_, size_t max_working_readers_)
     : SeekableReadBuffer(nullptr, 0)
     , max_working_readers(max_working_readers_)
     , schedule(std::move(schedule_))
@@ -71,7 +71,7 @@ bool ParallelReadBuffer::addReaderToPool()
     auto worker = read_workers.emplace_back(std::make_shared<ReadWorker>(std::move(reader)));
 
     ++active_working_reader;
-    schedule([this, worker = std::move(worker)]() mutable { readerThreadFunction(std::move(worker)); });
+    schedule([this, worker = std::move(worker)]() mutable { readerThreadFunction(std::move(worker)); }, 0);
 
     return true;
 }

src/IO/ParallelReadBuffer.h

@@ -33,12 +33,13 @@ public:
     class ReadBufferFactory : public WithFileSize
     {
     public:
+        ~ReadBufferFactory() override = default;
+
         virtual SeekableReadBufferPtr getReader() = 0;
-        virtual ~ReadBufferFactory() override = default;
         virtual off_t seek(off_t off, int whence) = 0;
     };
 
-    explicit ParallelReadBuffer(std::unique_ptr<ReadBufferFactory> reader_factory_, CallbackRunner schedule_, size_t max_working_readers);
+    ParallelReadBuffer(std::unique_ptr<ReadBufferFactory> reader_factory_, ThreadPoolCallbackRunner<void> schedule_, size_t max_working_readers);
 
     ~ParallelReadBuffer() override { finishAndWait(); }
 
@@ -75,7 +76,7 @@ private:
     size_t max_working_readers;
     std::atomic_size_t active_working_reader{0};
 
-    CallbackRunner schedule;
+    ThreadPoolCallbackRunner<void> schedule;
 
     std::unique_ptr<ReadBufferFactory> reader_factory;
 

src/IO/S3/PocoHTTPClient.cpp

@@ -1,3 +1,4 @@
+#include "Common/DNSResolver.h"
 #include <Common/config.h>
 
 #if USE_AWS_S3
@@ -257,6 +258,9 @@ void PocoHTTPClient::makeRequestInternal(
 
             if (!request_configuration.proxy_host.empty())
             {
+                if (enable_s3_requests_logging)
+                    LOG_TEST(log, "Due to reverse proxy host name ({}) won't be resolved on ClickHouse side", uri);
+
                 /// Reverse proxy can replace host header with resolved ip address instead of host name.
                 /// This can lead to request signature difference on S3 side.
                 session = makeHTTPSession(target_uri, timeouts, /* resolve_host = */ false);
@@ -443,6 +447,10 @@ void PocoHTTPClient::makeRequestInternal(
         response->SetClientErrorMessage(getCurrentExceptionMessage(false));
 
         addMetric(request, S3MetricType::Errors);
+
+        /// Probably this is socket timeout or something more or less related to DNS
+        /// Let's just remove this host from DNS cache to be more safe
+        DNSResolver::instance().removeHostFromCache(Poco::URI(uri).getHost());
     }
 }
 

src/IO/SynchronousReader.h

@@ -13,7 +13,8 @@ class SynchronousReader final : public IAsynchronousReader
 {
 public:
     std::future<Result> submit(Request request) override;
+
+    void wait() override {}
 };
 
 }
-

src/IO/WriteBufferFromS3.cpp

@@ -74,7 +74,7 @@ WriteBufferFromS3::WriteBufferFromS3(
     const S3Settings::ReadWriteSettings & s3_settings_,
     std::optional<std::map<String, String>> object_metadata_,
     size_t buffer_size_,
-    ScheduleFunc schedule_,
+    ThreadPoolCallbackRunner<void> schedule_,
     const WriteSettings & write_settings_)
     : BufferWithOwnMemory<WriteBuffer>(buffer_size_, nullptr, 0)
     , bucket(bucket_)
@@ -292,7 +292,7 @@ void WriteBufferFromS3::writePart()
                 }
 
                 task_finish_notify();
-            });
+            }, 0);
         }
         catch (...)
         {
@@ -442,7 +442,7 @@ void WriteBufferFromS3::makeSinglepartUpload()
                 }
 
                 task_notify_finish();
-            });
+            }, 0);
         }
         catch (...)
         {

src/IO/WriteBufferFromS3.h

@@ -15,6 +15,7 @@
 #include <IO/WriteBuffer.h>
 #include <IO/WriteSettings.h>
 #include <Storages/StorageS3Settings.h>
+#include <Interpreters/threadPoolCallbackRunner.h>
 
 #include <aws/core/utils/memory/stl/AWSStringStream.h>
 
@@ -33,7 +34,6 @@ namespace Aws::S3::Model
 namespace DB
 {
 
-using ScheduleFunc = std::function<void(std::function<void()>)>;
 class WriteBufferFromFile;
 
 /**
@@ -53,7 +53,7 @@ public:
         const S3Settings::ReadWriteSettings & s3_settings_,
         std::optional<std::map<String, String>> object_metadata_ = std::nullopt,
         size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE,
-        ScheduleFunc schedule_ = {},
+        ThreadPoolCallbackRunner<void> schedule_ = {},
         const WriteSettings & write_settings_ = {});
 
     ~WriteBufferFromS3() override;
@@ -106,7 +106,7 @@ private:
 
     /// Following fields are for background uploads in thread pool (if specified).
     /// We use std::function to avoid dependency of Interpreters
-    const ScheduleFunc schedule;
+    const ThreadPoolCallbackRunner<void> schedule;
 
     std::unique_ptr<PutObjectTask> put_object_task; /// Does not need protection by mutex because of the logic around is_finished field.
     std::list<UploadPartTask> TSA_GUARDED_BY(bg_tasks_mutex) upload_object_tasks;

src/Interpreters/Context.cpp

@@ -31,6 +31,9 @@
 #include <Storages/StorageS3Settings.h>
 #include <Disks/DiskLocal.h>
 #include <Disks/ObjectStorages/IObjectStorage.h>
+#include <Disks/IO/ThreadPoolRemoteFSReader.h>
+#include <Disks/IO/ThreadPoolReader.h>
+#include <IO/SynchronousReader.h>
 #include <TableFunctions/TableFunctionFactory.h>
 #include <Interpreters/ActionLocksManager.h>
 #include <Interpreters/ExternalLoaderXMLConfigRepository.h>
@@ -228,6 +231,12 @@ struct ContextSharedPart : boost::noncopyable
     mutable std::unique_ptr<BackgroundSchedulePool> distributed_schedule_pool; /// A thread pool that can run different jobs in background (used for distributed sends)
     mutable std::unique_ptr<BackgroundSchedulePool> message_broker_schedule_pool; /// A thread pool that can run different jobs in background (used for message brokers, like RabbitMQ and Kafka)
 
+    mutable std::unique_ptr<IAsynchronousReader> asynchronous_remote_fs_reader;
+    mutable std::unique_ptr<IAsynchronousReader> asynchronous_local_fs_reader;
+    mutable std::unique_ptr<IAsynchronousReader> synchronous_local_fs_reader;
+
+    mutable std::unique_ptr<ThreadPool> threadpool_writer;
+
     mutable ThrottlerPtr replicated_fetches_throttler;      /// A server-wide throttler for replicated fetches
     mutable ThrottlerPtr replicated_sends_throttler;        /// A server-wide throttler for replicated sends
     mutable ThrottlerPtr remote_read_throttler;             /// A server-wide throttler for remote IO reads
@@ -310,25 +319,78 @@ struct ContextSharedPart : boost::noncopyable
 
 
     ~ContextSharedPart()
+    {
+        /// Wait for thread pool for background reads and writes,
+        /// since it may use per-user MemoryTracker which will be destroyed here.
+        if (asynchronous_remote_fs_reader)
         {
             try
             {
-            /// Wait for thread pool for background writes,
+                LOG_DEBUG(log, "Desctructing remote fs threadpool reader");
-            /// since it may use per-user MemoryTracker which will be destroyed here.
+                asynchronous_remote_fs_reader->wait();
-            IObjectStorage::getThreadPoolWriter().wait();
+                asynchronous_remote_fs_reader.reset();
-            /// Make sure that threadpool is destructed before this->process_list
-            /// because thread_status, which was created for threads inside threadpool,
-            /// relies on it.
-            if (load_marks_threadpool)
-            {
-                load_marks_threadpool->wait();
-                load_marks_threadpool.reset();
-            }
             }
             catch (...)
             {
                 tryLogCurrentException(__PRETTY_FUNCTION__);
             }
+        }
+
+        if (asynchronous_local_fs_reader)
+        {
+            try
+            {
+                LOG_DEBUG(log, "Desctructing local fs threadpool reader");
+                asynchronous_local_fs_reader->wait();
+                asynchronous_local_fs_reader.reset();
+            }
+            catch (...)
+            {
+                tryLogCurrentException(__PRETTY_FUNCTION__);
+            }
+        }
+
+        if (synchronous_local_fs_reader)
+        {
+            try
+            {
+                LOG_DEBUG(log, "Desctructing local fs threadpool reader");
+                synchronous_local_fs_reader->wait();
+                synchronous_local_fs_reader.reset();
+            }
+            catch (...)
+            {
+                tryLogCurrentException(__PRETTY_FUNCTION__);
+            }
+        }
+
+        if (threadpool_writer)
+        {
+            try
+            {
+                LOG_DEBUG(log, "Desctructing threadpool writer");
+                threadpool_writer->wait();
+                threadpool_writer.reset();
+            }
+            catch (...)
+            {
+                tryLogCurrentException(__PRETTY_FUNCTION__);
+            }
+        }
+
+        if (load_marks_threadpool)
+        {
+            try
+            {
+                LOG_DEBUG(log, "Desctructing marks loader");
+                load_marks_threadpool->wait();
+                load_marks_threadpool.reset();
+            }
+            catch (...)
+            {
+                tryLogCurrentException(__PRETTY_FUNCTION__);
+            }
+        }
 
         try
         {
@@ -3378,6 +3440,66 @@ OrdinaryBackgroundExecutorPtr Context::getCommonExecutor() const
     return shared->common_executor;
 }
 
+IAsynchronousReader & Context::getThreadPoolReader(FilesystemReaderType type) const
+{
+    const auto & config = getConfigRef();
+
+    auto lock = getLock();
+
+    switch (type)
+    {
+        case FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER:
+        {
+            if (!shared->asynchronous_remote_fs_reader)
+            {
+                auto pool_size = config.getUInt(".threadpool_remote_fs_reader_pool_size", 100);
+                auto queue_size = config.getUInt(".threadpool_remote_fs_reader_queue_size", 1000000);
+
+                shared->asynchronous_remote_fs_reader = std::make_unique<ThreadPoolRemoteFSReader>(pool_size, queue_size);
+            }
+
+            return *shared->asynchronous_remote_fs_reader;
+        }
+        case FilesystemReaderType::ASYNCHRONOUS_LOCAL_FS_READER:
+        {
+            if (!shared->asynchronous_local_fs_reader)
+            {
+                auto pool_size = config.getUInt(".threadpool_local_fs_reader_pool_size", 100);
+                auto queue_size = config.getUInt(".threadpool_local_fs_reader_queue_size", 1000000);
+
+                shared->asynchronous_local_fs_reader = std::make_unique<ThreadPoolReader>(pool_size, queue_size);
+            }
+
+            return *shared->asynchronous_local_fs_reader;
+        }
+        case FilesystemReaderType::SYNCHRONOUS_LOCAL_FS_READER:
+        {
+            if (!shared->synchronous_local_fs_reader)
+            {
+                shared->synchronous_local_fs_reader = std::make_unique<SynchronousReader>();
+            }
+
+            return *shared->synchronous_local_fs_reader;
+        }
+    }
+}
+
+ThreadPool & Context::getThreadPoolWriter() const
+{
+    const auto & config = getConfigRef();
+
+    auto lock = getLock();
+
+    if (!shared->threadpool_writer)
+    {
+        auto pool_size = config.getUInt(".threadpool_writer_pool_size", 100);
+        auto queue_size = config.getUInt(".threadpool_writer_queue_size", 1000000);
+
+        shared->threadpool_writer = std::make_unique<ThreadPool>(pool_size, pool_size, queue_size);
+    }
+
+    return *shared->threadpool_writer;
+}
 
 ReadSettings Context::getReadSettings() const
 {

src/Interpreters/Context.h

@@ -1011,6 +1011,17 @@ public:
     OrdinaryBackgroundExecutorPtr getFetchesExecutor() const;
     OrdinaryBackgroundExecutorPtr getCommonExecutor() const;
 
+    enum class FilesystemReaderType
+    {
+        SYNCHRONOUS_LOCAL_FS_READER,
+        ASYNCHRONOUS_LOCAL_FS_READER,
+        ASYNCHRONOUS_REMOTE_FS_READER,
+    };
+
+    IAsynchronousReader & getThreadPoolReader(FilesystemReaderType type) const;
+
+    ThreadPool & getThreadPoolWriter() const;
+
     /** Get settings for reading from filesystem. */
     ReadSettings getReadSettings() const;
 

src/Interpreters/DDLTask.cpp

@@ -50,21 +50,26 @@ bool HostID::isLocalAddress(UInt16 clickhouse_port) const
 
 void DDLLogEntry::assertVersion() const
 {
-    constexpr UInt64 max_version = 2;
+    if (version == 0
-    if (version == 0 || max_version < version)
+    /// NORMALIZE_CREATE_ON_INITIATOR_VERSION does not change the entry format, it uses versioin 2, so there shouldn't be such version
+    || version == NORMALIZE_CREATE_ON_INITIATOR_VERSION
+    || version > DDL_ENTRY_FORMAT_MAX_VERSION)
         throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unknown DDLLogEntry format version: {}."
-                                                            "Maximum supported version is {}", version, max_version);
+                                                            "Maximum supported version is {}", version, DDL_ENTRY_FORMAT_MAX_VERSION);
 }
 
 void DDLLogEntry::setSettingsIfRequired(ContextPtr context)
 {
     version = context->getSettingsRef().distributed_ddl_entry_format_version;
+    if (version <= 0 || version > DDL_ENTRY_FORMAT_MAX_VERSION)
+        throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unknown distributed_ddl_entry_format_version: {}."
+                                                            "Maximum supported version is {}.", version, DDL_ENTRY_FORMAT_MAX_VERSION);
 
     /// NORMALIZE_CREATE_ON_INITIATOR_VERSION does not affect entry format in ZooKeeper
     if (version == NORMALIZE_CREATE_ON_INITIATOR_VERSION)
         version = SETTINGS_IN_ZK_VERSION;
 
-    if (version == SETTINGS_IN_ZK_VERSION)
+    if (version >= SETTINGS_IN_ZK_VERSION)
         settings.emplace(context->getSettingsRef().changes());
 }
 
@@ -94,6 +99,9 @@ String DDLLogEntry::toString() const
         wb << "settings: " << serializeAST(ast) << "\n";
     }
 
+    if (version >= OPENTELEMETRY_ENABLED_VERSION)
+        wb << "tracing: " << this->tracing_context;
+
     return wb.str();
 }
 
@@ -106,7 +114,7 @@ void DDLLogEntry::parse(const String & data)
 
     Strings host_id_strings;
     rb >> "query: " >> escape >> query >> "\n";
-    if (version == 1)
+    if (version == OLDEST_VERSION)
     {
         rb >> "hosts: " >> host_id_strings >> "\n";
 
@@ -115,9 +123,8 @@ void DDLLogEntry::parse(const String & data)
         else
             initiator.clear();
     }
-    else if (version == 2)
+    else if (version >= SETTINGS_IN_ZK_VERSION)
     {
-
         if (!rb.eof() && *rb.position() == 'h')
             rb >> "hosts: " >> host_id_strings >> "\n";
         if (!rb.eof() && *rb.position() == 'i')
@@ -134,6 +141,12 @@ void DDLLogEntry::parse(const String & data)
         }
     }
 
+    if (version >= OPENTELEMETRY_ENABLED_VERSION)
+    {
+        if (!rb.eof() && *rb.position() == 't')
+            rb >> "tracing: " >> this->tracing_context;
+    }
+
     assertEOF(rb);
 
     if (!host_id_strings.empty())

src/Interpreters/DDLTask.h

@@ -2,6 +2,7 @@
 
 #include <Core/Types.h>
 #include <Interpreters/Cluster.h>
+#include <Common/OpenTelemetryTraceContext.h>
 #include <Common/ZooKeeper/Types.h>
 #include <filesystem>
 
@@ -69,12 +70,18 @@ struct DDLLogEntry
     static constexpr const UInt64 OLDEST_VERSION = 1;
     static constexpr const UInt64 SETTINGS_IN_ZK_VERSION = 2;
     static constexpr const UInt64 NORMALIZE_CREATE_ON_INITIATOR_VERSION = 3;
+    static constexpr const UInt64 OPENTELEMETRY_ENABLED_VERSION = 4;
+    /// Add new version here
+
+    /// Remember to update the value below once new version is added
+    static constexpr const UInt64 DDL_ENTRY_FORMAT_MAX_VERSION = 4;
 
     UInt64 version = 1;
     String query;
     std::vector<HostID> hosts;
     String initiator; // optional
     std::optional<SettingsChanges> settings;
+    OpenTelemetry::TracingContext tracing_context;
 
     void setSettingsIfRequired(ContextPtr context);
     String toString() const;

src/Interpreters/DDLWorker.cpp

@@ -19,6 +19,7 @@
 #include <Interpreters/executeQuery.h>
 #include <Interpreters/Cluster.h>
 #include <Interpreters/Context.h>
+#include <Common/OpenTelemetryTraceContext.h>
 #include <Common/setThreadName.h>
 #include <Common/randomSeed.h>
 #include <Common/ZooKeeper/ZooKeeper.h>
@@ -515,6 +516,11 @@ void DDLWorker::processTask(DDLTaskBase & task, const ZooKeeperPtr & zookeeper)
     LOG_DEBUG(log, "Processing task {} ({})", task.entry_name, task.entry.query);
     chassert(!task.completely_processed);
 
+    /// Setup tracing context on current thread for current DDL
+    OpenTelemetry::TracingContextHolder tracing_ctx_holder(__PRETTY_FUNCTION__ ,
+        task.entry.tracing_context,
+        this->context->getOpenTelemetrySpanLog());
+
     String active_node_path = task.getActiveNodePath();
     String finished_node_path = task.getFinishedNodePath();
 

src/Interpreters/InterpreterCreateQuery.cpp

@@ -766,6 +766,16 @@ void InterpreterCreateQuery::validateTableStructure(const ASTCreateQuery & creat
             throw Exception("Column " + backQuoteIfNeed(column.name) + " already exists", ErrorCodes::DUPLICATE_COLUMN);
     }
 
+    /// Check if _row_exists for lightweight delete column in column_lists for merge tree family.
+    if (create.storage && create.storage->engine && endsWith(create.storage->engine->name, "MergeTree"))
+    {
+        auto search = all_columns.find(LightweightDeleteDescription::FILTER_COLUMN.name);
+        if (search != all_columns.end())
+            throw Exception("Cannot create table with column '" + LightweightDeleteDescription::FILTER_COLUMN.name + "' "
+                            "for *MergeTree engines because it is reserved for lightweight delete feature",
+                            ErrorCodes::ILLEGAL_COLUMN);
+    }
+
     const auto & settings = getContext()->getSettingsRef();
 
     /// Check low cardinality types in creating table if it was not allowed in setting

src/Interpreters/ThreadStatusExt.cpp

@@ -350,7 +350,10 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits)
 
     /// Avoid leaking of ThreadGroupStatus::finished_threads_counters_memory
     /// (this is in case someone uses system thread but did not call getProfileEventsCountersAndMemoryForThreads())
-    thread_group->getProfileEventsCountersAndMemoryForThreads();
+    {
+        std::lock_guard guard(thread_group->mutex);
+        auto stats = std::move(thread_group->finished_threads_counters_memory);
+    }
 
     thread_group.reset();
 

src/Interpreters/executeDDLQueryOnCluster.cpp

@@ -55,6 +55,8 @@ bool isSupportedAlterType(int type)
 
 BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, ContextPtr context, const DDLQueryOnClusterParams & params)
 {
+    OpenTelemetry::SpanHolder span(__FUNCTION__);
+
     if (context->getCurrentTransaction() && context->getSettingsRef().throw_on_unsupported_query_inside_transaction)
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "ON CLUSTER queries inside transactions are not supported");
 
@@ -88,6 +90,8 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, ContextPtr context,
         cluster = context->getCluster(query->cluster);
     }
 
+    span.addAttribute("clickhouse.cluster", query->cluster);
+
     /// TODO: support per-cluster grant
     context->checkAccess(AccessType::CLUSTER);
 
@@ -164,6 +168,7 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, ContextPtr context,
     entry.query = queryToString(query_ptr);
     entry.initiator = ddl_worker.getCommonHostID();
     entry.setSettingsIfRequired(context);
+    entry.tracing_context = OpenTelemetry::CurrentContext();
     String node_path = ddl_worker.enqueueQuery(entry);
 
     return getDistributedDDLStatus(node_path, entry, context);

src/Interpreters/executeQuery.cpp

@@ -73,6 +73,7 @@
 #include <memory>
 #include <random>
 
+#include <Parsers/Kusto/ParserKQLStatement.h>
 
 namespace ProfileEvents
 {
@@ -391,11 +392,21 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
     String query_database;
     String query_table;
     try
+    {
+        if (settings.dialect == Dialect::kusto && !internal)
+        {
+            ParserKQLStatement parser(end, settings.allow_settings_after_format_in_insert);
+
+            /// TODO: parser should fail early when max_query_size limit is reached.
+            ast = parseQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth);
+        }
+        else
         {
             ParserQuery parser(end, settings.allow_settings_after_format_in_insert);
 
             /// TODO: parser should fail early when max_query_size limit is reached.
             ast = parseQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth);
+        }
 
         if (auto txn = context->getCurrentTransaction())
         {

src/Interpreters/threadPoolCallbackRunner.cpp

@@ -1,18 +1,20 @@
 #include "threadPoolCallbackRunner.h"
 
 #include <Common/scope_guard_safe.h>
-
 #include <Common/CurrentThread.h>
+#include <Common/setThreadName.h>
+#include <IO/AsynchronousReader.h>
+#include <future>
+
 
 namespace DB
 {
 
-CallbackRunner threadPoolCallbackRunner(ThreadPool & pool)
+template <typename Result> ThreadPoolCallbackRunner<Result> threadPoolCallbackRunner(ThreadPool & pool, const std::string & thread_name)
 {
-    return [pool = &pool, thread_group = CurrentThread::getGroup()](auto callback) mutable
+    return [pool = &pool, thread_group = CurrentThread::getGroup(), thread_name](std::function<Result()> && callback, size_t priority) mutable -> std::future<Result>
     {
-        pool->scheduleOrThrow(
+        auto task = std::make_shared<std::packaged_task<Result()>>([thread_group, thread_name, callback = std::move(callback)]() -> Result
-            [&, callback = std::move(callback), thread_group]()
         {
             if (thread_group)
                 CurrentThread::attachTo(thread_group);
@@ -20,21 +22,23 @@ CallbackRunner threadPoolCallbackRunner(ThreadPool & pool)
             SCOPE_EXIT_SAFE({
                 if (thread_group)
                     CurrentThread::detachQueryIfNotDetached();
+            });
 
-                    /// After we detached from the thread_group, parent for memory_tracker inside ThreadStatus will be reset to it's parent.
+            setThreadName(thread_name.data());
-                    /// Typically, it may be changes from Process to User.
+
-                    /// Usually it could be ok, because thread pool task is executed before user-level memory tracker is destroyed.
+            return callback();
-                    /// However, thread could stay alive inside the thread pool, and it's ThreadStatus as well.
-                    /// When, finally, we destroy the thread (and the ThreadStatus),
-                    /// it can use memory tracker in the ~ThreadStatus in order to alloc/free untracked_memory,
-                    /// and by this time user-level memory tracker may be already destroyed.
-                    ///
-                    /// As a work-around, reset memory tracker to total, which is always alive.
-                    CurrentThread::get().memory_tracker.setParent(&total_memory_tracker);
-                });
-                callback();
         });
+
+        auto future = task->get_future();
+
+        /// ThreadPool is using "bigger is higher priority" instead of "smaller is more priority".
+        pool->scheduleOrThrow([task]{ (*task)(); }, -priority);
+
+        return future;
     };
 }
 
+template ThreadPoolCallbackRunner<void> threadPoolCallbackRunner(ThreadPool & pool, const std::string & thread_name);
+template ThreadPoolCallbackRunner<IAsynchronousReader::Result> threadPoolCallbackRunner(ThreadPool & pool, const std::string & thread_name);
+
 }

src/Interpreters/threadPoolCallbackRunner.h

@@ -1,15 +1,18 @@
 #pragma once
 
 #include <Common/ThreadPool.h>
+#include <future>
 
 
 namespace DB
 {
 
-/// High-order function to run callbacks (functions with 'void()' signature) somewhere asynchronously
+/// High-order function to run callbacks (functions with 'void()' signature) somewhere asynchronously.
-using CallbackRunner = std::function<void(std::function<void()>)>;
+template <typename Result>
+using ThreadPoolCallbackRunner = std::function<std::future<Result>(std::function<Result()> &&, size_t priority)>;
 
-/// Creates CallbackRunner that runs every callback with 'pool->scheduleOrThrow()'
+/// Creates CallbackRunner that runs every callback with 'pool->scheduleOrThrow()'.
-CallbackRunner threadPoolCallbackRunner(ThreadPool & pool);
+template <typename Result>
+ThreadPoolCallbackRunner<Result> threadPoolCallbackRunner(ThreadPool & pool, const std::string & thread_name);
 
 }

src/Parsers/CMakeLists.txt

@@ -3,6 +3,7 @@ include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake")
 add_headers_and_sources(clickhouse_parsers .)
 add_headers_and_sources(clickhouse_parsers ./Access)
 add_headers_and_sources(clickhouse_parsers ./MySQL)
+add_headers_and_sources(clickhouse_parsers ./Kusto)
 add_library(clickhouse_parsers ${clickhouse_parsers_headers} ${clickhouse_parsers_sources})
 target_link_libraries(clickhouse_parsers PUBLIC clickhouse_common_io clickhouse_common_access string_utils)
 

src/Parsers/ExpressionElementParsers.h

@@ -9,26 +9,6 @@ namespace DB
 {
 
 
-class ParserArray : public IParserBase
-{
-protected:
-    const char * getName() const override { return "array"; }
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
-
-/** If in parenthesis an expression from one element - returns this element in `node`;
-  *  or if there is a SELECT subquery in parenthesis, then this subquery returned in `node`;
-  *  otherwise returns `tuple` function from the contents of brackets.
-  */
-class ParserParenthesisExpression : public IParserBase
-{
-protected:
-    const char * getName() const override { return "parenthesized expression"; }
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
-
 /** The SELECT subquery is in parenthesis.
   */
 class ParserSubquery : public IParserBase
@@ -141,36 +121,6 @@ protected:
     ColumnTransformers allowed_transformers;
 };
 
-/** A function, for example, f(x, y + 1, g(z)).
-  * Or an aggregate function: sum(x + f(y)), corr(x, y). The syntax is the same as the usual function.
-  * Or a parametric aggregate function: quantile(0.9)(x + y).
-  *  Syntax - two pairs of parentheses instead of one. The first is for parameters, the second for arguments.
-  * For functions, the DISTINCT modifier can be specified, for example, count(DISTINCT x, y).
-  */
-class ParserFunction : public IParserBase
-{
-public:
-    explicit ParserFunction(bool allow_function_parameters_ = true, bool is_table_function_ = false)
-        : allow_function_parameters(allow_function_parameters_), is_table_function(is_table_function_)
-    {
-    }
-
-protected:
-    const char * getName() const override { return "function"; }
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-    bool allow_function_parameters;
-    bool is_table_function;
-};
-
-// A special function parser for view and viewIfPermitted table functions.
-// It parses an SELECT query as its argument and doesn't support getColumnName().
-class ParserTableFunctionView : public IParserBase
-{
-protected:
-    const char * getName() const override { return "function"; }
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
 // Allows to make queries like SELECT SUM(<expr>) FILTER(WHERE <cond>) FROM ...
 class ParserFilterClause : public IParserBase
 {
@@ -394,16 +344,6 @@ protected:
 };
 
 
-/** The expression element is one of: an expression in parentheses, an array, a literal, a function, an identifier, an asterisk.
-  */
-class ParserExpressionElement : public IParserBase
-{
-protected:
-    const char * getName() const override { return "element of expression"; }
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
-
 /** An expression element, possibly with an alias, if appropriate.
   */
 class ParserWithOptionalAlias : public IParserBase

src/Parsers/ExpressionListParsers.h

@@ -116,6 +116,36 @@ private:
     SelectUnionModes union_modes;
 };
 
+
+class ParserArray : public IParserBase
+{
+protected:
+    const char * getName() const override { return "array"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
+
+/** A function, for example, f(x, y + 1, g(z)).
+  * Or an aggregate function: sum(x + f(y)), corr(x, y). The syntax is the same as the usual function.
+  * Or a parametric aggregate function: quantile(0.9)(x + y).
+  *  Syntax - two pairs of parentheses instead of one. The first is for parameters, the second for arguments.
+  * For functions, the DISTINCT modifier can be specified, for example, count(DISTINCT x, y).
+  */
+class ParserFunction : public IParserBase
+{
+public:
+    explicit ParserFunction(bool allow_function_parameters_ = true, bool is_table_function_ = false)
+        : allow_function_parameters(allow_function_parameters_), is_table_function(is_table_function_)
+    {
+    }
+
+protected:
+    const char * getName() const override { return "function"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+    bool allow_function_parameters;
+    bool is_table_function;
+};
+
+
 /** An expression with an infix binary left-associative operator.
   * For example, a + b - c + d.
   */
@@ -123,31 +153,13 @@ class ParserLeftAssociativeBinaryOperatorList : public IParserBase
 {
 private:
     Operators_t operators;
-    Operators_t overlapping_operators_to_skip = { (const char *[]){ nullptr } };
+    ParserPtr elem_parser;
-    ParserPtr first_elem_parser;
-    ParserPtr remaining_elem_parser;
-    /// =, !=, <, > ALL (subquery) / ANY (subquery)
-    bool comparison_expression = false;
 
 public:
     /** `operators_` - allowed operators and their corresponding functions
       */
-    ParserLeftAssociativeBinaryOperatorList(Operators_t operators_, ParserPtr && first_elem_parser_)
+    ParserLeftAssociativeBinaryOperatorList(Operators_t operators_, ParserPtr && elem_parser_)
-        : operators(operators_), first_elem_parser(std::move(first_elem_parser_))
+        : operators(operators_), elem_parser(std::move(elem_parser_))
-    {
-    }
-
-    ParserLeftAssociativeBinaryOperatorList(Operators_t operators_,
-            Operators_t overlapping_operators_to_skip_, ParserPtr && first_elem_parser_, bool comparison_expression_ = false)
-        : operators(operators_), overlapping_operators_to_skip(overlapping_operators_to_skip_),
-          first_elem_parser(std::move(first_elem_parser_)), comparison_expression(comparison_expression_)
-    {
-    }
-
-    ParserLeftAssociativeBinaryOperatorList(Operators_t operators_, ParserPtr && first_elem_parser_,
-        ParserPtr && remaining_elem_parser_)
-        : operators(operators_), first_elem_parser(std::move(first_elem_parser_)),
-          remaining_elem_parser(std::move(remaining_elem_parser_))
     {
     }
 
@@ -158,295 +170,8 @@ protected:
 };
 
 
-/** Expression with an infix operator of arbitrary arity.
+class ParserExpression : public IParserBase
-  * For example, a AND b AND c AND d.
-  */
-class ParserVariableArityOperatorList : public IParserBase
 {
-private:
-    const char * infix;
-    const char * function_name;
-    ParserPtr elem_parser;
-
-public:
-    ParserVariableArityOperatorList(const char * infix_, const char * function_, ParserPtr && elem_parser_)
-        : infix(infix_), function_name(function_), elem_parser(std::move(elem_parser_))
-    {
-    }
-
-protected:
-    const char * getName() const override { return "list, delimited by operator of variable arity"; }
-
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
-
-/** An expression with a prefix unary operator.
-  * Example, NOT x.
-  */
-class ParserPrefixUnaryOperatorExpression : public IParserBase
-{
-private:
-    Operators_t operators;
-    ParserPtr elem_parser;
-
-public:
-    /** `operators_` - allowed operators and their corresponding functions
-      */
-    ParserPrefixUnaryOperatorExpression(Operators_t operators_, ParserPtr && elem_parser_)
-        : operators(operators_), elem_parser(std::move(elem_parser_))
-    {
-    }
-
-protected:
-    const char * getName() const override { return "expression with prefix unary operator"; }
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
-/// CAST operator "::". This parser is used if left argument
-/// of operator cannot be read as simple literal from text of query.
-/// Example: "[1, 1 + 1, 1 + 2]::Array(UInt8)"
-class ParserCastExpression : public IParserBase
-{
-private:
-    ParserPtr elem_parser;
-
-public:
-    explicit ParserCastExpression(ParserPtr && elem_parser_)
-        : elem_parser(std::move(elem_parser_))
-    {
-    }
-
-protected:
-    const char * getName() const override { return "CAST expression"; }
-
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
-
-class ParserArrayElementExpression : public IParserBase
-{
-private:
-    static const char * operators[];
-
-protected:
-    const char * getName() const  override{ return "array element expression"; }
-
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
-
-class ParserTupleElementExpression : public IParserBase
-{
-private:
-    static const char * operators[];
-
-protected:
-    const char * getName() const override { return "tuple element expression"; }
-
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
-
-class ParserUnaryExpression : public IParserBase
-{
-private:
-    static const char * operators[];
-    ParserPrefixUnaryOperatorExpression operator_parser {operators, std::make_unique<ParserCastExpression>(std::make_unique<ParserTupleElementExpression>())};
-
-protected:
-    const char * getName() const override { return "unary expression"; }
-
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
-
-class ParserMultiplicativeExpression : public IParserBase
-{
-private:
-    static const char * operators[];
-    ParserLeftAssociativeBinaryOperatorList operator_parser {operators, std::make_unique<ParserUnaryExpression>()};
-
-protected:
-    const char * getName() const  override { return "multiplicative expression"; }
-
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
-    {
-        return operator_parser.parse(pos, node, expected);
-    }
-};
-
-/// DATE operator. "DATE '2001-01-01'" would be parsed as "toDate('2001-01-01')".
-class ParserDateOperatorExpression : public IParserBase
-{
-protected:
-    ParserMultiplicativeExpression next_parser;
-
-    const char * getName() const  override { return "DATE operator expression"; }
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
-/// TIMESTAMP operator. "TIMESTAMP '2001-01-01 12:34:56'" would be parsed as "toDateTime('2001-01-01 12:34:56')".
-class ParserTimestampOperatorExpression : public IParserBase
-{
-protected:
-    ParserDateOperatorExpression next_parser;
-
-    const char * getName() const  override { return "TIMESTAMP operator expression"; }
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
-/// Optional conversion to INTERVAL data type. Example: "INTERVAL x SECOND" parsed as "toIntervalSecond(x)".
-class ParserIntervalOperatorExpression : public IParserBase
-{
-protected:
-    ParserTimestampOperatorExpression next_parser;
-
-    const char * getName() const  override { return "INTERVAL operator expression"; }
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-
-private:
-    static bool parseArgumentAndIntervalKind(Pos & pos, ASTPtr & expr, IntervalKind & interval_kind, Expected & expected);
-};
-
-class ParserAdditiveExpression : public IParserBase
-{
-private:
-    static const char * operators[];
-    ParserLeftAssociativeBinaryOperatorList operator_parser {operators, std::make_unique<ParserIntervalOperatorExpression>()};
-
-protected:
-    const char * getName() const  override { return "additive expression"; }
-
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
-    {
-        return operator_parser.parse(pos, node, expected);
-    }
-};
-
-
-class ParserConcatExpression : public IParserBase
-{
-    ParserVariableArityOperatorList operator_parser {"||", "concat", std::make_unique<ParserAdditiveExpression>()};
-
-protected:
-    const char * getName() const override { return "string concatenation expression"; }
-
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
-    {
-        return operator_parser.parse(pos, node, expected);
-    }
-};
-
-
-class ParserBetweenExpression : public IParserBase
-{
-private:
-    ParserConcatExpression elem_parser;
-
-protected:
-    const char * getName() const override { return "BETWEEN expression"; }
-
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
-
-class ParserComparisonExpression : public IParserBase
-{
-private:
-    static const char * operators[];
-    static const char * overlapping_operators_to_skip[];
-    ParserLeftAssociativeBinaryOperatorList operator_parser {operators,
-        overlapping_operators_to_skip, std::make_unique<ParserBetweenExpression>(), true};
-
-protected:
-    const char * getName() const  override{ return "comparison expression"; }
-
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
-    {
-        return operator_parser.parse(pos, node, expected);
-    }
-};
-
-/** Parser for nullity checking with IS (NOT) NULL.
-  */
-class ParserNullityChecking : public IParserBase
-{
-private:
-    ParserComparisonExpression elem_parser;
-
-protected:
-    const char * getName() const override { return "nullity checking"; }
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
-
-class ParserLogicalNotExpression : public IParserBase
-{
-private:
-    static const char * operators[];
-    ParserPrefixUnaryOperatorExpression operator_parser {operators, std::make_unique<ParserNullityChecking>()};
-
-protected:
-    const char * getName() const  override{ return "logical-NOT expression"; }
-
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
-    {
-        return operator_parser.parse(pos, node, expected);
-    }
-};
-
-
-class ParserLogicalAndExpression : public IParserBase
-{
-private:
-    ParserVariableArityOperatorList operator_parser {"AND", "and", std::make_unique<ParserLogicalNotExpression>()};
-
-protected:
-    const char * getName() const override { return "logical-AND expression"; }
-
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
-    {
-        return operator_parser.parse(pos, node, expected);
-    }
-};
-
-
-class ParserLogicalOrExpression : public IParserBase
-{
-private:
-    ParserVariableArityOperatorList operator_parser {"OR", "or", std::make_unique<ParserLogicalAndExpression>()};
-
-protected:
-    const char * getName() const override { return "logical-OR expression"; }
-
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
-    {
-        return operator_parser.parse(pos, node, expected);
-    }
-};
-
-
-/** An expression with ternary operator.
-  * For example, a = 1 ? b + 1 : c * 2.
-  */
-class ParserTernaryOperatorExpression : public IParserBase
-{
-private:
-    ParserLogicalOrExpression elem_parser;
-
-protected:
-    const char * getName() const override { return "expression with ternary operator"; }
-
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
-
-class ParserLambdaExpression : public IParserBase
-{
-private:
-    ParserTernaryOperatorExpression elem_parser;
-
 protected:
     const char * getName() const override { return "lambda expression"; }
 
@@ -457,9 +182,6 @@ protected:
 // It's used to parse expressions in table function.
 class ParserTableFunctionExpression : public IParserBase
 {
-private:
-    ParserLambdaExpression elem_parser;
-
 protected:
     const char * getName() const override { return "table function expression"; }
 
@@ -467,13 +189,10 @@ protected:
 };
 
 
-using ParserExpression = ParserLambdaExpression;
-
-
 class ParserExpressionWithOptionalAlias : public IParserBase
 {
 public:
-    explicit ParserExpressionWithOptionalAlias(bool allow_alias_without_as_keyword, bool is_table_function = false);
+    explicit ParserExpressionWithOptionalAlias(bool allow_alias_without_as_keyword_, bool is_table_function_ = false);
 protected:
     ParserPtr impl;
 

src/Parsers/IAST.cpp

@@ -138,14 +138,28 @@ void IAST::updateTreeHashImpl(SipHash & hash_state) const
 }
 
 
-size_t IAST::checkDepthImpl(size_t max_depth, size_t level) const
+size_t IAST::checkDepthImpl(size_t max_depth) const
 {
-    size_t res = level + 1;
+    std::vector<std::pair<ASTPtr, size_t>> stack;
-    for (const auto & child : children)
+    stack.reserve(children.size());
+
+    for (const auto & i: children)
+        stack.push_back({i, 1});
+
+    size_t res = 0;
+
+    while (!stack.empty())
     {
-        if (level >= max_depth)
+        auto top = stack.back();
+        stack.pop_back();
+
+        if (top.second >= max_depth)
             throw Exception("AST is too deep. Maximum: " + toString(max_depth), ErrorCodes::TOO_DEEP_AST);
-        res = std::max(res, child->checkDepthImpl(max_depth, level + 1));
+
+        res = std::max(res, top.second);
+
+        for (const auto & i: top.first->children)
+            stack.push_back({i, top.second + 1});
     }
 
     return res;

src/Parsers/IAST.h

@@ -92,7 +92,7 @@ public:
       */
     size_t checkDepth(size_t max_depth) const
     {
-        return checkDepthImpl(max_depth, 0);
+        return checkDepthImpl(max_depth);
     }
 
     /** Get total number of tree elements
@@ -273,7 +273,7 @@ public:
     static const char * hilite_none;
 
 private:
-    size_t checkDepthImpl(size_t max_depth, size_t level) const;
+    size_t checkDepthImpl(size_t max_depth) const;
 
     /** Forward linked list of ASTPtr to delete.
       * Used in IAST destructor to avoid possible stack overflow.

src/Parsers/Kusto/ParserKQLFilter.cpp

@@ -0,0 +1,26 @@
+#include <Parsers/ASTLiteral.h>
+#include <Parsers/IParserBase.h>
+#include <Parsers/ExpressionListParsers.h>
+#include <Parsers/Kusto/ParserKQLQuery.h>
+#include <Parsers/Kusto/ParserKQLFilter.h>
+#include <Parsers/Kusto/ParserKQLOperators.h>
+
+namespace DB
+{
+
+bool ParserKQLFilter :: parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    String expr = getExprFromToken(pos);
+    ASTPtr where_expression;
+
+    Tokens token_filter(expr.c_str(), expr.c_str()+expr.size());
+    IParser::Pos pos_filter(token_filter, pos.max_depth);
+    if (!ParserExpressionWithOptionalAlias(false).parse(pos_filter, where_expression, expected))
+        return false;
+
+    node->as<ASTSelectQuery>()->setExpression(ASTSelectQuery::Expression::WHERE, std::move(where_expression));
+
+    return true;
+}
+
+}

src/Parsers/Kusto/ParserKQLFilter.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <Parsers/IParserBase.h>
+#include <Parsers/Kusto/ParserKQLQuery.h>
+
+namespace DB
+{
+
+class ParserKQLFilter : public ParserKQLBase
+{
+protected:
+    const char * getName() const override { return "KQL where"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
+
+}

src/Parsers/Kusto/ParserKQLLimit.cpp

@@ -0,0 +1,29 @@
+#include <Parsers/IParserBase.h>
+#include <Parsers/ExpressionListParsers.h>
+#include <Parsers/Kusto/ParserKQLQuery.h>
+#include <Parsers/Kusto/ParserKQLLimit.h>
+#include <Parsers/ParserTablesInSelectQuery.h>
+#include <cstdlib>
+#include <format>
+
+namespace DB
+{
+
+bool ParserKQLLimit :: parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    ASTPtr limit_length;
+
+    auto expr = getExprFromToken(pos);
+
+    Tokens tokens(expr.c_str(), expr.c_str() + expr.size());
+    IParser::Pos new_pos(tokens, pos.max_depth);
+
+    if (!ParserExpressionWithOptionalAlias(false).parse(new_pos, limit_length, expected))
+        return false;
+
+    node->as<ASTSelectQuery>()->setExpression(ASTSelectQuery::Expression::LIMIT_LENGTH, std::move(limit_length));
+
+    return true;
+}
+
+}

src/Parsers/Kusto/ParserKQLLimit.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <Parsers/IParserBase.h>
+#include <Parsers/Kusto/ParserKQLQuery.h>
+
+namespace DB
+{
+
+class ParserKQLLimit : public ParserKQLBase
+{
+protected:
+    const char * getName() const override { return "KQL limit"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
+
+}

src/Parsers/Kusto/ParserKQLOperators.cpp

@@ -0,0 +1,359 @@
+#include <Parsers/ASTLiteral.h>
+#include <Parsers/Kusto/ParserKQLQuery.h>
+#include <Parsers/Kusto/ParserKQLOperators.h>
+#include <Parsers/Kusto/ParserKQLStatement.h>
+#include <Parsers/CommonParsers.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int SYNTAX_ERROR;
+}
+
+String KQLOperators::genHasAnyAllOpExpr(std::vector<String> &tokens, IParser::Pos &token_pos,String kql_op, String ch_op)
+{
+    String new_expr;
+    Expected expected;
+    ParserToken s_lparen(TokenType::OpeningRoundBracket);
+
+    ++token_pos;
+    if (!s_lparen.ignore(token_pos, expected))
+        throw Exception("Syntax error near " + kql_op, ErrorCodes::SYNTAX_ERROR);
+
+    auto haystack = tokens.back();
+
+    String logic_op = (kql_op == "has_all") ? " and " : " or ";
+
+    while (!token_pos->isEnd() && token_pos->type != TokenType::PipeMark && token_pos->type != TokenType::Semicolon)
+    {
+        auto tmp_arg = String(token_pos->begin, token_pos->end);
+        if (token_pos->type == TokenType::Comma)
+            new_expr = new_expr + logic_op;
+        else
+            new_expr = new_expr + ch_op + "(" + haystack + "," + tmp_arg + ")";
+
+        ++token_pos;
+        if (token_pos->type == TokenType::ClosingRoundBracket)
+            break;
+
+    }
+
+    tokens.pop_back();
+    return new_expr;
+}
+
+String KQLOperators::genInOpExpr(IParser::Pos &token_pos, String kql_op, String ch_op)
+{
+    String new_expr;
+
+    ParserToken s_lparen(TokenType::OpeningRoundBracket);
+
+    ASTPtr select;
+    Expected expected;
+
+    ++token_pos;
+    if (!s_lparen.ignore(token_pos, expected))
+        throw Exception("Syntax error near " + kql_op, ErrorCodes::SYNTAX_ERROR);
+
+    --token_pos;
+    --token_pos;
+    return ch_op;
+
+}
+
+String KQLOperators::genHaystackOpExpr(std::vector<String> &tokens,IParser::Pos &token_pos,String kql_op, String ch_op, WildcardsPos wildcards_pos, WildcardsPos space_pos)
+{
+    String new_expr, left_wildcards, right_wildcards, left_space, right_space;
+
+    switch (wildcards_pos)
+    {
+        case WildcardsPos::none:
+            break;
+
+        case WildcardsPos::left:
+            left_wildcards ="%";
+            break;
+
+        case WildcardsPos::right:
+            right_wildcards = "%";
+            break;
+
+        case WildcardsPos::both:
+            left_wildcards ="%";
+            right_wildcards = "%";
+            break;
+    }
+
+    switch (space_pos)
+    {
+        case WildcardsPos::none:
+            break;
+
+        case WildcardsPos::left:
+            left_space =" ";
+            break;
+
+        case WildcardsPos::right:
+            right_space = " ";
+            break;
+
+        case WildcardsPos::both:
+            left_space =" ";
+            right_space = " ";
+            break;
+    }
+
+    ++token_pos;
+
+    if (!tokens.empty() && ((token_pos)->type == TokenType::StringLiteral || token_pos->type == TokenType::QuotedIdentifier))
+        new_expr = ch_op +"(" + tokens.back() +", '"+left_wildcards + left_space + String(token_pos->begin + 1,token_pos->end - 1) + right_space + right_wildcards + "')";
+    else if (!tokens.empty() && ((token_pos)->type == TokenType::BareWord))
+    {
+        auto tmp_arg = String(token_pos->begin, token_pos->end);
+        new_expr = ch_op +"(" + tokens.back() +", concat('" + left_wildcards + left_space + "', " + tmp_arg +", '"+ right_space + right_wildcards + "'))";
+    }
+    else
+        throw Exception("Syntax error near " + kql_op, ErrorCodes::SYNTAX_ERROR);
+    tokens.pop_back();
+    return new_expr;
+}
+
+bool KQLOperators::convert(std::vector<String> &tokens,IParser::Pos &pos)
+{
+    auto begin = pos;
+
+    if (!pos->isEnd() && pos->type != TokenType::PipeMark && pos->type != TokenType::Semicolon)
+    {
+        KQLOperatorValue op_value = KQLOperatorValue::none;
+
+        auto token =  String(pos->begin,pos->end);
+
+        String op = token;
+        if (token == "!")
+        {
+            ++pos;
+            if (pos->isEnd() || pos->type == TokenType::PipeMark || pos->type == TokenType::Semicolon)
+                throw Exception("Invalid negative operator", ErrorCodes::SYNTAX_ERROR);
+            op ="!"+String(pos->begin,pos->end);
+        }
+        else if (token == "matches")
+        {
+            ++pos;
+            if (!pos->isEnd() && pos->type != TokenType::PipeMark && pos->type != TokenType::Semicolon)
+            {
+                if (String(pos->begin,pos->end) == "regex")
+                    op +=" regex";
+                else
+                    --pos;
+            }
+        }
+        else
+        {
+            op = token;
+        }
+
+        ++pos;
+        if (!pos->isEnd() && pos->type != TokenType::PipeMark && pos->type != TokenType::Semicolon)
+        {
+            if (String(pos->begin,pos->end) == "~")
+                op +="~";
+            else
+                --pos;
+        }
+        else
+            --pos;
+
+        if (KQLOperator.find(op) == KQLOperator.end())
+        {
+            pos = begin;
+            return false;
+        }
+
+        op_value = KQLOperator[op];
+
+        String new_expr;
+
+        if (op_value == KQLOperatorValue::none)
+            tokens.push_back(op);
+        else
+        {
+            auto last_op = tokens.back();
+            auto last_pos = pos;
+
+            switch (op_value)
+            {
+            case KQLOperatorValue::contains:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "ilike", WildcardsPos::both);
+                break;
+
+            case KQLOperatorValue::not_contains:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "not ilike", WildcardsPos::both);
+                break;
+
+            case KQLOperatorValue::contains_cs:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "like", WildcardsPos::both);
+                break;
+
+            case KQLOperatorValue::not_contains_cs:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "not like", WildcardsPos::both);
+                break;
+
+            case KQLOperatorValue::endswith:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "ilike", WildcardsPos::left);
+                break;
+
+            case KQLOperatorValue::not_endswith:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "not ilike", WildcardsPos::left);
+                break;
+
+            case KQLOperatorValue::endswith_cs:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "endsWith", WildcardsPos::none);
+                break;
+
+            case KQLOperatorValue::not_endswith_cs:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "not endsWith", WildcardsPos::none);
+                break;
+
+            case KQLOperatorValue::equal:
+                break;
+
+            case KQLOperatorValue::not_equal:
+                break;
+
+            case KQLOperatorValue::equal_cs:
+                new_expr = "==";
+                break;
+
+            case KQLOperatorValue::not_equal_cs:
+                new_expr = "!=";
+                break;
+            case KQLOperatorValue::has:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "hasTokenCaseInsensitive", WildcardsPos::none);
+                break;
+
+            case KQLOperatorValue::not_has:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "not hasTokenCaseInsensitive", WildcardsPos::none);
+                break;
+
+            case KQLOperatorValue::has_all:
+                new_expr = genHasAnyAllOpExpr(tokens, pos, "has_all", "hasTokenCaseInsensitive");
+                break;
+
+            case KQLOperatorValue::has_any:
+                new_expr = genHasAnyAllOpExpr(tokens, pos, "has_any", "hasTokenCaseInsensitive");
+                break;
+
+            case KQLOperatorValue::has_cs:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "hasToken", WildcardsPos::none);
+                break;
+
+            case KQLOperatorValue::not_has_cs:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "not hasToken", WildcardsPos::none);
+                break;
+
+            case KQLOperatorValue::hasprefix:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "ilike", WildcardsPos::right);
+                new_expr += " or ";
+                tokens.push_back(last_op);
+                new_expr += genHaystackOpExpr(tokens, last_pos, op, "ilike", WildcardsPos::both, WildcardsPos::left);
+                break;
+
+            case KQLOperatorValue::not_hasprefix:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "not ilike", WildcardsPos::right);
+                new_expr += " and ";
+                tokens.push_back(last_op);
+                new_expr += genHaystackOpExpr(tokens, last_pos, op, "not ilike", WildcardsPos::both, WildcardsPos::left);
+                break;
+
+            case KQLOperatorValue::hasprefix_cs:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "startsWith", WildcardsPos::none);
+                new_expr += " or ";
+                tokens.push_back(last_op);
+                new_expr += genHaystackOpExpr(tokens, last_pos, op, "like", WildcardsPos::both, WildcardsPos::left);
+                break;
+
+            case KQLOperatorValue::not_hasprefix_cs:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "not startsWith", WildcardsPos::none);
+                new_expr += " and  ";
+                tokens.push_back(last_op);
+                new_expr += genHaystackOpExpr(tokens, last_pos, op, "not like", WildcardsPos::both, WildcardsPos::left);
+                break;
+
+            case KQLOperatorValue::hassuffix:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "ilike", WildcardsPos::left);
+                new_expr += " or ";
+                tokens.push_back(last_op);
+                new_expr += genHaystackOpExpr(tokens, last_pos, op, "ilike", WildcardsPos::both, WildcardsPos::right);
+                break;
+
+            case KQLOperatorValue::not_hassuffix:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "not ilike", WildcardsPos::left);
+                new_expr += " and ";
+                tokens.push_back(last_op);
+                new_expr += genHaystackOpExpr(tokens, last_pos, op, "not ilike", WildcardsPos::both, WildcardsPos::right);
+                break;
+
+            case KQLOperatorValue::hassuffix_cs:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "endsWith", WildcardsPos::none);
+                new_expr += " or ";
+                tokens.push_back(last_op);
+                new_expr += genHaystackOpExpr(tokens, last_pos, op, "like", WildcardsPos::both, WildcardsPos::right);
+                break;
+
+            case KQLOperatorValue::not_hassuffix_cs:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "not endsWith", WildcardsPos::none);
+                new_expr += " and  ";
+                tokens.push_back(last_op);
+                new_expr += genHaystackOpExpr(tokens, last_pos, op, "not like", WildcardsPos::both, WildcardsPos::right);
+                break;
+
+            case KQLOperatorValue::in_cs:
+                new_expr = genInOpExpr(pos,op,"in");
+                break;
+
+            case KQLOperatorValue::not_in_cs:
+                new_expr = genInOpExpr(pos,op,"not in");
+                break;
+
+            case KQLOperatorValue::in:
+                break;
+
+            case KQLOperatorValue::not_in:
+                break;
+
+            case KQLOperatorValue::matches_regex:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "match", WildcardsPos::none);
+                break;
+
+            case KQLOperatorValue::startswith:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "ilike", WildcardsPos::right);
+                break;
+
+            case KQLOperatorValue::not_startswith:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "not ilike", WildcardsPos::right);
+                break;
+
+            case KQLOperatorValue::startswith_cs:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "startsWith", WildcardsPos::none);
+                break;
+
+            case KQLOperatorValue::not_startswith_cs:
+                new_expr = genHaystackOpExpr(tokens, pos, op, "not startsWith", WildcardsPos::none);
+                break;
+
+            default:
+                break;
+            }
+
+            tokens.push_back(new_expr);
+        }
+        return true;
+    }
+    pos = begin;
+    return false;
+}
+
+}
+

src/Parsers/Kusto/ParserKQLOperators.h

@@ -0,0 +1,106 @@
+#pragma once
+
+#include <Parsers/IParserBase.h>
+#include <Parsers/Kusto/ParserKQLQuery.h>
+#include <unordered_map>
+namespace DB
+{
+
+class KQLOperators
+{
+public:
+    bool convert(std::vector<String> &tokens,IParser::Pos &pos);
+protected:
+
+    enum class WildcardsPos:uint8_t
+    {
+        none,
+        left,
+        right,
+        both
+    };
+
+    enum class KQLOperatorValue : uint16_t
+    {
+        none,
+        contains,
+        not_contains,
+        contains_cs,
+        not_contains_cs,
+        endswith,
+        not_endswith,
+        endswith_cs,
+        not_endswith_cs,
+        equal, //=~
+        not_equal,//!~
+        equal_cs, //=
+        not_equal_cs,//!=
+        has,
+        not_has,
+        has_all,
+        has_any,
+        has_cs,
+        not_has_cs,
+        hasprefix,
+        not_hasprefix,
+        hasprefix_cs,
+        not_hasprefix_cs,
+        hassuffix,
+        not_hassuffix,
+        hassuffix_cs,
+        not_hassuffix_cs,
+        in_cs,  //in
+        not_in_cs, //!in
+        in, //in~
+        not_in ,//!in~
+        matches_regex,
+        startswith,
+        not_startswith,
+        startswith_cs,
+        not_startswith_cs,
+    };
+
+    std::unordered_map <String,KQLOperatorValue> KQLOperator =
+    {
+        {"contains" , KQLOperatorValue::contains},
+        {"!contains" , KQLOperatorValue::not_contains},
+        {"contains_cs" , KQLOperatorValue::contains_cs},
+        {"!contains_cs" , KQLOperatorValue::not_contains_cs},
+        {"endswith" , KQLOperatorValue::endswith},
+        {"!endswith" , KQLOperatorValue::not_endswith},
+        {"endswith_cs" , KQLOperatorValue::endswith_cs},
+        {"!endswith_cs" , KQLOperatorValue::not_endswith_cs},
+        {"=~" , KQLOperatorValue::equal},
+        {"!~" , KQLOperatorValue::not_equal},
+        {"==" , KQLOperatorValue::equal_cs},
+        {"!=" , KQLOperatorValue::not_equal_cs},
+        {"has" , KQLOperatorValue::has},
+        {"!has" , KQLOperatorValue::not_has},
+        {"has_all" , KQLOperatorValue::has_all},
+        {"has_any" , KQLOperatorValue::has_any},
+        {"has_cs" , KQLOperatorValue::has_cs},
+        {"!has_cs" , KQLOperatorValue::not_has_cs},
+        {"hasprefix" , KQLOperatorValue::hasprefix},
+        {"!hasprefix" , KQLOperatorValue::not_hasprefix},
+        {"hasprefix_cs" , KQLOperatorValue::hasprefix_cs},
+        {"!hasprefix_cs" , KQLOperatorValue::not_hasprefix_cs},
+        {"hassuffix" , KQLOperatorValue::hassuffix},
+        {"!hassuffix" , KQLOperatorValue::not_hassuffix},
+        {"hassuffix_cs" , KQLOperatorValue::hassuffix_cs},
+        {"!hassuffix_cs" , KQLOperatorValue::not_hassuffix_cs},
+        {"in" , KQLOperatorValue::in_cs},
+        {"!in" , KQLOperatorValue::not_in_cs},
+        {"in~" , KQLOperatorValue::in},
+        {"!in~" , KQLOperatorValue::not_in},
+        {"matches regex" , KQLOperatorValue::matches_regex},
+        {"startswith" , KQLOperatorValue::startswith},
+        {"!startswith" , KQLOperatorValue::not_startswith},
+        {"startswith_cs" , KQLOperatorValue::startswith_cs},
+        {"!startswith_cs" , KQLOperatorValue::not_startswith_cs},
+    };
+    static String genHaystackOpExpr(std::vector<String> &tokens,IParser::Pos &token_pos,String kql_op, String ch_op, WildcardsPos wildcards_pos, WildcardsPos space_pos = WildcardsPos::none);
+    static String genInOpExpr(IParser::Pos &token_pos,String kql_op, String ch_op);
+    static String genHasAnyAllOpExpr(std::vector<String> &tokens,IParser::Pos &token_pos,String kql_op, String ch_op);
+};
+
+}

src/Parsers/Kusto/ParserKQLProject.cpp

@@ -0,0 +1,25 @@
+#include <Parsers/ExpressionListParsers.h>
+#include <Parsers/Kusto/ParserKQLQuery.h>
+#include <Parsers/Kusto/ParserKQLProject.h>
+namespace DB
+{
+
+bool ParserKQLProject :: parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    ASTPtr select_expression_list;
+    String expr;
+
+    expr = getExprFromToken(pos);
+
+    Tokens tokens(expr.c_str(), expr.c_str()+expr.size());
+    IParser::Pos new_pos(tokens, pos.max_depth);
+
+    if (!ParserNotEmptyExpressionList(true).parse(new_pos, select_expression_list, expected))
+        return false;
+
+    node->as<ASTSelectQuery>()->setExpression(ASTSelectQuery::Expression::SELECT, std::move(select_expression_list));
+
+    return true;
+}
+
+}

src/Parsers/Kusto/ParserKQLProject.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <Parsers/IParserBase.h>
+#include <Parsers/Kusto/ParserKQLQuery.h>
+
+namespace DB
+{
+
+class ParserKQLProject : public ParserKQLBase
+{
+protected:
+    const char * getName() const override { return "KQL project"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
+
+}

src/Parsers/Kusto/ParserKQLQuery.cpp

@@ -0,0 +1,342 @@
+#include <Parsers/ASTLiteral.h>
+#include <Parsers/ASTSelectQuery.h>
+#include <Parsers/IParserBase.h>
+#include <Parsers/Kusto/ParserKQLQuery.h>
+#include <Parsers/Kusto/ParserKQLTable.h>
+#include <Parsers/Kusto/ParserKQLProject.h>
+#include <Parsers/Kusto/ParserKQLFilter.h>
+#include <Parsers/Kusto/ParserKQLSort.h>
+#include <Parsers/Kusto/ParserKQLSummarize.h>
+#include <Parsers/Kusto/ParserKQLLimit.h>
+#include <Parsers/Kusto/ParserKQLStatement.h>
+#include <Parsers/CommonParsers.h>
+#include <format>
+#include <Parsers/ASTTablesInSelectQuery.h>
+#include <Parsers/ASTSubquery.h>
+#include <Parsers/Kusto/ParserKQLOperators.h>
+
+namespace DB
+{
+
+String ParserKQLBase :: getExprFromToken(const String & text, const uint32_t & max_depth)
+{
+    Tokens tokens(text.c_str(), text.c_str() + text.size());
+    IParser::Pos pos(tokens, max_depth);
+
+    return getExprFromToken(pos);
+}
+
+String ParserKQLBase :: getExprFromPipe(Pos & pos)
+{
+    uint16_t bracket_count = 0;
+    auto begin = pos;
+    auto end = pos;
+    while (!end->isEnd() && end->type != TokenType::Semicolon)
+    {
+        if (end->type == TokenType::OpeningRoundBracket)
+            ++bracket_count;
+
+        if (end->type == TokenType::OpeningRoundBracket)
+            --bracket_count;
+
+        if (end->type == TokenType::PipeMark && bracket_count == 0)
+            break;
+
+        ++end;
+    }
+    --end;
+    return String(begin->begin, end->end);
+}
+
+String ParserKQLBase :: getExprFromToken(Pos & pos)
+{
+    String res;
+    std::vector<String> tokens;
+    String alias;
+
+    while (!pos->isEnd() && pos->type != TokenType::PipeMark && pos->type != TokenType::Semicolon)
+    {
+        String token = String(pos->begin,pos->end);
+
+        if (token == "=")
+        {
+            ++pos;
+            if (String(pos->begin,pos->end) != "~")
+            {
+                alias = tokens.back();
+                tokens.pop_back();
+            }
+            --pos;
+        }
+        else if (!KQLOperators().convert(tokens,pos))
+        {
+            tokens.push_back(token);
+        }
+
+        if (pos->type == TokenType::Comma && !alias.empty())
+        {
+            tokens.pop_back();
+            tokens.push_back("AS");
+            tokens.push_back(alias);
+            tokens.push_back(",");
+            alias.clear();
+        }
+        ++pos;
+    }
+
+    if (!alias.empty())
+    {
+        tokens.push_back("AS");
+        tokens.push_back(alias);
+    }
+
+    for (auto const &token : tokens)
+        res = res.empty()? token : res +" " + token;
+    return res;
+}
+
+std::unique_ptr<IParserBase> ParserKQLQuery::getOperator(String & op_name)
+{
+    if (op_name == "filter" || op_name == "where")
+        return std::make_unique<ParserKQLFilter>();
+    else if (op_name == "limit" || op_name == "take")
+        return std::make_unique<ParserKQLLimit>();
+    else if (op_name == "project")
+        return std::make_unique<ParserKQLProject>();
+    else if (op_name == "sort by" || op_name == "order by")
+        return std::make_unique<ParserKQLSort>();
+    else if (op_name == "summarize")
+        return std::make_unique<ParserKQLSummarize>();
+    else if (op_name == "table")
+        return std::make_unique<ParserKQLTable>();
+    else
+        return nullptr;
+}
+
+bool ParserKQLQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    struct KQLOperatorDataFlowState
+    {
+        String operator_name;
+        bool need_input;
+        bool gen_output;
+        int8_t backspace_steps; // how many steps to last token of previous pipe
+    };
+
+    auto select_query = std::make_shared<ASTSelectQuery>();
+    node = select_query;
+    ASTPtr tables;
+
+    std::unordered_map<std::string, KQLOperatorDataFlowState> kql_parser =
+    {
+        { "filter", {"filter", false, false, 3}},
+        { "where", {"filter", false, false, 3}},
+        { "limit", {"limit", false, true, 3}},
+        { "take", {"limit", false, true, 3}},
+        { "project", {"project", false, false, 3}},
+        { "sort by", {"order by", false, false, 4}},
+        { "order by", {"order by", false, false, 4}},
+        { "table", {"table", false, false, 3}},
+        { "summarize", {"summarize", true, true, 3}}
+    };
+
+    std::vector<std::pair<String, Pos>> operation_pos;
+
+    String table_name(pos->begin, pos->end);
+
+    operation_pos.push_back(std::make_pair("table", pos));
+    ++pos;
+    uint16_t bracket_count = 0;
+
+    while (!pos->isEnd() && pos->type != TokenType::Semicolon)
+    {
+        if (pos->type == TokenType::OpeningRoundBracket)
+            ++bracket_count;
+        if (pos->type == TokenType::OpeningRoundBracket)
+            --bracket_count;
+
+        if (pos->type == TokenType::PipeMark && bracket_count == 0)
+        {
+            ++pos;
+            String kql_operator(pos->begin, pos->end);
+            if (kql_operator == "order" || kql_operator == "sort")
+            {
+                ++pos;
+                ParserKeyword s_by("by");
+                if (s_by.ignore(pos,expected))
+                {
+                    kql_operator = "order by";
+                    --pos;
+                }
+            }
+            if (pos->type != TokenType::BareWord || kql_parser.find(kql_operator) == kql_parser.end())
+                return false;
+            ++pos;
+            operation_pos.push_back(std::make_pair(kql_operator, pos));
+        }
+        else
+            ++pos;
+    }
+
+    auto kql_operator_str = operation_pos.back().first;
+    auto npos = operation_pos.back().second;
+    if (!npos.isValid())
+        return false;
+
+    auto kql_operator_p = getOperator(kql_operator_str);
+
+    if (!kql_operator_p)
+        return false;
+
+    if (operation_pos.size() == 1)
+    {
+        if (!kql_operator_p->parse(npos, node, expected))
+            return false;
+    }
+    else if (operation_pos.size() == 2 && operation_pos.front().first == "table")
+    {
+        if (!kql_operator_p->parse(npos, node, expected))
+            return false;
+        npos = operation_pos.front().second;
+        if (!ParserKQLTable().parse(npos, node, expected))
+            return false;
+    }
+    else
+    {
+        String project_clause, order_clause, where_clause, limit_clause;
+        auto last_pos = operation_pos.back().second;
+        auto last_op = operation_pos.back().first;
+
+        auto set_main_query_clause =[&](String & op, Pos & op_pos)
+        {
+            auto op_str = ParserKQLBase::getExprFromPipe(op_pos);
+            if (op == "project")
+                project_clause = op_str;
+            else if (op == "where" || op == "filter")
+                where_clause = where_clause.empty() ? std::format("({})", op_str) : where_clause + std::format("AND ({})", op_str);
+            else if (op == "limit" || op == "take")
+                limit_clause = op_str;
+            else if (op == "order by" || op == "sort by")
+                order_clause = order_clause.empty() ? op_str : order_clause + "," + op_str;
+        };
+
+        set_main_query_clause(last_op, last_pos);
+
+        operation_pos.pop_back();
+
+        if (kql_parser[last_op].need_input)
+        {
+            if (!kql_operator_p->parse(npos, node, expected))
+                return false;
+        }
+        else
+        {
+            while (!operation_pos.empty())
+            {
+                auto prev_op = operation_pos.back().first;
+                auto prev_pos = operation_pos.back().second;
+
+                if (kql_parser[prev_op].gen_output)
+                    break;
+                if (!project_clause.empty() && prev_op == "project")
+                    break;
+                set_main_query_clause(prev_op, prev_pos);
+                operation_pos.pop_back();
+                last_op = prev_op;
+                last_pos = prev_pos;
+            }
+        }
+
+        if (!operation_pos.empty())
+        {
+            for (auto i = 0; i< kql_parser[last_op].backspace_steps; ++i)
+                --last_pos;
+
+            String sub_query = std::format("({})", String(operation_pos.front().second->begin, last_pos->end));
+            Tokens token_subquery(sub_query.c_str(), sub_query.c_str() + sub_query.size());
+            IParser::Pos pos_subquery(token_subquery, pos.max_depth);
+
+            if (!ParserKQLSubquery().parse(pos_subquery, tables, expected))
+                return false;
+            select_query->setExpression(ASTSelectQuery::Expression::TABLES, std::move(tables));
+        }
+        else
+        {
+            if (!ParserKQLTable().parse(last_pos, node, expected))
+                return false;
+        }
+
+        auto set_query_clasue =[&](String op_str, String op_calsue)
+        {
+            auto oprator = getOperator(op_str);
+            if (oprator)
+            {
+                Tokens token_clause(op_calsue.c_str(), op_calsue.c_str() + op_calsue.size());
+                IParser::Pos pos_clause(token_clause, pos.max_depth);
+                if (!oprator->parse(pos_clause, node, expected))
+                    return false;
+            }
+            return true;
+        };
+
+        if (!select_query->select())
+        {
+            if (project_clause.empty())
+                project_clause = "*";
+            if (!set_query_clasue("project", project_clause))
+                return false;
+        }
+
+        if (!order_clause.empty())
+            if (!set_query_clasue("order by", order_clause))
+                return false;
+
+        if (!where_clause.empty())
+            if (!set_query_clasue("where", where_clause))
+                return false;
+
+        if (!limit_clause.empty())
+            if (!set_query_clasue("limit", limit_clause))
+                return false;
+        return true;
+    }
+
+    if (!select_query->select())
+    {
+        auto expr = String("*");
+        Tokens tokens(expr.c_str(), expr.c_str()+expr.size());
+        IParser::Pos new_pos(tokens, pos.max_depth);
+        if (!std::make_unique<ParserKQLProject>()->parse(new_pos, node, expected))
+            return false;
+    }
+
+     return true;
+}
+
+bool ParserKQLSubquery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    ASTPtr select_node;
+
+    if (!ParserKQLTaleFunction().parse(pos, select_node, expected))
+        return false;
+
+    ASTPtr node_subquery = std::make_shared<ASTSubquery>();
+    node_subquery->children.push_back(select_node);
+
+    ASTPtr node_table_expr = std::make_shared<ASTTableExpression>();
+    node_table_expr->as<ASTTableExpression>()->subquery = node_subquery;
+
+    node_table_expr->children.emplace_back(node_subquery);
+
+    ASTPtr node_table_in_select_query_emlement = std::make_shared<ASTTablesInSelectQueryElement>();
+    node_table_in_select_query_emlement->as<ASTTablesInSelectQueryElement>()->table_expression = node_table_expr;
+
+    ASTPtr res = std::make_shared<ASTTablesInSelectQuery>();
+
+    res->children.emplace_back(node_table_in_select_query_emlement);
+
+    node = res;
+    return true;
+}
+
+}

src/Parsers/Kusto/ParserKQLQuery.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <Parsers/IParserBase.h>
+#include <Parsers/ASTSelectQuery.h>
+
+namespace DB
+{
+class ParserKQLBase : public IParserBase
+{
+public:
+    static String getExprFromToken(Pos & pos);
+    static String getExprFromPipe(Pos & pos);
+    static String getExprFromToken(const String & text, const uint32_t & max_depth);
+};
+
+class ParserKQLQuery : public IParserBase
+{
+
+protected:
+    static std::unique_ptr<IParserBase> getOperator(String &op_name);
+    const char * getName() const override { return "KQL query"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
+
+class ParserKQLSubquery : public IParserBase
+{
+protected:
+    const char * getName() const override { return "KQL subquery"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
+
+}

src/Parsers/Kusto/ParserKQLSort.cpp

@@ -0,0 +1,60 @@
+#include <Parsers/ASTLiteral.h>
+#include <Parsers/IParserBase.h>
+#include <Parsers/ExpressionListParsers.h>
+#include <Parsers/ASTOrderByElement.h>
+#include <Parsers/Kusto/ParserKQLQuery.h>
+#include <Parsers/Kusto/ParserKQLSort.h>
+
+namespace DB
+{
+
+bool ParserKQLSort :: parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    bool has_dir = false;
+    std::vector <bool> has_directions;
+    ParserOrderByExpressionList order_list;
+    ASTPtr order_expression_list;
+
+    auto expr = getExprFromToken(pos);
+
+    Tokens tokens(expr.c_str(), expr.c_str() + expr.size());
+    IParser::Pos new_pos(tokens, pos.max_depth);
+
+    auto pos_backup = new_pos;
+    if (!order_list.parse(pos_backup, order_expression_list, expected))
+        return false;
+
+    while (!new_pos->isEnd() && new_pos->type != TokenType::PipeMark && new_pos->type != TokenType::Semicolon)
+    {
+        String tmp(new_pos->begin, new_pos->end);
+        if (tmp == "desc" or tmp == "asc")
+            has_dir = true;
+
+        if (new_pos->type == TokenType::Comma)
+        {
+            has_directions.push_back(has_dir);
+            has_dir = false;
+        }
+        ++new_pos;
+    }
+    has_directions.push_back(has_dir);
+
+    for (uint64_t i = 0; i < order_expression_list->children.size(); ++i)
+    {
+        if (!has_directions[i])
+        {
+            auto *order_expr =  order_expression_list->children[i]->as<ASTOrderByElement>();
+            order_expr->direction = -1; // default desc
+            if (!order_expr->nulls_direction_was_explicitly_specified)
+                order_expr->nulls_direction = -1;
+            else
+                order_expr->nulls_direction = order_expr->nulls_direction == 1 ? -1 : 1;
+        }
+    }
+
+    node->as<ASTSelectQuery>()->setExpression(ASTSelectQuery::Expression::ORDER_BY, std::move(order_expression_list));
+
+    return true;
+}
+
+}