Merge branch 'master' into clickhouse-local

2024-11-24 16:42:05 +00:00 · 2021-10-18 14:24:08 +03:00 · 2021-10-18 14:24:08 +03:00 · 003955b59f
commit 003955b59f
parent a28b048415 71b474793c
836 changed files with 9865 additions and 24837 deletions
--- a/.gitmodules
+++ b/.gitmodules
@ -213,6 +213,7 @@
 [submodule "contrib/boringssl"]
 	path = contrib/boringssl
 	url = https://github.com/ClickHouse-Extras/boringssl.git
+	branch = MergeWithUpstream
 [submodule "contrib/NuRaft"]
 	path = contrib/NuRaft
 	url = https://github.com/ClickHouse-Extras/NuRaft.git
@ -249,6 +250,9 @@
 [submodule "contrib/magic_enum"]
 	path = contrib/magic_enum
 	url = https://github.com/Neargye/magic_enum
+[submodule "contrib/libprotobuf-mutator"]
+	path = contrib/libprotobuf-mutator
+	url = https://github.com/google/libprotobuf-mutator
 [submodule "contrib/sysroot"]
 	path = contrib/sysroot
 	url = https://github.com/ClickHouse-Extras/sysroot.git
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -1,4 +1,4 @@
-### ClickHouse release v21.10, 2021-10-14
+### ClickHouse release v21.10, 2021-10-16

 #### Backward Incompatible Change

--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -136,6 +136,21 @@ if (ENABLE_FUZZING)
    message (STATUS "Fuzzing instrumentation enabled")
    set (FUZZER "libfuzzer")
    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostdlib++")
+    set (ENABLE_CLICKHOUSE_ODBC_BRIDGE OFF)
+    set (ENABLE_LIBRARIES 0)
+    set (ENABLE_SSL 1)
+    set (USE_INTERNAL_SSL_LIBRARY 1)
+    set (USE_UNWIND ON)
+    set (ENABLE_EMBEDDED_COMPILER 0)
+    set (ENABLE_EXAMPLES 0)
+    set (ENABLE_UTILS 0)
+    set (ENABLE_THINLTO 0)
+    set (ENABLE_TCMALLOC 0)
+    set (ENABLE_JEMALLOC 0)
+    set (ENABLE_CHECK_HEAVY_BUILDS 1)
+    set (GLIBC_COMPATIBILITY OFF)
+    set (ENABLE_PROTOBUF ON)
+    set (USE_INTERNAL_PROTOBUF_LIBRARY ON)
 endif()

 # Global libraries
@ -188,7 +203,7 @@ endif ()
 option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
 option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF)

-if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0")
+if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND NOT USE_MUSL)
    # Only for Linux, x86_64 or aarch64.
    option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON)
 elseif(GLIBC_COMPATIBILITY)
@ -203,10 +218,6 @@ if (GLIBC_COMPATIBILITY)
    set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -include ${CMAKE_CURRENT_SOURCE_DIR}/base/glibc-compatibility/glibc-compat-2.32.h")
 endif()

-if (NOT CMAKE_VERSION VERSION_GREATER "3.9.0")
-    message (WARNING "CMake version must be greater than 3.9.0 for production builds.")
-endif ()
-
 # Make sure the final executable has symbols exported
 set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic")

@ -582,6 +593,7 @@ include (cmake/find/cassandra.cmake)
 include (cmake/find/sentry.cmake)
 include (cmake/find/stats.cmake)
 include (cmake/find/datasketches.cmake)
+include (cmake/find/libprotobuf-mutator.cmake)

 set (USE_INTERNAL_CITYHASH_LIBRARY ON CACHE INTERNAL "")
 find_contrib_lib(cityhash)
--- a/base/base/LineReader.cpp
+++ b/base/base/LineReader.cpp
@ -5,6 +5,10 @@

 #include <string.h>
 #include <unistd.h>
+#include <sys/select.h>
+#include <sys/time.h>
+#include <sys/types.h>
+

 #ifdef OS_LINUX
 /// We can detect if code is linked with one or another readline variants or open the library dynamically.
--- a/base/base/phdr_cache.cpp
+++ b/base/base/phdr_cache.cpp
@ -6,7 +6,7 @@

 #include <base/defines.h>

-#if defined(__linux__) && !defined(THREAD_SANITIZER)
+#if defined(__linux__) && !defined(THREAD_SANITIZER) && !defined(USE_MUSL)
    #define USE_PHDR_CACHE 1
 #endif

--- a/base/daemon/SentryWriter.cpp
+++ b/base/daemon/SentryWriter.cpp
@ -13,6 +13,7 @@
 #include <Common/StackTrace.h>
 #include <Common/getNumberOfPhysicalCPUCores.h>
 #include <Core/ServerUUID.h>
+#include <Common/hex.h>

 #if !defined(ARCADIA_BUILD)
 #    include "Common/config_version.h"
@ -64,41 +65,6 @@ void setExtras()
        sentry_set_extra("disk_free_space", sentry_value_new_string(formatReadableSizeWithBinarySuffix(fs::space(server_data_path).free).c_str()));
 }

-void sentry_logger(sentry_level_e level, const char * message, va_list args, void *)
-{
-    auto * logger = &Poco::Logger::get("SentryWriter");
-    size_t size = 1024;
-    char buffer[size];
-#ifdef __clang__
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wformat-nonliteral"
-#endif
-    if (vsnprintf(buffer, size, message, args) >= 0)
-    {
-#ifdef __clang__
-#pragma clang diagnostic pop
-#endif
-        switch (level)
-        {
-            case SENTRY_LEVEL_DEBUG:
-                logger->debug(buffer);
-                break;
-            case SENTRY_LEVEL_INFO:
-                logger->information(buffer);
-                break;
-            case SENTRY_LEVEL_WARNING:
-                logger->warning(buffer);
-                break;
-            case SENTRY_LEVEL_ERROR:
-                logger->error(buffer);
-                break;
-            case SENTRY_LEVEL_FATAL:
-                logger->fatal(buffer);
-                break;
-        }
-    }
-}
-
 }


@ -107,13 +73,13 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config)
    bool enabled = false;
    bool debug = config.getBool("send_crash_reports.debug", false);
    auto * logger = &Poco::Logger::get("SentryWriter");
+
    if (config.getBool("send_crash_reports.enabled", false))
    {
        if (debug || (strlen(VERSION_OFFICIAL) > 0)) //-V560
-        {
            enabled = true;
-        }
    }
+
    if (enabled)
    {
        server_data_path = config.getString("path", "");
@ -126,7 +92,6 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config)

        sentry_options_t * options = sentry_options_new();  /// will be freed by sentry_init or sentry_shutdown
        sentry_options_set_release(options, VERSION_STRING_SHORT);
-        sentry_options_set_logger(options, &sentry_logger, nullptr);
        if (debug)
        {
            sentry_options_set_debug(options, 1);
@ -199,34 +164,34 @@ void SentryWriter::onFault(int sig, const std::string & error_message, const Sta
        if (stack_size > 0)
        {
            ssize_t offset = stack_trace.getOffset();
-            char instruction_addr[100];
+
+            char instruction_addr[19]
+            {
+                '0', 'x',
+                '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f',
+                '\0'
+            };
+
            StackTrace::Frames frames;
            StackTrace::symbolize(stack_trace.getFramePointers(), offset, stack_size, frames);
+
            for (ssize_t i = stack_size - 1; i >= offset; --i)
            {
                const StackTrace::Frame & current_frame = frames[i];
                sentry_value_t sentry_frame = sentry_value_new_object();
                UInt64 frame_ptr = reinterpret_cast<UInt64>(current_frame.virtual_addr);

-                if (std::snprintf(instruction_addr, sizeof(instruction_addr), "0x%" PRIx64, frame_ptr) >= 0)
-                {
-                    sentry_value_set_by_key(sentry_frame, "instruction_addr", sentry_value_new_string(instruction_addr));
-                }
+                writeHexUIntLowercase(frame_ptr, instruction_addr + 2);
+                sentry_value_set_by_key(sentry_frame, "instruction_addr", sentry_value_new_string(instruction_addr));

                if (current_frame.symbol.has_value())
-                {
                    sentry_value_set_by_key(sentry_frame, "function", sentry_value_new_string(current_frame.symbol.value().c_str()));
-                }

                if (current_frame.file.has_value())
-                {
                    sentry_value_set_by_key(sentry_frame, "filename", sentry_value_new_string(current_frame.file.value().c_str()));
-                }

                if (current_frame.line.has_value())
-                {
                    sentry_value_set_by_key(sentry_frame, "lineno", sentry_value_new_int32(current_frame.line.value()));
-                }

                sentry_value_append(sentry_frames, sentry_frame);
            }
--- a/base/loggers/Loggers.cpp
+++ b/base/loggers/Loggers.cpp
@ -84,7 +84,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log

        Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, log_file);
        log->setLevel(log_level);
-        split->addChannel(log);
+        split->addChannel(log, "log");
    }

    const auto errorlog_path = config.getString("logger.errorlog", "");
@ -116,7 +116,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
        Poco::AutoPtr<DB::OwnFormattingChannel> errorlog = new DB::OwnFormattingChannel(pf, error_log_file);
        errorlog->setLevel(errorlog_level);
        errorlog->open();
-        split->addChannel(errorlog);
+        split->addChannel(errorlog, "errorlog");
    }

    if (config.getBool("logger.use_syslog", false))
@ -155,7 +155,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
        Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, syslog_channel);
        log->setLevel(syslog_level);

-        split->addChannel(log);
+        split->addChannel(log, "syslog");
    }

    bool should_log_to_console = isatty(STDIN_FILENO) || isatty(STDERR_FILENO);
@ -177,7 +177,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
        Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, new Poco::ConsoleChannel);
        logger.warning("Logging " + console_log_level_string + " to console");
        log->setLevel(console_log_level);
-        split->addChannel(log);
+        split->addChannel(log, "console");
    }

    split->open();
@ -224,6 +224,89 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
    }
 }

+void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger)
+{
+    int max_log_level = 0;
+
+    const auto log_level_string = config.getString("logger.level", "trace");
+    int log_level = Poco::Logger::parseLevel(log_level_string);
+    if (log_level > max_log_level)
+        max_log_level = log_level;
+
+    const auto log_path = config.getString("logger.log", "");
+    if (!log_path.empty())
+        split->setLevel("log", log_level);
+    else
+        split->setLevel("log", 0);
+
+    // Set level to console
+    bool is_daemon = config.getBool("application.runAsDaemon", false);
+    bool should_log_to_console = isatty(STDIN_FILENO) || isatty(STDERR_FILENO);
+    if (config.getBool("logger.console", false)
+        || (!config.hasProperty("logger.console") && !is_daemon && should_log_to_console))
+        split->setLevel("console", log_level);
+    else
+        split->setLevel("console", 0);
+
+    // Set level to errorlog
+    int errorlog_level = 0;
+    const auto errorlog_path = config.getString("logger.errorlog", "");
+    if (!errorlog_path.empty())
+    {
+        errorlog_level = Poco::Logger::parseLevel(config.getString("logger.errorlog_level", "notice"));
+        if (errorlog_level > max_log_level)
+            max_log_level = errorlog_level;
+    }
+    split->setLevel("errorlog", errorlog_level);
+
+    // Set level to syslog
+    int syslog_level = 0;
+    if (config.getBool("logger.use_syslog", false))
+    {
+        syslog_level = Poco::Logger::parseLevel(config.getString("logger.syslog_level", log_level_string));
+        if (syslog_level > max_log_level)
+            max_log_level = syslog_level;
+    }
+    split->setLevel("syslog", syslog_level);
+
+    // Global logging level (it can be overridden for specific loggers).
+    logger.setLevel(max_log_level);
+
+    // Set level to all already created loggers
+    std::vector<std::string> names;
+
+    logger.root().names(names);
+    for (const auto & name : names)
+        logger.root().get(name).setLevel(max_log_level);
+
+    logger.root().setLevel(max_log_level);
+
+    // Explicitly specified log levels for specific loggers.
+    {
+        Poco::Util::AbstractConfiguration::Keys loggers_level;
+        config.keys("logger.levels", loggers_level);
+
+        if (!loggers_level.empty())
+        {
+            for (const auto & key : loggers_level)
+            {
+                if (key == "logger" || key.starts_with("logger["))
+                {
+                    const std::string name(config.getString("logger.levels." + key + ".name"));
+                    const std::string level(config.getString("logger.levels." + key + ".level"));
+                    logger.root().get(name).setLevel(level);
+                }
+                else
+                {
+                    // Legacy syntax
+                    const std::string level(config.getString("logger.levels." + key, "trace"));
+                    logger.root().get(key).setLevel(level);
+                }
+            }
+        }
+    }
+}
+
 void Loggers::closeLogs(Poco::Logger & logger)
 {
    if (log_file)
--- a/base/loggers/Loggers.h
+++ b/base/loggers/Loggers.h
@ -19,6 +19,8 @@ class Loggers
 public:
    void buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger, const std::string & cmd_name = "");

+    void updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger);
+
    /// Close log files. On next log write files will be reopened.
    void closeLogs(Poco::Logger & logger);

--- a/base/loggers/OwnFormattingChannel.h
+++ b/base/loggers/OwnFormattingChannel.h
@ -1,4 +1,5 @@
 #pragma once
+#include <atomic>
 #include <Poco/AutoPtr.h>
 #include <Poco/Channel.h>
 #include <Poco/FormattingChannel.h>
@ -14,7 +15,7 @@ class OwnFormattingChannel : public Poco::Channel, public ExtendedLogChannel
 public:
    explicit OwnFormattingChannel(
        Poco::AutoPtr<OwnPatternFormatter> pFormatter_ = nullptr, Poco::AutoPtr<Poco::Channel> pChannel_ = nullptr)
-        : pFormatter(std::move(pFormatter_)), pChannel(std::move(pChannel_))
+        : pFormatter(std::move(pFormatter_)), pChannel(std::move(pChannel_)), priority(Poco::Message::PRIO_TRACE)
    {
    }

@ -45,7 +46,7 @@ public:
 private:
    Poco::AutoPtr<OwnPatternFormatter> pFormatter;
    Poco::AutoPtr<Poco::Channel> pChannel;
-    Poco::Message::Priority priority = Poco::Message::PRIO_TRACE;
+    std::atomic<Poco::Message::Priority> priority;
 };

 }
--- a/base/loggers/OwnSplitChannel.cpp
+++ b/base/loggers/OwnSplitChannel.cpp
@ -1,4 +1,5 @@
 #include "OwnSplitChannel.h"
+#include "OwnFormattingChannel.h"

 #include <iostream>
 #include <Core/Block.h>
@ -75,7 +76,7 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg)
    ExtendedLogMessage msg_ext = ExtendedLogMessage::getFrom(msg);

    /// Log data to child channels
-    for (auto & channel : channels)
+    for (auto & [name, channel] : channels)
    {
        if (channel.second)
            channel.second->logExtended(msg_ext); // extended child
@ -100,7 +101,7 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg)
        columns[i++]->insert(msg.getSource());
        columns[i++]->insert(msg.getText());

-        logs_queue->emplace(std::move(columns));
+        [[maybe_unused]] bool push_result = logs_queue->emplace(std::move(columns));
    }

    /// Also log to system.text_log table, if message is not too noisy
@ -137,9 +138,9 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg)
 }


-void OwnSplitChannel::addChannel(Poco::AutoPtr<Poco::Channel> channel)
+void OwnSplitChannel::addChannel(Poco::AutoPtr<Poco::Channel> channel, const std::string & name)
 {
-    channels.emplace_back(std::move(channel), dynamic_cast<ExtendedLogChannel *>(channel.get()));
+    channels.emplace(name, ExtendedChannelPtrPair(std::move(channel), dynamic_cast<ExtendedLogChannel *>(channel.get())));
 }

 void OwnSplitChannel::addTextLog(std::shared_ptr<DB::TextLog> log, int max_priority)
@ -149,4 +150,14 @@ void OwnSplitChannel::addTextLog(std::shared_ptr<DB::TextLog> log, int max_prior
    text_log_max_priority.store(max_priority, std::memory_order_relaxed);
 }

+void OwnSplitChannel::setLevel(const std::string & name, int level)
+{
+     auto it = channels.find(name);
+     if (it != channels.end())
+     {
+         if (auto * channel = dynamic_cast<DB::OwnFormattingChannel *>(it->second.first.get()))
+            channel->setLevel(level);
+     }
+}
+
 }
--- a/base/loggers/OwnSplitChannel.h
+++ b/base/loggers/OwnSplitChannel.h
@ -18,10 +18,12 @@ public:
    /// Makes an extended message from msg and passes it to the client logs queue and child (if possible)
    void log(const Poco::Message & msg) override;
    /// Adds a child channel
-    void addChannel(Poco::AutoPtr<Poco::Channel> channel);
+    void addChannel(Poco::AutoPtr<Poco::Channel> channel, const std::string & name);

    void addTextLog(std::shared_ptr<DB::TextLog> log, int max_priority);

+    void setLevel(const std::string & name, int level);
+
 private:
    void logSplit(const Poco::Message & msg);
    void tryLogSplit(const Poco::Message & msg);
@ -29,7 +31,7 @@ private:
    using ChannelPtr = Poco::AutoPtr<Poco::Channel>;
    /// Handler and its pointer casted to extended interface
    using ExtendedChannelPtrPair = std::pair<ChannelPtr, ExtendedLogChannel *>;
-    std::vector<ExtendedChannelPtrPair> channels;
+    std::map<std::string, ExtendedChannelPtrPair> channels;

    std::mutex text_log_mutex;

--- a/benchmark/hardware.sh
+++ b/benchmark/hardware.sh
@ -13,6 +13,7 @@ TRIES=3

 AMD64_BIN_URL="https://builds.clickhouse.com/master/amd64/clickhouse"
 AARCH64_BIN_URL="https://builds.clickhouse.com/master/aarch64/clickhouse"
+POWERPC64_BIN_URL="https://builds.clickhouse.com/master/ppc64le/clickhouse"

 # Note: on older Ubuntu versions, 'axel' does not support IPv6. If you are using IPv6-only servers on very old Ubuntu, just don't install 'axel'.

@ -38,6 +39,8 @@ if [[ ! -f clickhouse ]]; then
        $FASTER_DOWNLOAD "$AMD64_BIN_URL"
    elif [[ $CPU == aarch64 ]]; then
        $FASTER_DOWNLOAD "$AARCH64_BIN_URL"
+    elif [[ $CPU == powerpc64le ]]; then
+        $FASTER_DOWNLOAD "$POWERPC64_BIN_URL"
    else
        echo "Unsupported CPU type: $CPU"
        exit 1
@ -52,7 +55,7 @@ fi

 if [[ ! -d data ]]; then
    if [[ ! -f $DATASET ]]; then
-        $FASTER_DOWNLOAD "https://clickhouse-datasets.s3.yandex.net/hits/partitions/$DATASET"
+        $FASTER_DOWNLOAD "https://datasets.clickhouse.com/hits/partitions/$DATASET"
    fi

    tar $TAR_PARAMS --strip-components=1 --directory=. -x -v -f $DATASET
--- a/cmake/find/amqpcpp.cmake
+++ b/cmake/find/amqpcpp.cmake
@ -10,7 +10,7 @@ if (NOT ENABLE_AMQPCPP)
 endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/CMakeLists.txt")
-    message (WARNING "submodule contrib/AMQP-CPP is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/AMQP-CPP is missing. to fix try run: \n git submodule update --init")
    message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal AMQP-CPP library")
    set (USE_AMQPCPP 0)
    return()
--- a/cmake/find/avro.cmake
+++ b/cmake/find/avro.cmake
@ -13,7 +13,7 @@ option (USE_INTERNAL_AVRO_LIBRARY

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/CMakeLists.txt")
    if (USE_INTERNAL_AVRO_LIBRARY)
-        message(WARNING "submodule contrib/avro is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/avro is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find internal avro")
        set(USE_INTERNAL_AVRO_LIBRARY 0)
    endif()
--- a/cmake/find/base64.cmake
+++ b/cmake/find/base64.cmake
@ -10,11 +10,11 @@ endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64/LICENSE")
    set (MISSING_INTERNAL_BASE64_LIBRARY 1)
-    message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init")
 endif ()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64")
-    message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init")
 else()
    set (BASE64_LIBRARY base64)
    set (USE_BASE64 1)
--- a/cmake/find/brotli.cmake
+++ b/cmake/find/brotli.cmake
@ -16,7 +16,7 @@ endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include/brotli/decode.h")
    if (USE_INTERNAL_BROTLI_LIBRARY)
-        message (WARNING "submodule contrib/brotli is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/brotli is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find internal brotli")
        set (USE_INTERNAL_BROTLI_LIBRARY 0)
    endif ()
--- a/cmake/find/bzip2.cmake
+++ b/cmake/find/bzip2.cmake
@ -6,7 +6,7 @@ if (NOT ENABLE_BZIP2)
 endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/bzip2/bzlib.h")
-    message (WARNING "submodule contrib/bzip2 is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/bzip2 is missing. to fix try run: \n git submodule update --init")
    message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal bzip2 library")
    set (USE_NLP 0)
    return()
--- a/cmake/find/capnp.cmake
+++ b/cmake/find/capnp.cmake
@ -11,7 +11,7 @@ option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/capnproto/CMakeLists.txt")
    if(USE_INTERNAL_CAPNP_LIBRARY)
-        message(WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init")
        message(${RECONFIGURE_MESSAGE_LEVEL} "cannot find internal capnproto")
        set(USE_INTERNAL_CAPNP_LIBRARY 0)
    endif()
--- a/cmake/find/cassandra.cmake
+++ b/cmake/find/cassandra.cmake
@ -14,7 +14,7 @@ if (APPLE)
 endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra")
-    message (ERROR "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init --recursive")
+    message (ERROR "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init")
    message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal Cassandra")
    set (USE_CASSANDRA 0)
    return()
--- a/cmake/find/cxx.cmake
+++ b/cmake/find/cxx.cmake
@ -17,7 +17,7 @@ option (USE_INTERNAL_LIBCXX_LIBRARY "Disable to use system libcxx and libcxxabi

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/CMakeLists.txt")
    if (USE_INTERNAL_LIBCXX_LIBRARY)
-        message(WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libcxx")
        set(USE_INTERNAL_LIBCXX_LIBRARY 0)
    endif()
--- a/cmake/find/cyrus-sasl.cmake
+++ b/cmake/find/cyrus-sasl.cmake
@ -6,7 +6,7 @@ endif()

 OPTION(ENABLE_CYRUS_SASL "Enable cyrus-sasl" ${DEFAULT_ENABLE_CYRUS_SASL})
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cyrus-sasl/README")
-    message (WARNING "submodule contrib/cyrus-sasl is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/cyrus-sasl is missing. to fix try run: \n git submodule update --init")
    set (ENABLE_CYRUS_SASL 0)
 endif ()

--- a/cmake/find/datasketches.cmake
+++ b/cmake/find/datasketches.cmake
@ -6,7 +6,7 @@ option (USE_INTERNAL_DATASKETCHES_LIBRARY "Set to FALSE to use system DataSketch

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/theta/CMakeLists.txt")
    if (USE_INTERNAL_DATASKETCHES_LIBRARY)
-       message(WARNING "submodule contrib/datasketches-cpp is missing. to fix try run: \n git submodule update --init --recursive")
+       message(WARNING "submodule contrib/datasketches-cpp is missing. to fix try run: \n git submodule update --init")
    endif()
    set(MISSING_INTERNAL_DATASKETCHES_LIBRARY 1)
    set(USE_INTERNAL_DATASKETCHES_LIBRARY 0)
--- a/cmake/find/fast_float.cmake
+++ b/cmake/find/fast_float.cmake
@ -1,5 +1,5 @@
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/fast_float/include/fast_float/fast_float.h")
-    message (FATAL_ERROR "submodule contrib/fast_float is missing. to fix try run: \n git submodule update --init --recursive")
+    message (FATAL_ERROR "submodule contrib/fast_float is missing. to fix try run: \n git submodule update --init")
 endif ()

 set(FAST_FLOAT_LIBRARY fast_float)
--- a/cmake/find/fastops.cmake
+++ b/cmake/find/fastops.cmake
@ -10,7 +10,7 @@ if(NOT ENABLE_FASTOPS)
 endif()

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/fastops/fastops/fastops.h")
-    message(WARNING "submodule contrib/fastops is missing. to fix try run: \n git submodule update --init --recursive")
+    message(WARNING "submodule contrib/fastops is missing. to fix try run: \n git submodule update --init")
    message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal fastops library")
    set(MISSING_INTERNAL_FASTOPS_LIBRARY 1)
 endif()
--- a/cmake/find/grpc.cmake
+++ b/cmake/find/grpc.cmake
@ -26,7 +26,7 @@ option(USE_INTERNAL_GRPC_LIBRARY "Set to FALSE to use system gRPC library instea

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/grpc/CMakeLists.txt")
  if(USE_INTERNAL_GRPC_LIBRARY)
-    message(WARNING "submodule contrib/grpc is missing. to fix try run: \n git submodule update --init --recursive")
+    message(WARNING "submodule contrib/grpc is missing. to fix try run: \n git submodule update --init")
    message(${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal grpc")
    set(USE_INTERNAL_GRPC_LIBRARY 0)
  endif()
--- a/cmake/find/gtest.cmake
+++ b/cmake/find/gtest.cmake
@ -4,7 +4,7 @@ option (USE_INTERNAL_GTEST_LIBRARY "Set to FALSE to use system Google Test inste

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest/CMakeLists.txt")
   if (USE_INTERNAL_GTEST_LIBRARY)
-       message (WARNING "submodule contrib/googletest is missing. to fix try run: \n git submodule update --init --recursive")
+       message (WARNING "submodule contrib/googletest is missing. to fix try run: \n git submodule update --init")
       message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal gtest")
       set (USE_INTERNAL_GTEST_LIBRARY 0)
   endif ()
--- a/cmake/find/h3.cmake
+++ b/cmake/find/h3.cmake
@ -11,7 +11,7 @@ option(USE_INTERNAL_H3_LIBRARY "Set to FALSE to use system h3 library instead of

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib/include/h3Index.h")
    if(USE_INTERNAL_H3_LIBRARY)
-        message(WARNING "submodule contrib/h3 is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/h3 is missing. to fix try run: \n git submodule update --init")
        message(${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal h3 library")
        set(USE_INTERNAL_H3_LIBRARY 0)
    endif()
--- a/cmake/find/hdfs3.cmake
+++ b/cmake/find/hdfs3.cmake
@ -16,7 +16,7 @@ option(USE_INTERNAL_HDFS3_LIBRARY "Set to FALSE to use system HDFS3 instead of b

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/hdfs.h")
    if(USE_INTERNAL_HDFS3_LIBRARY)
-        message(WARNING "submodule contrib/libhdfs3 is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/libhdfs3 is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal HDFS3 library")
        set(USE_INTERNAL_HDFS3_LIBRARY 0)
    endif()
--- a/cmake/find/icu.cmake
+++ b/cmake/find/icu.cmake
@ -16,7 +16,7 @@ option (USE_INTERNAL_ICU_LIBRARY "Set to FALSE to use system ICU library instead

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/icu/icu4c/LICENSE")
    if (USE_INTERNAL_ICU_LIBRARY)
-        message (WARNING "submodule contrib/icu is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/icu is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal ICU")
        set (USE_INTERNAL_ICU_LIBRARY 0)
    endif ()
--- a/cmake/find/krb5.cmake
+++ b/cmake/find/krb5.cmake
@ -1,7 +1,7 @@
 OPTION(ENABLE_KRB5 "Enable krb5" ${ENABLE_LIBRARIES})

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/krb5/README")
-    message (WARNING "submodule contrib/krb5 is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/krb5 is missing. to fix try run: \n git submodule update --init")
    set (ENABLE_KRB5 0)
 endif ()

--- a/cmake/find/ldap.cmake
+++ b/cmake/find/ldap.cmake
@ -15,7 +15,7 @@ option (USE_INTERNAL_LDAP_LIBRARY "Set to FALSE to use system *LDAP library inst

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/openldap/README")
    if (USE_INTERNAL_LDAP_LIBRARY)
-        message (WARNING "Submodule contrib/openldap is missing. To fix try running:\n git submodule update --init --recursive")
+        message (WARNING "Submodule contrib/openldap is missing. To fix try running:\n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal LDAP library")
    endif ()

--- a/cmake/find/libgsasl.cmake
+++ b/cmake/find/libgsasl.cmake
@ -16,7 +16,7 @@ endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src/gsasl.h")
    if (USE_INTERNAL_LIBGSASL_LIBRARY)
-        message (WARNING "submodule contrib/libgsasl is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/libgsasl is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libgsasl")
        set (USE_INTERNAL_LIBGSASL_LIBRARY 0)
    endif ()
--- a/cmake/find/libpqxx.cmake
+++ b/cmake/find/libpqxx.cmake
@ -5,14 +5,14 @@ if (NOT ENABLE_LIBPQXX)
 endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libpqxx/CMakeLists.txt")
-    message (WARNING "submodule contrib/libpqxx is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/libpqxx is missing. to fix try run: \n git submodule update --init")
    message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libpqxx library")
    set (USE_LIBPQXX 0)
    return()
 endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libpq/include")
-    message (ERROR "submodule contrib/libpq is missing. to fix try run: \n git submodule update --init --recursive")
+    message (ERROR "submodule contrib/libpq is missing. to fix try run: \n git submodule update --init")
    message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libpq needed for libpqxx")
    set (USE_LIBPQXX 0)
    return()
--- a/cmake/find/libprotobuf-mutator.cmake
+++ b/cmake/find/libprotobuf-mutator.cmake
@ -0,0 +1,11 @@
+option(USE_LIBPROTOBUF_MUTATOR "Enable libprotobuf-mutator" ${ENABLE_FUZZING})
+
+if (NOT USE_LIBPROTOBUF_MUTATOR)
+    return()
+endif()
+
+set(LibProtobufMutator_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libprotobuf-mutator")
+
+if (NOT EXISTS "${LibProtobufMutator_SOURCE_DIR}/README.md")
+    message (ERROR "submodule contrib/libprotobuf-mutator is missing. to fix try run: \n git submodule update --init")
+endif()
--- a/cmake/find/libuv.cmake
+++ b/cmake/find/libuv.cmake
@ -5,7 +5,7 @@ if (OS_DARWIN AND COMPILER_GCC)
 endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libuv")
-    message (WARNING "submodule contrib/libuv is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/libuv is missing. to fix try run: \n git submodule update --init")
    SET(MISSING_INTERNAL_LIBUV_LIBRARY 1)
    return()
 endif()
--- a/cmake/find/libxml2.cmake
+++ b/cmake/find/libxml2.cmake
@ -2,7 +2,7 @@ option (USE_INTERNAL_LIBXML2_LIBRARY "Set to FALSE to use system libxml2 library

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libxml2/libxml.h")
    if (USE_INTERNAL_LIBXML2_LIBRARY)
-        message (WARNING "submodule contrib/libxml2 is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/libxml2 is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libxml")
        set (USE_INTERNAL_LIBXML2_LIBRARY 0)
    endif ()
--- a/cmake/find/llvm.cmake
+++ b/cmake/find/llvm.cmake
@ -12,7 +12,7 @@ if (NOT ENABLE_EMBEDDED_COMPILER)
 endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/CMakeLists.txt")
-    message (${RECONFIGURE_MESSAGE_LEVEL} "submodule /contrib/llvm is missing. to fix try run: \n git submodule update --init --recursive")
+    message (${RECONFIGURE_MESSAGE_LEVEL} "submodule /contrib/llvm is missing. to fix try run: \n git submodule update --init")
 endif ()

 set (USE_EMBEDDED_COMPILER 1)
--- a/cmake/find/msgpack.cmake
+++ b/cmake/find/msgpack.cmake
@ -11,7 +11,7 @@ option (USE_INTERNAL_MSGPACK_LIBRARY "Set to FALSE to use system msgpack library

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/msgpack-c/include/msgpack.hpp")
    if(USE_INTERNAL_MSGPACK_LIBRARY)
-        message(WARNING "Submodule contrib/msgpack-c is missing. To fix try run: \n git submodule update --init --recursive")
+        message(WARNING "Submodule contrib/msgpack-c is missing. To fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal msgpack")
        set(USE_INTERNAL_MSGPACK_LIBRARY 0)
    endif()
--- a/cmake/find/mysqlclient.cmake
+++ b/cmake/find/mysqlclient.cmake
@ -16,7 +16,7 @@ option(USE_INTERNAL_MYSQL_LIBRARY "Set to FALSE to use system mysqlclient librar

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mariadb-connector-c/README")
    if(USE_INTERNAL_MYSQL_LIBRARY)
-        message(WARNING "submodule contrib/mariadb-connector-c is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/mariadb-connector-c is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal mysql library")
        set(USE_INTERNAL_MYSQL_LIBRARY 0)
    endif()
--- a/cmake/find/nanodbc.cmake
+++ b/cmake/find/nanodbc.cmake
@ -7,7 +7,7 @@ if (NOT USE_INTERNAL_NANODBC_LIBRARY)
 endif ()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/nanodbc/CMakeLists.txt")
-    message (FATAL_ERROR "submodule contrib/nanodbc is missing. to fix try run: \n git submodule update --init --recursive")
+    message (FATAL_ERROR "submodule contrib/nanodbc is missing. to fix try run: \n git submodule update --init")
 endif()

 set (NANODBC_LIBRARY nanodbc)
--- a/cmake/find/nlp.cmake
+++ b/cmake/find/nlp.cmake
@ -7,21 +7,21 @@ if (NOT ENABLE_NLP)
 endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libstemmer_c/Makefile")
-    message (WARNING "submodule contrib/libstemmer_c is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/libstemmer_c is missing. to fix try run: \n git submodule update --init")
    message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libstemmer_c library, NLP functions will be disabled")
    set (USE_NLP 0)
    return()
 endif ()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/wordnet-blast/CMakeLists.txt")
-    message (WARNING "submodule contrib/wordnet-blast is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/wordnet-blast is missing. to fix try run: \n git submodule update --init")
    message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal wordnet-blast library, NLP functions will be disabled")
    set (USE_NLP 0)
    return()
 endif ()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/lemmagen-c/README.md")
-    message (WARNING "submodule contrib/lemmagen-c is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/lemmagen-c is missing. to fix try run: \n git submodule update --init")
    message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal lemmagen-c library, NLP functions will be disabled")
    set (USE_NLP 0)
    return()
--- a/cmake/find/nuraft.cmake
+++ b/cmake/find/nuraft.cmake
@ -5,7 +5,7 @@ if (NOT ENABLE_NURAFT)
 endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/NuRaft/CMakeLists.txt")
-    message (WARNING "submodule contrib/NuRaft is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/NuRaft is missing. to fix try run: \n git submodule update --init")
    message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal NuRaft library")
    set (USE_NURAFT 0)
    return()
--- a/cmake/find/orc.cmake
+++ b/cmake/find/orc.cmake
@ -18,7 +18,7 @@ include(cmake/find/snappy.cmake)

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include/orc/OrcFile.hh")
   if(USE_INTERNAL_ORC_LIBRARY)
-       message(WARNING "submodule contrib/orc is missing. to fix try run: \n git submodule update --init --recursive")
+       message(WARNING "submodule contrib/orc is missing. to fix try run: \n git submodule update --init")
       message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal ORC")
       set(USE_INTERNAL_ORC_LIBRARY 0)
   endif()
--- a/cmake/find/parquet.cmake
+++ b/cmake/find/parquet.cmake
@ -20,7 +20,7 @@ endif()

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/CMakeLists.txt")
    if(USE_INTERNAL_PARQUET_LIBRARY)
-        message(WARNING "submodule contrib/arrow (required for Parquet) is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/arrow (required for Parquet) is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal parquet library")
        set(USE_INTERNAL_PARQUET_LIBRARY 0)
    endif()
--- a/cmake/find/protobuf.cmake
+++ b/cmake/find/protobuf.cmake
@ -15,7 +15,7 @@ option(USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instea

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/protobuf/cmake/CMakeLists.txt")
  if(USE_INTERNAL_PROTOBUF_LIBRARY)
-    message(WARNING "submodule contrib/protobuf is missing. to fix try run: \n git submodule update --init --recursive")
+    message(WARNING "submodule contrib/protobuf is missing. to fix try run: \n git submodule update --init")
    message(${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal protobuf")
    set(USE_INTERNAL_PROTOBUF_LIBRARY 0)
  endif()
--- a/cmake/find/rapidjson.cmake
+++ b/cmake/find/rapidjson.cmake
@ -10,7 +10,7 @@ option(USE_INTERNAL_RAPIDJSON_LIBRARY "Set to FALSE to use system rapidjson libr

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/rapidjson/include/rapidjson/rapidjson.h")
    if(USE_INTERNAL_RAPIDJSON_LIBRARY)
-       message(WARNING "submodule contrib/rapidjson is missing. to fix try run: \n git submodule update --init --recursive")
+       message(WARNING "submodule contrib/rapidjson is missing. to fix try run: \n git submodule update --init")
       message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal rapidjson library")
       set(USE_INTERNAL_RAPIDJSON_LIBRARY 0)
    endif()
--- a/cmake/find/rdkafka.cmake
+++ b/cmake/find/rdkafka.cmake
@ -11,7 +11,7 @@ option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka inst

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/CMakeLists.txt")
    if(USE_INTERNAL_RDKAFKA_LIBRARY)
-        message (WARNING "submodule contrib/cppkafka is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/cppkafka is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal cppkafka")
        set (USE_INTERNAL_RDKAFKA_LIBRARY 0)
    endif()
@ -20,7 +20,7 @@ endif ()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/CMakeLists.txt")
    if(USE_INTERNAL_RDKAFKA_LIBRARY OR MISSING_INTERNAL_CPPKAFKA_LIBRARY)
-        message (WARNING "submodule contrib/librdkafka is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/librdkafka is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal rdkafka")
        set (USE_INTERNAL_RDKAFKA_LIBRARY 0)
    endif()
--- a/cmake/find/re2.cmake
+++ b/cmake/find/re2.cmake
@ -2,7 +2,7 @@ option (USE_INTERNAL_RE2_LIBRARY "Set to FALSE to use system re2 library instead

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/re2/CMakeLists.txt")
    if(USE_INTERNAL_RE2_LIBRARY)
-        message(WARNING "submodule contrib/re2 is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/re2 is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal re2 library")
    endif()
    set(USE_INTERNAL_RE2_LIBRARY 0)
--- a/cmake/find/rocksdb.cmake
+++ b/cmake/find/rocksdb.cmake
@ -15,7 +15,7 @@ option(USE_INTERNAL_ROCKSDB_LIBRARY "Set to FALSE to use system ROCKSDB library

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/rocksdb/CMakeLists.txt")
    if (USE_INTERNAL_ROCKSDB_LIBRARY)
-        message (WARNING "submodule contrib is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib is missing. to fix try run: \n git submodule update --init")
        message(${RECONFIGURE_MESSAGE_LEVEL} "cannot find internal rocksdb")
    endif()
    set (MISSING_INTERNAL_ROCKSDB 1)
--- a/cmake/find/s2geometry.cmake
+++ b/cmake/find/s2geometry.cmake
@ -3,7 +3,7 @@ option(ENABLE_S2_GEOMETRY "Enable S2 geometry library" ${ENABLE_LIBRARIES})

 if (ENABLE_S2_GEOMETRY)
    if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/s2geometry")
-        message (WARNING "submodule contrib/s2geometry is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/s2geometry is missing. to fix try run: \n git submodule update --init")
        set (ENABLE_S2_GEOMETRY 0)
        set (USE_S2_GEOMETRY 0)
    else()
--- a/cmake/find/s3.cmake
+++ b/cmake/find/s3.cmake
@ -23,7 +23,7 @@ if (NOT USE_INTERNAL_AWS_S3_LIBRARY)
 endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3")
-    message (WARNING "submodule contrib/aws is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/aws is missing. to fix try run: \n git submodule update --init")
    if (USE_INTERNAL_AWS_S3_LIBRARY)
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal S3 library")
    endif ()
--- a/cmake/find/sentry.cmake
+++ b/cmake/find/sentry.cmake
@ -2,7 +2,7 @@ set (SENTRY_LIBRARY "sentry")

 set (SENTRY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/sentry-native/include")
 if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h")
-    message (WARNING "submodule contrib/sentry-native is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/sentry-native is missing. to fix try run: \n git submodule update --init")
    if (USE_SENTRY)
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal sentry library")
    endif()
--- a/cmake/find/simdjson.cmake
+++ b/cmake/find/simdjson.cmake
@ -1,7 +1,7 @@
 option (USE_SIMDJSON "Use simdjson" ${ENABLE_LIBRARIES})

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/simdjson/include/simdjson.h")
-    message (WARNING "submodule contrib/simdjson is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/simdjson is missing. to fix try run: \n git submodule update --init")
    if (USE_SIMDJSON)
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal simdjson library")
    endif()
--- a/cmake/find/sqlite.cmake
+++ b/cmake/find/sqlite.cmake
@ -5,7 +5,7 @@ if (NOT ENABLE_SQLITE)
 endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/sqlite-amalgamation/sqlite3.c")
-    message (WARNING "submodule contrib/sqlite3-amalgamation is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/sqlite3-amalgamation is missing. to fix try run: \n git submodule update --init")
    message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal sqlite library")
    set (USE_SQLITE 0)
    return()
--- a/cmake/find/ssl.cmake
+++ b/cmake/find/ssl.cmake
@ -13,7 +13,7 @@ option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boringssl/README.md")
    if(USE_INTERNAL_SSL_LIBRARY)
-        message(WARNING "submodule contrib/boringssl is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/boringssl is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal ssl library")
    endif()
    set(USE_INTERNAL_SSL_LIBRARY 0)
@ -53,12 +53,7 @@ endif ()
 if (NOT OPENSSL_FOUND AND NOT MISSING_INTERNAL_SSL_LIBRARY)
    set (USE_INTERNAL_SSL_LIBRARY 1)
    set (OPENSSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/boringssl")
-
-    if (ARCH_AMD64)
-        set (OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include")
-    elseif (ARCH_AARCH64)
-        set (OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include")
-    endif ()
+    set (OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include")
    set (OPENSSL_CRYPTO_LIBRARY crypto)
    set (OPENSSL_SSL_LIBRARY ssl)
    set (OPENSSL_FOUND 1)
--- a/cmake/find/stats.cmake
+++ b/cmake/find/stats.cmake
@ -2,11 +2,11 @@ option(ENABLE_STATS "Enable StatsLib library" ${ENABLE_LIBRARIES})

 if (ENABLE_STATS)
    if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/stats")
-        message (WARNING "submodule contrib/stats is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/stats is missing. to fix try run: \n git submodule update --init")
        set (ENABLE_STATS 0)
        set (USE_STATS 0)
    elseif (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/gcem")
-        message (WARNING "submodule contrib/gcem is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/gcem is missing. to fix try run: \n git submodule update --init")
        set (ENABLE_STATS 0)
        set (USE_STATS 0)
    else()
--- a/cmake/find/xz.cmake
+++ b/cmake/find/xz.cmake
@ -2,7 +2,7 @@ option (USE_INTERNAL_XZ_LIBRARY "Set to OFF to use system xz (lzma) library inst

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/xz/src/liblzma/api/lzma.h")
    if(USE_INTERNAL_XZ_LIBRARY)
-        message(WARNING "submodule contrib/xz is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/xz is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal xz (lzma) library")
        set(USE_INTERNAL_XZ_LIBRARY 0)
    endif()
--- a/cmake/find/yaml-cpp.cmake
+++ b/cmake/find/yaml-cpp.cmake
@ -5,5 +5,5 @@ if (NOT USE_YAML_CPP)
 endif()

 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/yaml-cpp/README.md")
-    message (ERROR "submodule contrib/yaml-cpp is missing. to fix try run: \n git submodule update --init --recursive")
+    message (ERROR "submodule contrib/yaml-cpp is missing. to fix try run: \n git submodule update --init")
 endif()
--- a/cmake/find/zlib.cmake
+++ b/cmake/find/zlib.cmake
@ -12,7 +12,7 @@ endif ()

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}/zlib.h")
    if(USE_INTERNAL_ZLIB_LIBRARY)
-        message(WARNING "submodule contrib/${INTERNAL_ZLIB_NAME} is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/${INTERNAL_ZLIB_NAME} is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal zlib library")
    endif()
    set(USE_INTERNAL_ZLIB_LIBRARY 0)
--- a/cmake/find/zstd.cmake
+++ b/cmake/find/zstd.cmake
@ -2,7 +2,7 @@ option (USE_INTERNAL_ZSTD_LIBRARY "Set to FALSE to use system zstd library inste

 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/zstd/lib/zstd.h")
    if(USE_INTERNAL_ZSTD_LIBRARY)
-        message(WARNING "submodule contrib/zstd is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/zstd is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal zstd library")
        set(USE_INTERNAL_ZSTD_LIBRARY 0)
    endif()
--- a/cmake/linux/default_libs.cmake
+++ b/cmake/linux/default_libs.cmake
@ -5,7 +5,7 @@ set (DEFAULT_LIBS "-nodefaultlibs")

 # We need builtins from Clang's RT even without libcxx - for ubsan+int128.
 # See https://bugs.llvm.org/show_bug.cgi?id=16404
-if (COMPILER_CLANG AND NOT (CMAKE_CROSSCOMPILING AND ARCH_AARCH64))
+if (COMPILER_CLANG AND NOT CMAKE_CROSSCOMPILING)
    execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
 else ()
    set (BUILTINS_LIBRARY "-lgcc")
@ -14,6 +14,8 @@ endif ()
 if (OS_ANDROID)
    # pthread and rt are included in libc
    set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -ldl")
+elseif (USE_MUSL)
+    set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -static -lc")
 else ()
    set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread -ldl")
 endif ()
@ -26,7 +28,7 @@ set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
 # glibc-compatibility library relies to constant version of libc headers
 # (because minor changes in function attributes between different glibc versions will introduce incompatibilities)
 # This is for x86_64. For other architectures we have separate toolchains.
-if (ARCH_AMD64 AND NOT_UNBUNDLED)
+if (ARCH_AMD64 AND NOT_UNBUNDLED AND NOT CMAKE_CROSSCOMPILING)
    set(CMAKE_C_STANDARD_INCLUDE_DIRECTORIES ${ClickHouse_SOURCE_DIR}/contrib/libc-headers/x86_64-linux-gnu ${ClickHouse_SOURCE_DIR}/contrib/libc-headers)
    set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES ${ClickHouse_SOURCE_DIR}/contrib/libc-headers/x86_64-linux-gnu ${ClickHouse_SOURCE_DIR}/contrib/libc-headers)
 endif ()
@ -37,8 +39,10 @@ set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)

 if (NOT OS_ANDROID)
-    # Our compatibility layer doesn't build under Android, many errors in musl.
-    add_subdirectory(base/glibc-compatibility)
+    if (NOT USE_MUSL)
+        # Our compatibility layer doesn't build under Android, many errors in musl.
+        add_subdirectory(base/glibc-compatibility)
+    endif ()
    add_subdirectory(base/harmful)
 endif ()

--- a/cmake/linux/toolchain-ppc64le.cmake
+++ b/cmake/linux/toolchain-ppc64le.cmake
@ -0,0 +1,32 @@
+set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+
+set (CMAKE_SYSTEM_NAME "Linux")
+set (CMAKE_SYSTEM_PROCESSOR "ppc64le")
+set (CMAKE_C_COMPILER_TARGET "ppc64le-linux-gnu")
+set (CMAKE_CXX_COMPILER_TARGET "ppc64le-linux-gnu")
+set (CMAKE_ASM_COMPILER_TARGET "ppc64le-linux-gnu")
+
+set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-powerpc64le")
+
+set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/powerpc64le-linux-gnu/libc")
+
+find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
+find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
+
+set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
+set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
+
+set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+
+set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
+
+set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
+set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
+
+set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
+
+set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
--- a/cmake/linux/toolchain-riscv64.cmake
+++ b/cmake/linux/toolchain-riscv64.cmake
@ -0,0 +1,32 @@
+set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+
+set (CMAKE_SYSTEM_NAME "Linux")
+set (CMAKE_SYSTEM_PROCESSOR "riscv64")
+set (CMAKE_C_COMPILER_TARGET "riscv64-linux-gnu")
+set (CMAKE_CXX_COMPILER_TARGET "riscv64-linux-gnu")
+set (CMAKE_ASM_COMPILER_TARGET "riscv64-linux-gnu")
+
+set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-riscv64")
+
+set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}")
+
+find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
+find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
+
+set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
+set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
+
+set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+
+set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
+
+set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
+set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
+
+set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
+
+set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
--- a/cmake/linux/toolchain-x86_64-musl.cmake
+++ b/cmake/linux/toolchain-x86_64-musl.cmake
@ -0,0 +1,35 @@
+set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+
+set (CMAKE_SYSTEM_NAME "Linux")
+set (CMAKE_SYSTEM_PROCESSOR "x86_64")
+set (CMAKE_C_COMPILER_TARGET "x86_64-linux-musl")
+set (CMAKE_CXX_COMPILER_TARGET "x86_64-linux-musl")
+set (CMAKE_ASM_COMPILER_TARGET "x86_64-linux-musl")
+
+set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-x86_64-musl")
+
+set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}")
+
+find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
+find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
+
+set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
+set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
+
+set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+
+set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
+
+set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
+set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
+
+set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
+
+set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
+
+set (USE_MUSL 1)
+add_definitions(-DUSE_MUSL=1)
--- a/cmake/target.cmake
+++ b/cmake/target.cmake
@ -20,40 +20,30 @@ endif ()
 if (CMAKE_CROSSCOMPILING)
    if (OS_DARWIN)
        # FIXME: broken dependencies
-        set (ENABLE_PROTOBUF OFF CACHE INTERNAL "")
        set (ENABLE_GRPC OFF CACHE INTERNAL "") # no protobuf -> no grpc
-
-        set (USE_SNAPPY OFF CACHE INTERNAL "")
-        set (ENABLE_PARQUET OFF CACHE INTERNAL "") # no snappy and protobuf -> no parquet
-        set (ENABLE_ORC OFF CACHE INTERNAL "") # no arrow (parquet) -> no orc
-
        set (ENABLE_ICU OFF CACHE INTERNAL "")
        set (ENABLE_FASTOPS OFF CACHE INTERNAL "")
    elseif (OS_LINUX OR OS_ANDROID)
        if (ARCH_AARCH64)
            # FIXME: broken dependencies
-            set (ENABLE_PROTOBUF OFF CACHE INTERNAL "")
            set (ENABLE_GRPC OFF CACHE INTERNAL "")
-
-            set (ENABLE_PARQUET OFF CACHE INTERNAL "")
-            set (ENABLE_ORC OFF CACHE INTERNAL "")
-
-            set (ENABLE_MYSQL OFF CACHE INTERNAL "")
+            set (USE_SENTRY OFF CACHE INTERNAL "")
+        elseif (ARCH_PPC64LE)
+            set (ENABLE_GRPC OFF CACHE INTERNAL "")
+            set (USE_SENTRY OFF CACHE INTERNAL "")
        endif ()
    elseif (OS_FREEBSD)
        # FIXME: broken dependencies
-        set (ENABLE_PROTOBUF OFF CACHE INTERNAL "")
+        set (ENABLE_PARQUET OFF CACHE INTERNAL "")
+        set (ENABLE_ORC OFF CACHE INTERNAL "")
        set (ENABLE_GRPC OFF CACHE INTERNAL "")
-
-        set (ENABLE_ORC OFF CACHE INTERNAL "")  # no protobuf -> no parquet -> no orc
-
        set (ENABLE_EMBEDDED_COMPILER OFF CACHE INTERNAL "")
    else ()
        message (FATAL_ERROR "Trying to cross-compile to unsupported system: ${CMAKE_SYSTEM_NAME}!")
    endif ()

    # Don't know why but CXX_STANDARD doesn't work for cross-compilation
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20")

    message (STATUS "Cross-compiling for target: ${CMAKE_CXX_COMPILE_TARGET}")
 endif ()
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@ -49,6 +49,10 @@ add_subdirectory (replxx-cmake)
 add_subdirectory (unixodbc-cmake)
 add_subdirectory (nanodbc-cmake)

+if (ENABLE_FUZZING)
+    add_subdirectory (libprotobuf-mutator-cmake)
+endif()
+
 if (USE_YAML_CPP)
    add_subdirectory (yaml-cpp-cmake)
 endif()
@ -278,7 +282,7 @@ if (USE_FASTOPS)
 endif()

 if (USE_AMQPCPP OR USE_CASSANDRA)
-    add_subdirectory (libuv)
+    add_subdirectory (libuv-cmake)
 endif()
 if (USE_AMQPCPP)
    add_subdirectory (amqpcpp-cmake)
--- a/contrib/boost
+++ b/contrib/boost
@ -1 +1 @@
-Subproject commit 66d17f060c4867aeea99fa2a20cfdae89ae2a2ec
+Subproject commit 79358a3106aab6af464430ed67c7efafebf5cd6f
--- a/contrib/boringssl
+++ b/contrib/boringssl
@ -1 +1 @@
-Subproject commit a6a2e2ab3e44d97ce98e51c558e989f211de7eb3
+Subproject commit c1e01a441d6db234f4f12e63a7657d1f9e6db9c1
--- a/contrib/boringssl-cmake/CMakeLists.txt
+++ b/contrib/boringssl-cmake/CMakeLists.txt
@ -4,7 +4,7 @@

 # This file is created by generate_build_files.py and edited accordingly.

-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.5)

 project(BoringSSL LANGUAGES C CXX)

@ -20,12 +20,7 @@ if(CMAKE_COMPILER_IS_GNUCXX OR CLANG)
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
  endif()

-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-common")
-  if((CMAKE_C_COMPILER_VERSION VERSION_GREATER "4.8.99") OR CLANG)
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11")
-  else()
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
-  endif()
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-common -std=c11")
 endif()

 # pthread_rwlock_t requires a feature flag.
@ -55,7 +50,7 @@ add_definitions(-DBORINGSSL_IMPLEMENTATION)
 # builds.
 if(NOT OPENSSL_NO_ASM AND CMAKE_OSX_ARCHITECTURES)
  list(LENGTH CMAKE_OSX_ARCHITECTURES NUM_ARCHES)
-  if(NOT ${NUM_ARCHES} EQUAL 1)
+  if(NOT NUM_ARCHES EQUAL 1)
    message(FATAL_ERROR "Universal binaries not supported.")
  endif()
  list(GET CMAKE_OSX_ARCHITECTURES 0 CMAKE_SYSTEM_PROCESSOR)
@ -78,7 +73,13 @@ elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "AMD64")
 elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86")
  set(ARCH "x86")
 elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i386")
-  set(ARCH "x86")
+  # cmake uses `uname -p` to set the system processor, but Solaris
+  # systems support multiple architectures.
+  if((${CMAKE_SYSTEM_NAME} STREQUAL "SunOS") AND CMAKE_SIZEOF_VOID_P EQUAL 8)
+    set(ARCH "x86_64")
+  else()
+    set(ARCH "x86")
+  endif()
 elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i686")
  set(ARCH "x86")
 elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64")
@ -289,6 +290,21 @@ set(
  mac-x86_64/crypto/test/trampoline-x86_64.S
 )

+set(
+  CRYPTO_win_aarch64_SOURCES
+
+  win-aarch64/crypto/chacha/chacha-armv8.S
+  win-aarch64/crypto/fipsmodule/aesv8-armx64.S
+  win-aarch64/crypto/fipsmodule/armv8-mont.S
+  win-aarch64/crypto/fipsmodule/ghash-neon-armv8.S
+  win-aarch64/crypto/fipsmodule/ghashv8-armx64.S
+  win-aarch64/crypto/fipsmodule/sha1-armv8.S
+  win-aarch64/crypto/fipsmodule/sha256-armv8.S
+  win-aarch64/crypto/fipsmodule/sha512-armv8.S
+  win-aarch64/crypto/fipsmodule/vpaes-armv8.S
+  win-aarch64/crypto/test/trampoline-armv8.S
+)
+
 set(
  CRYPTO_win_x86_SOURCES

@ -331,9 +347,9 @@ set(
  win-x86_64/crypto/test/trampoline-x86_64.asm
 )

-if(APPLE AND ${ARCH} STREQUAL "aarch64")
+if(APPLE AND ARCH STREQUAL "aarch64")
  set(CRYPTO_ARCH_SOURCES ${CRYPTO_ios_aarch64_SOURCES})
-elseif(APPLE AND ${ARCH} STREQUAL "arm")
+elseif(APPLE AND ARCH STREQUAL "arm")
  set(CRYPTO_ARCH_SOURCES ${CRYPTO_ios_arm_SOURCES})
 elseif(APPLE)
  set(CRYPTO_ARCH_SOURCES ${CRYPTO_mac_${ARCH}_SOURCES})
@ -360,6 +376,7 @@ add_library(
  "${BORINGSSL_SOURCE_DIR}/crypto/asn1/a_object.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/asn1/a_octet.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/asn1/a_print.c"
+  "${BORINGSSL_SOURCE_DIR}/crypto/asn1/a_strex.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/asn1/a_strnid.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/asn1/a_time.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/asn1/a_type.c"
@ -389,6 +406,7 @@ add_library(
  "${BORINGSSL_SOURCE_DIR}/crypto/bio/printf.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/bio/socket.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/bio/socket_helper.c"
+  "${BORINGSSL_SOURCE_DIR}/crypto/blake2/blake2.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/bn_extra/bn_asn1.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/bn_extra/convert.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/buf/buf.c"
@ -413,6 +431,7 @@ add_library(
  "${BORINGSSL_SOURCE_DIR}/crypto/conf/conf.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/cpu-aarch64-fuchsia.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/cpu-aarch64-linux.c"
+  "${BORINGSSL_SOURCE_DIR}/crypto/cpu-aarch64-win.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/cpu-arm-linux.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/cpu-arm.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/cpu-intel.c"
@ -452,7 +471,6 @@ add_library(
  "${BORINGSSL_SOURCE_DIR}/crypto/ex_data.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/fipsmodule/bcm.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/fipsmodule/fips_shared_support.c"
-  "${BORINGSSL_SOURCE_DIR}/crypto/fipsmodule/is_fips.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/hkdf/hkdf.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/hpke/hpke.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/hrss/hrss.c"
@ -499,13 +517,13 @@ add_library(
  "${BORINGSSL_SOURCE_DIR}/crypto/trust_token/voprf.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/a_digest.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/a_sign.c"
-  "${BORINGSSL_SOURCE_DIR}/crypto/x509/a_strex.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/a_verify.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/algorithm.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/asn1_gen.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/by_dir.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/by_file.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/i2d_pr.c"
+  "${BORINGSSL_SOURCE_DIR}/crypto/x509/name_print.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/rsa_pss.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/t_crl.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/t_req.c"
@ -519,7 +537,6 @@ add_library(
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/x509_ext.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/x509_lu.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/x509_obj.c"
-  "${BORINGSSL_SOURCE_DIR}/crypto/x509/x509_r2x.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/x509_req.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/x509_set.c"
  "${BORINGSSL_SOURCE_DIR}/crypto/x509/x509_trs.c"
@ -589,6 +606,8 @@ add_library(
  "${BORINGSSL_SOURCE_DIR}/ssl/d1_srtp.cc"
  "${BORINGSSL_SOURCE_DIR}/ssl/dtls_method.cc"
  "${BORINGSSL_SOURCE_DIR}/ssl/dtls_record.cc"
+  "${BORINGSSL_SOURCE_DIR}/ssl/encrypted_client_hello.cc"
+  "${BORINGSSL_SOURCE_DIR}/ssl/extensions.cc"
  "${BORINGSSL_SOURCE_DIR}/ssl/handoff.cc"
  "${BORINGSSL_SOURCE_DIR}/ssl/handshake.cc"
  "${BORINGSSL_SOURCE_DIR}/ssl/handshake_client.cc"
@ -611,7 +630,6 @@ add_library(
  "${BORINGSSL_SOURCE_DIR}/ssl/ssl_versions.cc"
  "${BORINGSSL_SOURCE_DIR}/ssl/ssl_x509.cc"
  "${BORINGSSL_SOURCE_DIR}/ssl/t1_enc.cc"
-  "${BORINGSSL_SOURCE_DIR}/ssl/t1_lib.cc"
  "${BORINGSSL_SOURCE_DIR}/ssl/tls13_both.cc"
  "${BORINGSSL_SOURCE_DIR}/ssl/tls13_client.cc"
  "${BORINGSSL_SOURCE_DIR}/ssl/tls13_enc.cc"
@ -633,6 +651,7 @@ add_executable(
  "${BORINGSSL_SOURCE_DIR}/tool/digest.cc"
  "${BORINGSSL_SOURCE_DIR}/tool/fd.cc"
  "${BORINGSSL_SOURCE_DIR}/tool/file.cc"
+  "${BORINGSSL_SOURCE_DIR}/tool/generate_ech.cc"
  "${BORINGSSL_SOURCE_DIR}/tool/generate_ed25519.cc"
  "${BORINGSSL_SOURCE_DIR}/tool/genrsa.cc"
  "${BORINGSSL_SOURCE_DIR}/tool/pkcs12.cc"
--- a/contrib/fastops
+++ b/contrib/fastops
@ -1 +1 @@
-Subproject commit 012b777df9e2d145a24800a6c8c3d4a0249bb09e
+Subproject commit 1460583af7d13c0e980ce46aec8ee9400314669a
--- a/contrib/grpc-cmake/protobuf_generate_grpc.cmake
+++ b/contrib/grpc-cmake/protobuf_generate_grpc.cmake
@ -187,12 +187,12 @@ function(protobuf_generate_grpc)

    add_custom_command(
      OUTPUT ${_generated_srcs}
-      COMMAND $<TARGET_FILE:protobuf::protoc>
+      COMMAND $<TARGET_FILE:protoc>
      ARGS --${protobuf_generate_grpc_LANGUAGE}_out ${_dll_export_decl}${protobuf_generate_grpc_PROTOC_OUT_DIR}
           --grpc_out ${_dll_export_decl}${protobuf_generate_grpc_PROTOC_OUT_DIR}
           --plugin=protoc-gen-grpc=$<TARGET_FILE:${protobuf_generate_grpc_PLUGIN}>
           ${_dll_desc_out} ${_protobuf_include_path} ${_abs_file}
-      DEPENDS ${_abs_file} protobuf::protoc ${protobuf_generate_grpc_PLUGIN}
+      DEPENDS ${_abs_file} protoc ${protobuf_generate_grpc_PLUGIN}
      COMMENT "Running ${protobuf_generate_grpc_LANGUAGE} protocol buffer compiler on ${_proto}"
      VERBATIM)
  endforeach()
--- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h
+++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h
@ -18,8 +18,10 @@
 * Define overrides for non-standard allocator-related functions if they are
 * present on the system.
 */
-#define JEMALLOC_OVERRIDE_MEMALIGN
-#define JEMALLOC_OVERRIDE_VALLOC
+#if !defined(USE_MUSL)
+    #define JEMALLOC_OVERRIDE_MEMALIGN
+    #define JEMALLOC_OVERRIDE_VALLOC
+#endif

 /*
 * At least Linux omits the "const" in:
--- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h
+++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h
@ -1,6 +1,6 @@
 // OSX does not have this for system alloc functions, so you will get
 // "exception specification in declaration" error.
-#if defined(__APPLE__) || defined(__FreeBSD__)
+#if defined(__APPLE__) || defined(__FreeBSD__) || defined(USE_MUSL)
 # undef JEMALLOC_NOTHROW
 # define JEMALLOC_NOTHROW

--- a/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in
+++ b/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in
@ -81,7 +81,7 @@
 /* #undef JEMALLOC_HAVE_ISSETUGID */

 /* Defined if pthread_atfork(3) is available. */
-#define JEMALLOC_HAVE_PTHREAD_ATFORK
+/* #undef JEMALLOC_HAVE_PTHREAD_ATFORK */

 /* Defined if pthread_setname_np(3) is available. */
 #define JEMALLOC_HAVE_PTHREAD_SETNAME_NP
@ -284,7 +284,7 @@
 #define JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS

 /* Defined if madvise(2) is available but MADV_FREE is not (x86 Linux only). */
-/* #undef JEMALLOC_DEFINE_MADVISE_FREE */
+#define JEMALLOC_DEFINE_MADVISE_FREE

 /*
 * Defined if MADV_DO[NT]DUMP is supported as an argument to madvise.
--- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
+++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
@ -13,12 +13,14 @@
 * Define overrides for non-standard allocator-related functions if they are
 * present on the system.
 */
-#define JEMALLOC_OVERRIDE___LIBC_CALLOC
-#define JEMALLOC_OVERRIDE___LIBC_FREE
-#define JEMALLOC_OVERRIDE___LIBC_MALLOC
-#define JEMALLOC_OVERRIDE___LIBC_MEMALIGN
-#define JEMALLOC_OVERRIDE___LIBC_REALLOC
-#define JEMALLOC_OVERRIDE___LIBC_VALLOC
+#if !defined(USE_MUSL)
+    #define JEMALLOC_OVERRIDE___LIBC_CALLOC
+    #define JEMALLOC_OVERRIDE___LIBC_FREE
+    #define JEMALLOC_OVERRIDE___LIBC_MALLOC
+    #define JEMALLOC_OVERRIDE___LIBC_MEMALIGN
+    #define JEMALLOC_OVERRIDE___LIBC_REALLOC
+    #define JEMALLOC_OVERRIDE___LIBC_VALLOC
+#endif
 /* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */

 /*
--- a/contrib/libcxx-cmake/CMakeLists.txt
+++ b/contrib/libcxx-cmake/CMakeLists.txt
@ -56,6 +56,10 @@ if (USE_UNWIND)
    target_compile_definitions(cxx PUBLIC -DSTD_EXCEPTION_HAS_STACK_TRACE=1)
 endif ()

+if (USE_MUSL)
+    target_compile_definitions(cxx PUBLIC -D_LIBCPP_HAS_MUSL_LIBC=1)
+endif ()
+
 # Override the deduced attribute support that causes error.
 if (OS_DARWIN AND COMPILER_GCC)
    add_compile_definitions(_LIBCPP_INIT_PRIORITY_MAX)
--- a/contrib/libprotobuf-mutator
+++ b/contrib/libprotobuf-mutator
@ -0,0 +1 @@
+Subproject commit ffd86a32874e5c08a143019aad1aaf0907294c9f
--- a/contrib/libprotobuf-mutator-cmake/CMakeLists.txt
+++ b/contrib/libprotobuf-mutator-cmake/CMakeLists.txt
@ -0,0 +1,14 @@
+set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/libprotobuf-mutator)
+
+add_library(protobuf-mutator
+    ${LIBRARY_DIR}/src/libfuzzer/libfuzzer_macro.cc
+    ${LIBRARY_DIR}/src/libfuzzer/libfuzzer_mutator.cc
+    ${LIBRARY_DIR}/src/binary_format.cc
+    ${LIBRARY_DIR}/src/mutator.cc
+    ${LIBRARY_DIR}/src/text_format.cc
+    ${LIBRARY_DIR}/src/utf8_fix.cc)
+
+target_include_directories(protobuf-mutator BEFORE PRIVATE "${LIBRARY_DIR}")
+target_include_directories(protobuf-mutator BEFORE PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/protobuf/src")
+
+target_link_libraries(protobuf-mutator ${Protobuf_LIBRARY})
--- a/contrib/libuv
+++ b/contrib/libuv
@ -1 +1 @@
-Subproject commit e2e9b7e9f978ce8a1367b5fe781d97d1ce9f94ab
+Subproject commit 95081e7c16c9857babe6d4e2bc1c779198ea89ae
--- a/contrib/libuv-cmake/CMakeLists.txt
+++ b/contrib/libuv-cmake/CMakeLists.txt
@ -0,0 +1,160 @@
+# This file is a modified version of contrib/libuv/CMakeLists.txt
+
+include(CMakeDependentOption)
+
+set (SOURCE_DIR "${CMAKE_SOURCE_DIR}/contrib/libuv")
+set (BINARY_DIR "${CMAKE_BINARY_DIR}/contrib/libuv")
+
+
+if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU")
+  list(APPEND uv_cflags -fvisibility=hidden --std=gnu89)
+  list(APPEND uv_cflags -Wall -Wextra -Wstrict-prototypes)
+  list(APPEND uv_cflags -Wno-unused-parameter)
+endif()
+
+set(uv_sources
+    src/fs-poll.c
+    src/idna.c
+    src/inet.c
+    src/random.c
+    src/strscpy.c
+    src/threadpool.c
+    src/timer.c
+    src/uv-common.c
+    src/uv-data-getter-setters.c
+    src/version.c
+    src/unix/async.c
+    src/unix/core.c
+    src/unix/dl.c
+    src/unix/fs.c
+    src/unix/getaddrinfo.c
+    src/unix/getnameinfo.c
+    src/unix/loop-watcher.c
+    src/unix/loop.c
+    src/unix/pipe.c
+    src/unix/poll.c
+    src/unix/process.c
+    src/unix/random-devurandom.c
+    src/unix/signal.c
+    src/unix/stream.c
+    src/unix/tcp.c
+    src/unix/thread.c
+    src/unix/tty.c
+    src/unix/udp.c)
+
+if(APPLE OR CMAKE_SYSTEM_NAME MATCHES "Android|Linux|OS/390")
+  list(APPEND uv_sources src/unix/proctitle.c)
+endif()
+
+if(CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD")
+  list(APPEND uv_sources src/unix/freebsd.c)
+endif()
+
+if(CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD|NetBSD|OpenBSD")
+  list(APPEND uv_sources src/unix/posix-hrtime.c src/unix/bsd-proctitle.c)
+endif()
+
+if(APPLE OR CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD|NetBSD|OpenBSD")
+  list(APPEND uv_sources src/unix/bsd-ifaddrs.c src/unix/kqueue.c)
+endif()
+
+if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+  list(APPEND uv_sources src/unix/random-getrandom.c)
+endif()
+
+if(APPLE OR CMAKE_SYSTEM_NAME STREQUAL "OpenBSD")
+  list(APPEND uv_sources src/unix/random-getentropy.c)
+endif()
+
+if(APPLE)
+  list(APPEND uv_defines _DARWIN_UNLIMITED_SELECT=1 _DARWIN_USE_64_BIT_INODE=1)
+  list(APPEND uv_sources
+       src/unix/darwin-proctitle.c
+       src/unix/darwin.c
+       src/unix/fsevents.c)
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+  list(APPEND uv_defines _GNU_SOURCE _POSIX_C_SOURCE=200112)
+  list(APPEND uv_libraries dl rt)
+  list(APPEND uv_sources
+       src/unix/linux-core.c
+       src/unix/linux-inotify.c
+       src/unix/linux-syscalls.c
+       src/unix/procfs-exepath.c
+       src/unix/random-getrandom.c
+       src/unix/random-sysctl-linux.c
+       src/unix/sysinfo-loadavg.c)
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "NetBSD")
+  list(APPEND uv_sources src/unix/netbsd.c)
+  list(APPEND uv_libraries kvm)
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "OpenBSD")
+  list(APPEND uv_sources src/unix/openbsd.c)
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "OS/390")
+  list(APPEND uv_defines PATH_MAX=255)
+  list(APPEND uv_defines _AE_BIMODAL)
+  list(APPEND uv_defines _ALL_SOURCE)
+  list(APPEND uv_defines _LARGE_TIME_API)
+  list(APPEND uv_defines _OPEN_MSGQ_EXT)
+  list(APPEND uv_defines _OPEN_SYS_FILE_EXT)
+  list(APPEND uv_defines _OPEN_SYS_IF_EXT)
+  list(APPEND uv_defines _OPEN_SYS_SOCK_EXT3)
+  list(APPEND uv_defines _OPEN_SYS_SOCK_IPV6)
+  list(APPEND uv_defines _UNIX03_SOURCE)
+  list(APPEND uv_defines _UNIX03_THREADS)
+  list(APPEND uv_defines _UNIX03_WITHDRAWN)
+  list(APPEND uv_defines _XOPEN_SOURCE_EXTENDED)
+  list(APPEND uv_sources
+       src/unix/pthread-fixes.c
+       src/unix/pthread-barrier.c
+       src/unix/os390.c
+       src/unix/os390-syscalls.c)
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "SunOS")
+  list(APPEND uv_defines __EXTENSIONS__ _XOPEN_SOURCE=500)
+  list(APPEND uv_libraries kstat nsl sendfile socket)
+  list(APPEND uv_sources src/unix/no-proctitle.c src/unix/sunos.c)
+endif()
+
+set(uv_sources_tmp "")
+foreach(file ${uv_sources})
+  list(APPEND uv_sources_tmp "${SOURCE_DIR}/${file}")
+endforeach(file)
+set(uv_sources "${uv_sources_tmp}")
+
+list(APPEND uv_defines CLICKHOUSE_GLIBC_COMPATIBILITY)
+
+add_library(uv ${uv_sources})
+target_compile_definitions(uv
+                           INTERFACE USING_UV_SHARED=1
+                           PRIVATE ${uv_defines} BUILDING_UV_SHARED=1)
+target_compile_options(uv PRIVATE ${uv_cflags})
+target_include_directories(uv PUBLIC ${SOURCE_DIR}/include PRIVATE ${SOURCE_DIR}/src)
+target_link_libraries(uv ${uv_libraries})
+
+add_library(uv_a STATIC ${uv_sources})
+target_compile_definitions(uv_a PRIVATE ${uv_defines})
+target_compile_options(uv_a PRIVATE ${uv_cflags})
+target_include_directories(uv_a PUBLIC ${SOURCE_DIR}/include PRIVATE ${SOURCE_DIR}/src)
+target_link_libraries(uv_a ${uv_libraries})
+
+if(UNIX)
+  # Now for some gibbering horrors from beyond the stars...
+  foreach(x ${uv_libraries})
+    set(LIBS "${LIBS} -l${x}")
+  endforeach(x)
+  file(STRINGS ${SOURCE_DIR}/configure.ac configure_ac REGEX ^AC_INIT)
+  string(REGEX MATCH [0-9]+[.][0-9]+[.][0-9]+ PACKAGE_VERSION "${configure_ac}")
+  string(REGEX MATCH ^[0-9]+ UV_VERSION_MAJOR "${PACKAGE_VERSION}")
+  # The version in the filename is mirroring the behaviour of autotools.
+  set_target_properties(uv PROPERTIES VERSION ${UV_VERSION_MAJOR}.0.0
+                      SOVERSION ${UV_VERSION_MAJOR})
+endif()
+
--- a/contrib/openldap-cmake/linux_x86_64/include/portable.h
+++ b/contrib/openldap-cmake/linux_x86_64/include/portable.h
@ -98,7 +98,9 @@
 #define HAVE_BCOPY 1

 /* Define to 1 if you have the <bits/types.h> header file. */
-#define HAVE_BITS_TYPES_H 1
+#if !defined(USE_MUSL)
+    #define HAVE_BITS_TYPES_H 1
+#endif

 /* Define to 1 if you have the `chroot' function. */
 #define HAVE_CHROOT 1
--- a/contrib/protobuf-cmake/CMakeLists.txt
+++ b/contrib/protobuf-cmake/CMakeLists.txt
@ -10,8 +10,82 @@ else ()
    set(protobuf_BUILD_SHARED_LIBS ON CACHE INTERNAL "" FORCE)
 endif ()

+if (CMAKE_CROSSCOMPILING)
+    # Will build 'protoc' for host arch instead of cross-compiling
+    set(protobuf_BUILD_PROTOC_BINARIES OFF CACHE INTERNAL "" FORCE)
+endif ()
+
 add_subdirectory("${protobuf_SOURCE_DIR}/cmake" "${protobuf_BINARY_DIR}")

 # We don't want to stop compilation on warnings in protobuf's headers.
-# The following line overrides the value assigned by the command target_include_directories() in libprotobuf.cmake 
+# The following line overrides the value assigned by the command target_include_directories() in libprotobuf.cmake
 set_property(TARGET libprotobuf PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${protobuf_SOURCE_DIR}/src")
+
+if (CMAKE_CROSSCOMPILING)
+    # Build 'protoc' for host arch
+    set (PROTOC_BUILD_DIR "${protobuf_BINARY_DIR}/build")
+
+    if (NOT EXISTS "${PROTOC_BUILD_DIR}/protoc")
+
+        # This is quite ugly but I cannot make dependencies work propery.
+
+        execute_process(
+            COMMAND mkdir -p ${PROTOC_BUILD_DIR}
+            COMMAND_ECHO STDOUT)
+
+        execute_process(
+            COMMAND ${CMAKE_COMMAND}
+                "-G${CMAKE_GENERATOR}"
+                "-DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}"
+                "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
+                "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
+                "-Dprotobuf_BUILD_TESTS=0"
+                "-Dprotobuf_BUILD_CONFORMANCE=0"
+                "-Dprotobuf_BUILD_EXAMPLES=0"
+                "-Dprotobuf_BUILD_PROTOC_BINARIES=1"
+                "${protobuf_SOURCE_DIR}/cmake"
+            WORKING_DIRECTORY "${PROTOC_BUILD_DIR}"
+            COMMAND_ECHO STDOUT)
+
+        execute_process(
+            COMMAND ${CMAKE_COMMAND} --build "${PROTOC_BUILD_DIR}"
+            COMMAND_ECHO STDOUT)
+    endif ()
+
+#    add_custom_command (
+#        OUTPUT ${PROTOC_BUILD_DIR}
+#        COMMAND mkdir -p ${PROTOC_BUILD_DIR})
+#
+#    add_custom_command (
+#        OUTPUT "${PROTOC_BUILD_DIR}/CMakeCache.txt"
+#
+#        COMMAND ${CMAKE_COMMAND}
+#             -G"${CMAKE_GENERATOR}"
+#             -DCMAKE_MAKE_PROGRAM="${CMAKE_MAKE_PROGRAM}"
+#             -DCMAKE_C_COMPILER="${CMAKE_C_COMPILER}"
+#             -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}"
+#             -Dprotobuf_BUILD_TESTS=0
+#             -Dprotobuf_BUILD_CONFORMANCE=0
+#             -Dprotobuf_BUILD_EXAMPLES=0
+#             -Dprotobuf_BUILD_PROTOC_BINARIES=1
+#             "${protobuf_SOURCE_DIR}/cmake"
+#
+#         DEPENDS "${PROTOC_BUILD_DIR}"
+#         WORKING_DIRECTORY "${PROTOC_BUILD_DIR}"
+#         COMMENT "Configuring 'protoc' for host architecture."
+#         USES_TERMINAL)
+#
+#     add_custom_command (
+#         OUTPUT "${PROTOC_BUILD_DIR}/protoc"
+#         COMMAND ${CMAKE_COMMAND} --build "${PROTOC_BUILD_DIR}"
+#         DEPENDS "${PROTOC_BUILD_DIR}/CMakeCache.txt"
+#         COMMENT "Building 'protoc' for host architecture."
+#         USES_TERMINAL)
+#
+#     add_custom_target (protoc-host DEPENDS "${PROTOC_BUILD_DIR}/protoc")
+
+    add_executable(protoc IMPORTED GLOBAL)
+    set_target_properties (protoc PROPERTIES IMPORTED_LOCATION "${PROTOC_BUILD_DIR}/protoc")
+    add_dependencies(protoc "${PROTOC_BUILD_DIR}/protoc")
+
+endif ()
--- a/contrib/protobuf-cmake/protobuf_generate.cmake
+++ b/contrib/protobuf-cmake/protobuf_generate.cmake
@ -181,9 +181,9 @@ function(protobuf_generate)

    add_custom_command(
      OUTPUT ${_generated_srcs}
-      COMMAND $<TARGET_FILE:protobuf::protoc>
+      COMMAND $<TARGET_FILE:protoc>
      ARGS --${protobuf_generate_LANGUAGE}_out ${_dll_export_decl}${protobuf_generate_PROTOC_OUT_DIR} ${_dll_desc_out} ${_protobuf_include_path} ${_abs_file}
-      DEPENDS ${_abs_file} protobuf::protoc
+      DEPENDS ${_abs_file} protoc
      COMMENT "Running ${protobuf_generate_LANGUAGE} protocol buffer compiler on ${_proto}"
      VERBATIM)
  endforeach()
--- a/contrib/s2geometry
+++ b/contrib/s2geometry
@ -1 +1 @@
-Subproject commit 20ea540d81f4575a3fc0aea585aac611bcd03ede
+Subproject commit 38b7a290f927cc372218c2094602b83e35b18c05
--- a/contrib/sysroot
+++ b/contrib/sysroot
@ -1 +1 @@
-Subproject commit 611d3315e9e369a338de4ffa128eb87b4fb87dec
+Subproject commit 6172893931e19b028f9cabb7095a44361be863df
--- a/docker/packager/packager
+++ b/docker/packager/packager
@ -61,6 +61,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
    DARWIN_ARM_SUFFIX = "-darwin-aarch64"
    ARM_SUFFIX = "-aarch64"
    FREEBSD_SUFFIX = "-freebsd"
+    PPC_SUFFIX = '-ppc64le'

    result = []
    cmake_flags = ['$CMAKE_FLAGS', '-DADD_GDB_INDEX_FOR_GOLD=1']
@ -69,8 +70,9 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
    is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
    is_cross_darwin_arm = compiler.endswith(DARWIN_ARM_SUFFIX)
    is_cross_arm = compiler.endswith(ARM_SUFFIX)
+    is_cross_ppc = compiler.endswith(PPC_SUFFIX)
    is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
-    is_cross_compile = is_cross_darwin or is_cross_darwin_arm or is_cross_arm or is_cross_freebsd
+    is_cross_compile = is_cross_darwin or is_cross_darwin_arm or is_cross_arm or is_cross_freebsd or is_cross_ppc

    # Explicitly use LLD with Clang by default.
    # Don't force linker for cross-compilation.
@ -97,6 +99,9 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
    elif is_cross_freebsd:
        cc = compiler[:-len(FREEBSD_SUFFIX)]
        cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake")
+    elif is_cross_ppc:
+        cc = compiler[:-len(PPC_SUFFIX)]
+        cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-ppc64le.cmake")
    else:
        cc = compiler

@ -205,7 +210,7 @@ if __name__ == "__main__":
    parser.add_argument("--build-type", choices=("debug", ""), default="")
    parser.add_argument("--compiler", choices=("clang-11", "clang-11-darwin", "clang-11-darwin-aarch64", "clang-11-aarch64",
                                               "clang-12", "clang-12-darwin", "clang-12-darwin-aarch64", "clang-12-aarch64",
-                                               "clang-13", "clang-13-darwin", "clang-13-darwin-aarch64", "clang-13-aarch64",
+                                               "clang-13", "clang-13-darwin", "clang-13-darwin-aarch64", "clang-13-aarch64", "clang-13-ppc64le",
                                               "clang-11-freebsd", "clang-12-freebsd", "clang-13-freebsd", "gcc-11"), default="clang-13")
    parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="")
    parser.add_argument("--unbundled", action="store_true")
--- a/docker/test/fasttest/Dockerfile
+++ b/docker/test/fasttest/Dockerfile
@ -67,7 +67,7 @@ RUN apt-get update \
        unixodbc \
       --yes --no-install-recommends

-RUN pip3 install numpy scipy pandas Jinja2 pandas clickhouse_driver
+RUN pip3 install numpy scipy pandas Jinja2

 # This symlink required by gcc to find lld compiler
 RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@ -189,7 +189,7 @@ function clone_submodules
        )

        git submodule sync
-        git submodule update --depth 1 --init --recursive "${SUBMODULES_TO_UPDATE[@]}"
+        git submodule update --depth 1 --init "${SUBMODULES_TO_UPDATE[@]}"
        git submodule foreach git reset --hard
        git submodule foreach git checkout @ -f
        git submodule foreach git clean -xfd
@ -262,11 +262,13 @@ function run_tests

    start_server

+    set +e
    time clickhouse-test --hung-check -j 8 --order=random \
            --fast-tests-only --no-long --testname --shard --zookeeper \
            -- "$FASTTEST_FOCUS" 2>&1 \
        | ts '%Y-%m-%d %H:%M:%S' \
        | tee "$FASTTEST_OUTPUT/test_result.txt"
+    set -e
 }

 case "$stage" in
--- a/docker/test/fuzzer/Dockerfile
+++ b/docker/test/fuzzer/Dockerfile
@ -27,7 +27,7 @@ RUN apt-get update \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*

-RUN pip3 install Jinja2 pandas clickhouse_driver
+RUN pip3 install Jinja2

 COPY * /

--- a/docker/test/stateful/run.sh
+++ b/docker/test/stateful/run.sh
@ -108,8 +108,10 @@ function run_tests()
        ADDITIONAL_OPTIONS+=('--replicated-database')
    fi

+    set +e
    clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --print-time "${ADDITIONAL_OPTIONS[@]}" \
        "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
+    set -e
 }

 export -f run_tests
--- a/docker/test/stateless/Dockerfile
+++ b/docker/test/stateless/Dockerfile
@ -34,7 +34,7 @@ RUN apt-get update -y \
            postgresql-client \
            sqlite3

-RUN pip3 install numpy scipy pandas Jinja2 clickhouse_driver
+RUN pip3 install numpy scipy pandas Jinja2

 RUN mkdir -p /tmp/clickhouse-odbc-tmp \
   && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@ -96,10 +96,12 @@ function run_tests()
        ADDITIONAL_OPTIONS+=('8')
    fi

+    set +e
    clickhouse-test --testname --shard --zookeeper --hung-check --print-time \
            --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
        | ts '%Y-%m-%d %H:%M:%S' \
        | tee -a test_output/test_result.txt
+    set -e
 }

 export -f run_tests
@ -114,12 +116,6 @@ grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||:
 pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz &
 clickhouse-client -q "select * from system.query_log format TSVWithNamesAndTypes" | pigz > /test_output/query-log.tsv.gz &
 clickhouse-client -q "select * from system.query_thread_log format TSVWithNamesAndTypes" | pigz > /test_output/query-thread-log.tsv.gz &
-clickhouse-client --allow_introspection_functions=1 -q "
-    WITH
-        arrayMap(x -> concat(demangle(addressToSymbol(x)), ':', addressToLine(x)), trace) AS trace_array,
-        arrayStringConcat(trace_array, '\n') AS trace_string
-    SELECT * EXCEPT(trace), trace_string FROM system.trace_log FORMAT TSVWithNamesAndTypes
-" | pigz > /test_output/trace-log.tsv.gz &

 # Also export trace log in flamegraph-friendly format.
 for trace_type in CPU Memory Real
@ -146,6 +142,7 @@ fi
 tar -chf /test_output/text_log_dump.tar /var/lib/clickhouse/data/system/text_log ||:
 tar -chf /test_output/query_log_dump.tar /var/lib/clickhouse/data/system/query_log ||:
 tar -chf /test_output/zookeeper_log_dump.tar /var/lib/clickhouse/data/system/zookeeper_log ||:
+tar -chf /test_output/trace_log_dump.tar /var/lib/clickhouse/data/system/trace_log ||:
 tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:

 if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
--- a/docker/test/style/Dockerfile
+++ b/docker/test/style/Dockerfile
@ -10,7 +10,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
    python3-pip \
    pylint \
    yamllint \
-    && pip3 install codespell pandas clickhouse_driver
+    && pip3 install codespell

 COPY run.sh /
 COPY process_style_check_result.py /
--- a/docker/test/util/process_functional_tests_result.py
+++ b/docker/test/util/process_functional_tests_result.py
@ -28,6 +28,7 @@ def process_test_log(log_path):
    test_results = []
    with open(log_path, 'r') as test_file:
        for line in test_file:
+            original_line = line
            line = line.strip()
            if any(s in line for s in NO_TASK_TIMEOUT_SIGNS):
                task_timeout = False
@ -49,19 +50,24 @@ def process_test_log(log_path):
                total += 1
                if TIMEOUT_SIGN in line:
                    failed += 1
-                    test_results.append((test_name, "Timeout", test_time))
+                    test_results.append((test_name, "Timeout", test_time, []))
                elif FAIL_SIGN in line:
                    failed += 1
-                    test_results.append((test_name, "FAIL", test_time))
+                    test_results.append((test_name, "FAIL", test_time, []))
                elif UNKNOWN_SIGN in line:
                    unknown += 1
-                    test_results.append((test_name, "FAIL", test_time))
+                    test_results.append((test_name, "FAIL", test_time, []))
                elif SKIPPED_SIGN in line:
                    skipped += 1
-                    test_results.append((test_name, "SKIPPED", test_time))
+                    test_results.append((test_name, "SKIPPED", test_time, []))
                else:
                    success += int(OK_SIGN in line)
-                    test_results.append((test_name, "OK", test_time))
+                    test_results.append((test_name, "OK", test_time, []))
+            elif len(test_results) > 0 and test_results[-1][1] == "FAIL":
+                test_results[-1][3].append(original_line)
+
+    test_results = [(test[0], test[1], test[2], ''.join(test[3])) for test in test_results]
+
    return total, skipped, unknown, failed, success, hung, task_timeout, retries, test_results

 def process_result(result_path):
@ -89,14 +95,14 @@ def process_result(result_path):
        if hung:
            description = "Some queries hung, "
            state = "failure"
-            test_results.append(("Some queries hung", "FAIL", "0"))
+            test_results.append(("Some queries hung", "FAIL", "0", ""))
        elif task_timeout:
            description = "Timeout, "
            state = "failure"
-            test_results.append(("Timeout", "FAIL", "0"))
+            test_results.append(("Timeout", "FAIL", "0", ""))
        elif retries:
            description = "Some tests restarted, "
-            test_results.append(("Some tests restarted", "SKIPPED", "0"))
+            test_results.append(("Some tests restarted", "SKIPPED", "0", ""))
        else:
            description = ""

--- a/docs/_includes/install/universal.sh
+++ b/docs/_includes/install/universal.sh
@ -0,0 +1,63 @@
+#!/bin/sh -e
+
+OS=$(uname -s)
+ARCH=$(uname -m)
+
+DIR=
+
+if [ "${OS}" = "Linux" ]
+then
+    if [ "${ARCH}" = "x86_64" ]
+    then
+        DIR="amd64"
+    elif [ "${ARCH}" = "aarch64" ]
+    then
+        DIR="aarch64"
+    elif [ "${ARCH}" = "powerpc64le" ]
+    then
+        DIR="powerpc64le"
+    fi
+elif [ "${OS}" = "FreeBSD" ]
+then
+    if [ "${ARCH}" = "x86_64" ]
+    then
+        DIR="freebsd"
+    elif [ "${ARCH}" = "aarch64" ]
+    then
+        DIR="freebsd-aarch64"
+    elif [ "${ARCH}" = "powerpc64le" ]
+    then
+        DIR="freebsd-powerpc64le"
+    fi
+elif [ "${OS}" = "Darwin" ]
+then
+    if [ "${ARCH}" = "x86_64" ]
+    then
+        DIR="macos"
+    elif [ "${ARCH}" = "aarch64" ]
+    then
+        DIR="macos-aarch64"
+    fi
+fi
+
+if [ -z "${DIR}" ]
+then
+    echo "The '${OS}' operating system with the '${ARCH}' architecture is not supported."
+    exit 1
+fi
+
+URL="https://builds.clickhouse.com/master/${DIR}/clickhouse"
+echo
+echo "Will download ${URL}"
+echo
+curl -O "${URL}" && chmod a+x clickhouse &&
+echo
+echo "Successfully downloaded the ClickHouse binary, you can run it as:
+    ./clickhouse"
+
+if [ "${OS}" = "Linux" ]
+then
+    echo
+    echo "You can also install it:
+    sudo ./clickhouse install"
+fi
--- a/docs/en/development/developer-instruction.md
+++ b/docs/en/development/developer-instruction.md
@ -37,7 +37,7 @@ Next, you need to download the source files onto your working machine. This is c

 In the command line terminal run:

-    git clone --recursive git@github.com:your_github_username/ClickHouse.git
+    git clone git@github.com:your_github_username/ClickHouse.git
    cd ClickHouse

 Note: please, substitute *your_github_username* with what is appropriate!
@ -79,7 +79,7 @@ After successfully running this command you will be able to pull updates from th

 Working with submodules in git could be painful. Next commands will help to manage it:

-    # ! each command accepts --recursive
+    # ! each command accepts
    # Update remote URLs for submodules. Barely rare case
    git submodule sync
    # Add new submodules
@ -92,16 +92,16 @@ Working with submodules in git could be painful. Next commands will help to mana
 The next commands would help you to reset all submodules to the initial state (!WARNING! - any changes inside will be deleted):

    # Synchronizes submodules' remote URL with .gitmodules
-    git submodule sync --recursive
+    git submodule sync
    # Update the registered submodules with initialize not yet initialized
-    git submodule update --init --recursive
+    git submodule update --init
    # Reset all changes done after HEAD
    git submodule foreach git reset --hard
    # Clean files from .gitignore
    git submodule foreach git clean -xfd
    # Repeat last 4 commands for all submodule
-    git submodule foreach git submodule sync --recursive
-    git submodule foreach git submodule update --init --recursive
+    git submodule foreach git submodule sync
+    git submodule foreach git submodule update --init
    git submodule foreach git submodule foreach git reset --hard
    git submodule foreach git submodule foreach git clean -xfd

--- a/docs/en/engines/table-engines/special/distributed.md
+++ b/docs/en/engines/table-engines/special/distributed.md
@ -188,14 +188,15 @@ When the `max_parallel_replicas` option is enabled, query processing is parallel

 ## Virtual Columns {#virtual-columns}

-   `_shard_num` — Contains the `shard_num` (from `system.clusters`). Type: [UInt32](../../../sql-reference/data-types/int-uint.md).
+-   `_shard_num` — Contains the `shard_num` value from the table `system.clusters`. Type: [UInt32](../../../sql-reference/data-types/int-uint.md).

 !!! note "Note"
-    Since [`remote`](../../../sql-reference/table-functions/remote.md)/`cluster` table functions internally create temporary instance of the same Distributed engine, `_shard_num` is available there too.
+    Since [remote](../../../sql-reference/table-functions/remote.md) and [cluster](../../../sql-reference/table-functions/cluster.md) table functions internally create temporary Distributed table, `_shard_num` is available there too.

 **See Also**

-   [Virtual columns](../../../engines/table-engines/special/index.md#table_engines-virtual_columns)
-   [background_distributed_schedule_pool_size](../../../operations/settings/settings.md#background_distributed_schedule_pool_size)
+-   [Virtual columns](../../../engines/table-engines/index.md#table_engines-virtual_columns) description
+-   [background_distributed_schedule_pool_size](../../../operations/settings/settings.md#background_distributed_schedule_pool_size) setting
+-   [shardNum()](../../../sql-reference/functions/other-functions.md#shard-num) and [shardCount()](../../../sql-reference/functions/other-functions.md#shard-count) functions
+

-[Original article](https://clickhouse.com/docs/en/operations/table_engines/distributed/) <!--hide-->
--- a/docs/en/getting-started/install.md
+++ b/docs/en/getting-started/install.md
@ -27,10 +27,11 @@ It is recommended to use official pre-compiled `deb` packages for Debian or Ubun
 {% include 'install/deb.sh' %}
 ```

-If you want to use the most recent version, replace `stable` with `testing` (this is recommended for your testing environments).
+You can replace `stable` with `lts` or `testing` to use different [“release trains”](../faq/operations/production.md) based on your needs.

 You can also download and install packages manually from [here](https://repo.clickhouse.com/deb/stable/main/).

+
 #### Packages {#packages}

 -   `clickhouse-common-static` — Installs ClickHouse compiled binary files.
--- a/docs/en/interfaces/cli.md
+++ b/docs/en/interfaces/cli.md
@ -127,6 +127,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
 -   `--secure` – If specified, will connect to server over secure connection.
 -   `--history_file` — Path to a file containing command history.
 -   `--param_<name>` — Value for a [query with parameters](#cli-queries-with-parameters).
+-   `--hardware-utilization` — Print hardware utilization information in progress bar.

 Since version 20.5, `clickhouse-client` has automatic syntax highlighting (always enabled).

--- a/docs/en/introduction/adopters.md
+++ b/docs/en/introduction/adopters.md
@ -15,6 +15,7 @@ toc_title: Adopters
 | <a href="http://www.adscribe.tv/" class="favicon">AdScribe</a> | Ads | TV Analytics | — | — | [A quote from CTO](https://altinity.com/24x7-support/) |
 | <a href="https://ahrefs.com/" class="favicon">Ahrefs</a> | SEO | Analytics | — | — | [Job listing](https://ahrefs.com/jobs/data-scientist-search) |
 | <a href="https://cn.aliyun.com/" class="favicon">Alibaba Cloud</a> | Cloud | Managed Service | — | — | [Official Website](https://help.aliyun.com/product/144466.html) |
+| <a href="https://cn.aliyun.com/" class="favicon">Alibaba Cloud</a> | Cloud | E-MapReduce | — | — | [Official Website](https://help.aliyun.com/document_detail/212195.html) |
 | <a href="https://alohabrowser.com/" class="favicon">Aloha Browser</a> | Mobile App | Browser backend | — | — | [Slides in Russian, May 2019](https://presentations.clickhouse.com/meetup22/aloha.pdf) |
 | <a href="https://altinity.com/" class="favicon">Altinity</a> | Cloud, SaaS | Main product | — | — | [Official Website](https://altinity.com/) |
 | <a href="https://amadeus.com/" class="favicon">Amadeus</a> | Travel | Analytics | — | — | [Press Release, April 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) |
@ -31,6 +32,7 @@ toc_title: Adopters
 | <a href="https://cardsmobile.ru/" class="favicon">CardsMobile</a> | Finance | Analytics | — | — | [VC.ru](https://vc.ru/s/cardsmobile/143449-rukovoditel-gruppy-analiza-dannyh) |
 | <a href="https://carto.com/" class="favicon">CARTO</a> | Business Intelligence | Geo analytics | — | — | [Geospatial processing with ClickHouse](https://carto.com/blog/geospatial-processing-with-clickhouse/) |
 | <a href="http://public.web.cern.ch/public/" class="favicon">CERN</a> | Research | Experiment | — | — | [Press release, April 2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) |
+| <a href="https://www.checklyhq.com/" class="favicon">Checkly</a> | Software Development | Analytics | — | — | [Tweet, October 2021](https://twitter.com/tim_nolet/status/1445810665743081474?s=20) |
 | <a href="http://cisco.com/" class="favicon">Cisco</a> | Networking | Traffic analysis | — | — | [Lightning talk, October 2019](https://youtu.be/-hI1vDR2oPY?t=5057) |
 | <a href="https://www.citadelsecurities.com/" class="favicon">Citadel Securities</a> | Finance | — | — | — | [Contribution, March 2019](https://github.com/ClickHouse/ClickHouse/pull/4774) |
 | <a href="https://city-mobil.ru" class="favicon">Citymobil</a> | Taxi | Analytics | — | — | [Blog Post in Russian, March 2020](https://habr.com/en/company/citymobil/blog/490660/) |
@ -115,6 +117,7 @@ toc_title: Adopters
 | <a href="https://seo.do/" class="favicon">seo.do</a> | Analytics | Main product | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) |
 | <a href="http://www.sgk.gov.tr/wps/portal/sgk/tr" class="favicon">SGK</a> | Government Social Security | Analytics | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) |
 | <a href="http://english.sina.com/index.html" class="favicon">Sina</a> | News | — | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) |
+| <a href="https://www.sipfront.com/" class="favicon">Sipfront</a> | Software Development | Analytics | — | — | [Tweet, October 2021](https://twitter.com/andreasgranig/status/1446404332337913895?s=20) |
 | <a href="https://smi2.ru/" class="favicon">SMI2</a> | News | Analytics | — | — | [Blog Post in Russian, November 2017](https://habr.com/ru/company/smi2/blog/314558/) |
 | <a href="https://www.spark.co.nz/" class="favicon">Spark New Zealand</a> | Telecommunications | Security Operations | — | — | [Blog Post, Feb 2020](https://blog.n0p.me/2020/02/2020-02-05-dnsmonster/) |
 | <a href="https://splitbee.io" class="favicon">Splitbee</a> | Analytics | Main Product | — | — | [Blog Post, Mai 2021](https://splitbee.io/blog/new-pricing) |
--- a/Show More
+++ b/Show More
				`@ -0,0 +1 @@`
				`Subproject commit ffd86a32874e5c08a143019aad1aaf0907294c9f`