Merge branch 'master' into debug_keeper

.gitmodules

@@ -213,6 +213,7 @@
 [submodule "contrib/boringssl"]
 	path = contrib/boringssl
 	url = https://github.com/ClickHouse-Extras/boringssl.git
+	branch = MergeWithUpstream
 [submodule "contrib/NuRaft"]
 	path = contrib/NuRaft
 	url = https://github.com/ClickHouse-Extras/NuRaft.git
@@ -249,6 +250,9 @@
 [submodule "contrib/magic_enum"]
 	path = contrib/magic_enum
 	url = https://github.com/Neargye/magic_enum
+[submodule "contrib/libprotobuf-mutator"]
+	path = contrib/libprotobuf-mutator
+	url = https://github.com/google/libprotobuf-mutator
 [submodule "contrib/sysroot"]
 	path = contrib/sysroot
 	url = https://github.com/ClickHouse-Extras/sysroot.git

CHANGELOG.md

@@ -1,4 +1,4 @@
-### ClickHouse release v21.10, 2021-10-14
+### ClickHouse release v21.10, 2021-10-16
 
 #### Backward Incompatible Change
 

CMakeLists.txt

@@ -136,6 +136,21 @@ if (ENABLE_FUZZING)
     message (STATUS "Fuzzing instrumentation enabled")
     set (FUZZER "libfuzzer")
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostdlib++")
+    set (ENABLE_CLICKHOUSE_ODBC_BRIDGE OFF)
+    set (ENABLE_LIBRARIES 0)
+    set (ENABLE_SSL 1)
+    set (USE_INTERNAL_SSL_LIBRARY 1)
+    set (USE_UNWIND ON)
+    set (ENABLE_EMBEDDED_COMPILER 0)
+    set (ENABLE_EXAMPLES 0)
+    set (ENABLE_UTILS 0)
+    set (ENABLE_THINLTO 0)
+    set (ENABLE_TCMALLOC 0)
+    set (ENABLE_JEMALLOC 0)
+    set (ENABLE_CHECK_HEAVY_BUILDS 1)
+    set (GLIBC_COMPATIBILITY OFF)
+    set (ENABLE_PROTOBUF ON)
+    set (USE_INTERNAL_PROTOBUF_LIBRARY ON)
 endif()
 
 # Global libraries
@@ -188,7 +203,7 @@ endif ()
 option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
 option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF)
 
-if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0")
+if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND NOT USE_MUSL)
     # Only for Linux, x86_64 or aarch64.
     option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON)
 elseif(GLIBC_COMPATIBILITY)
@@ -203,10 +218,6 @@ if (GLIBC_COMPATIBILITY)
     set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -include ${CMAKE_CURRENT_SOURCE_DIR}/base/glibc-compatibility/glibc-compat-2.32.h")
 endif()
 
-if (NOT CMAKE_VERSION VERSION_GREATER "3.9.0")
-    message (WARNING "CMake version must be greater than 3.9.0 for production builds.")
-endif ()
-
 # Make sure the final executable has symbols exported
 set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic")
 
@@ -582,6 +593,7 @@ include (cmake/find/cassandra.cmake)
 include (cmake/find/sentry.cmake)
 include (cmake/find/stats.cmake)
 include (cmake/find/datasketches.cmake)
+include (cmake/find/libprotobuf-mutator.cmake)
 
 set (USE_INTERNAL_CITYHASH_LIBRARY ON CACHE INTERNAL "")
 find_contrib_lib(cityhash)

base/base/LineReader.cpp

@@ -5,6 +5,10 @@
 
 #include <string.h>
 #include <unistd.h>
+#include <sys/select.h>
+#include <sys/time.h>
+#include <sys/types.h>
+
 
 #ifdef OS_LINUX
 /// We can detect if code is linked with one or another readline variants or open the library dynamically.

base/base/phdr_cache.cpp

@@ -6,7 +6,7 @@
 
 #include <base/defines.h>
 
-#if defined(__linux__) && !defined(THREAD_SANITIZER)
+#if defined(__linux__) && !defined(THREAD_SANITIZER) && !defined(USE_MUSL)
     #define USE_PHDR_CACHE 1
 #endif
 

base/loggers/Loggers.cpp

@@ -84,7 +84,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
 
         Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, log_file);
         log->setLevel(log_level);
-        split->addChannel(log);
+        split->addChannel(log, "log");
     }
 
     const auto errorlog_path = config.getString("logger.errorlog", "");
@@ -116,7 +116,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
         Poco::AutoPtr<DB::OwnFormattingChannel> errorlog = new DB::OwnFormattingChannel(pf, error_log_file);
         errorlog->setLevel(errorlog_level);
         errorlog->open();
-        split->addChannel(errorlog);
+        split->addChannel(errorlog, "errorlog");
     }
 
     if (config.getBool("logger.use_syslog", false))
@@ -155,7 +155,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
         Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, syslog_channel);
         log->setLevel(syslog_level);
 
-        split->addChannel(log);
+        split->addChannel(log, "syslog");
     }
 
     bool should_log_to_console = isatty(STDIN_FILENO) || isatty(STDERR_FILENO);
@@ -177,7 +177,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
         Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, new Poco::ConsoleChannel);
         logger.warning("Logging " + console_log_level_string + " to console");
         log->setLevel(console_log_level);
-        split->addChannel(log);
+        split->addChannel(log, "console");
     }
 
     split->open();
@@ -224,6 +224,89 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
     }
 }
 
+void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger)
+{
+    int max_log_level = 0;
+
+    const auto log_level_string = config.getString("logger.level", "trace");
+    int log_level = Poco::Logger::parseLevel(log_level_string);
+    if (log_level > max_log_level)
+        max_log_level = log_level;
+
+    const auto log_path = config.getString("logger.log", "");
+    if (!log_path.empty())
+        split->setLevel("log", log_level);
+    else
+        split->setLevel("log", 0);
+
+    // Set level to console
+    bool is_daemon = config.getBool("application.runAsDaemon", false);
+    bool should_log_to_console = isatty(STDIN_FILENO) || isatty(STDERR_FILENO);
+    if (config.getBool("logger.console", false)
+        || (!config.hasProperty("logger.console") && !is_daemon && should_log_to_console))
+        split->setLevel("console", log_level);
+    else
+        split->setLevel("console", 0);
+
+    // Set level to errorlog
+    int errorlog_level = 0;
+    const auto errorlog_path = config.getString("logger.errorlog", "");
+    if (!errorlog_path.empty())
+    {
+        errorlog_level = Poco::Logger::parseLevel(config.getString("logger.errorlog_level", "notice"));
+        if (errorlog_level > max_log_level)
+            max_log_level = errorlog_level;
+    }
+    split->setLevel("errorlog", errorlog_level);
+
+    // Set level to syslog
+    int syslog_level = 0;
+    if (config.getBool("logger.use_syslog", false))
+    {
+        syslog_level = Poco::Logger::parseLevel(config.getString("logger.syslog_level", log_level_string));
+        if (syslog_level > max_log_level)
+            max_log_level = syslog_level;
+    }
+    split->setLevel("syslog", syslog_level);
+
+    // Global logging level (it can be overridden for specific loggers).
+    logger.setLevel(max_log_level);
+
+    // Set level to all already created loggers
+    std::vector<std::string> names;
+
+    logger.root().names(names);
+    for (const auto & name : names)
+        logger.root().get(name).setLevel(max_log_level);
+
+    logger.root().setLevel(max_log_level);
+
+    // Explicitly specified log levels for specific loggers.
+    {
+        Poco::Util::AbstractConfiguration::Keys loggers_level;
+        config.keys("logger.levels", loggers_level);
+
+        if (!loggers_level.empty())
+        {
+            for (const auto & key : loggers_level)
+            {
+                if (key == "logger" || key.starts_with("logger["))
+                {
+                    const std::string name(config.getString("logger.levels." + key + ".name"));
+                    const std::string level(config.getString("logger.levels." + key + ".level"));
+                    logger.root().get(name).setLevel(level);
+                }
+                else
+                {
+                    // Legacy syntax
+                    const std::string level(config.getString("logger.levels." + key, "trace"));
+                    logger.root().get(key).setLevel(level);
+                }
+            }
+        }
+    }
+}
+
 void Loggers::closeLogs(Poco::Logger & logger)
 {
     if (log_file)

base/loggers/Loggers.h

@@ -19,6 +19,8 @@ class Loggers
 public:
     void buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger, const std::string & cmd_name = "");
 
+    void updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger);
+
     /// Close log files. On next log write files will be reopened.
     void closeLogs(Poco::Logger & logger);
 

base/loggers/OwnFormattingChannel.h

@@ -1,4 +1,5 @@
 #pragma once
+#include <atomic>
 #include <Poco/AutoPtr.h>
 #include <Poco/Channel.h>
 #include <Poco/FormattingChannel.h>
@@ -14,7 +15,7 @@ class OwnFormattingChannel : public Poco::Channel, public ExtendedLogChannel
 public:
     explicit OwnFormattingChannel(
         Poco::AutoPtr<OwnPatternFormatter> pFormatter_ = nullptr, Poco::AutoPtr<Poco::Channel> pChannel_ = nullptr)
-        : pFormatter(std::move(pFormatter_)), pChannel(std::move(pChannel_))
+        : pFormatter(std::move(pFormatter_)), pChannel(std::move(pChannel_)), priority(Poco::Message::PRIO_TRACE)
     {
     }
 
@@ -45,7 +46,7 @@ public:
 private:
     Poco::AutoPtr<OwnPatternFormatter> pFormatter;
     Poco::AutoPtr<Poco::Channel> pChannel;
-    Poco::Message::Priority priority = Poco::Message::PRIO_TRACE;
+    std::atomic<Poco::Message::Priority> priority;
 };
 
 }

base/loggers/OwnSplitChannel.cpp

@@ -1,4 +1,5 @@
 #include "OwnSplitChannel.h"
+#include "OwnFormattingChannel.h"
 
 #include <iostream>
 #include <Core/Block.h>
@@ -75,7 +76,7 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg)
     ExtendedLogMessage msg_ext = ExtendedLogMessage::getFrom(msg);
 
     /// Log data to child channels
-    for (auto & channel : channels)
+    for (auto & [name, channel] : channels)
     {
         if (channel.second)
             channel.second->logExtended(msg_ext); // extended child
@@ -100,7 +101,7 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg)
         columns[i++]->insert(msg.getSource());
         columns[i++]->insert(msg.getText());
 
-        logs_queue->emplace(std::move(columns));
+        [[maybe_unused]] bool push_result = logs_queue->emplace(std::move(columns));
     }
 
     /// Also log to system.text_log table, if message is not too noisy
@@ -137,9 +138,9 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg)
 }
 
 
-void OwnSplitChannel::addChannel(Poco::AutoPtr<Poco::Channel> channel)
+void OwnSplitChannel::addChannel(Poco::AutoPtr<Poco::Channel> channel, const std::string & name)
 {
-    channels.emplace_back(std::move(channel), dynamic_cast<ExtendedLogChannel *>(channel.get()));
+    channels.emplace(name, ExtendedChannelPtrPair(std::move(channel), dynamic_cast<ExtendedLogChannel *>(channel.get())));
 }
 
 void OwnSplitChannel::addTextLog(std::shared_ptr<DB::TextLog> log, int max_priority)
@@ -149,4 +150,14 @@ void OwnSplitChannel::addTextLog(std::shared_ptr<DB::TextLog> log, int max_prior
     text_log_max_priority.store(max_priority, std::memory_order_relaxed);
 }
 
+void OwnSplitChannel::setLevel(const std::string & name, int level)
+{
+     auto it = channels.find(name);
+     if (it != channels.end())
+     {
+         if (auto * channel = dynamic_cast<DB::OwnFormattingChannel *>(it->second.first.get()))
+            channel->setLevel(level);
+     }
+}
+
 }

base/loggers/OwnSplitChannel.h

@@ -18,10 +18,12 @@ public:
     /// Makes an extended message from msg and passes it to the client logs queue and child (if possible)
     void log(const Poco::Message & msg) override;
     /// Adds a child channel
-    void addChannel(Poco::AutoPtr<Poco::Channel> channel);
+    void addChannel(Poco::AutoPtr<Poco::Channel> channel, const std::string & name);
 
     void addTextLog(std::shared_ptr<DB::TextLog> log, int max_priority);
 
+    void setLevel(const std::string & name, int level);
+
 private:
     void logSplit(const Poco::Message & msg);
     void tryLogSplit(const Poco::Message & msg);
@@ -29,7 +31,7 @@ private:
     using ChannelPtr = Poco::AutoPtr<Poco::Channel>;
     /// Handler and its pointer casted to extended interface
     using ExtendedChannelPtrPair = std::pair<ChannelPtr, ExtendedLogChannel *>;
-    std::vector<ExtendedChannelPtrPair> channels;
+    std::map<std::string, ExtendedChannelPtrPair> channels;
 
     std::mutex text_log_mutex;
 

benchmark/hardware.sh

@@ -13,6 +13,7 @@ TRIES=3
 
 AMD64_BIN_URL="https://builds.clickhouse.com/master/amd64/clickhouse"
 AARCH64_BIN_URL="https://builds.clickhouse.com/master/aarch64/clickhouse"
+POWERPC64_BIN_URL="https://builds.clickhouse.com/master/ppc64le/clickhouse"
 
 # Note: on older Ubuntu versions, 'axel' does not support IPv6. If you are using IPv6-only servers on very old Ubuntu, just don't install 'axel'.
 
@@ -38,6 +39,8 @@ if [[ ! -f clickhouse ]]; then
         $FASTER_DOWNLOAD "$AMD64_BIN_URL"
     elif [[ $CPU == aarch64 ]]; then
         $FASTER_DOWNLOAD "$AARCH64_BIN_URL"
+    elif [[ $CPU == powerpc64le ]]; then
+        $FASTER_DOWNLOAD "$POWERPC64_BIN_URL"
     else
         echo "Unsupported CPU type: $CPU"
         exit 1
@@ -52,7 +55,7 @@ fi
 
 if [[ ! -d data ]]; then
     if [[ ! -f $DATASET ]]; then
-        $FASTER_DOWNLOAD "https://clickhouse-datasets.s3.yandex.net/hits/partitions/$DATASET"
+        $FASTER_DOWNLOAD "https://datasets.clickhouse.com/hits/partitions/$DATASET"
     fi
 
     tar $TAR_PARAMS --strip-components=1 --directory=. -x -v -f $DATASET

cmake/find/amqpcpp.cmake

@@ -10,7 +10,7 @@ if (NOT ENABLE_AMQPCPP)
 endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/CMakeLists.txt")
-    message (WARNING "submodule contrib/AMQP-CPP is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/AMQP-CPP is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal AMQP-CPP library")
     set (USE_AMQPCPP 0)
     return()

cmake/find/avro.cmake

@@ -13,7 +13,7 @@ option (USE_INTERNAL_AVRO_LIBRARY
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/CMakeLists.txt")
     if (USE_INTERNAL_AVRO_LIBRARY)
-        message(WARNING "submodule contrib/avro is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/avro is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find internal avro")
         set(USE_INTERNAL_AVRO_LIBRARY 0)
     endif()

cmake/find/base64.cmake

@@ -10,11 +10,11 @@ endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64/LICENSE")
     set (MISSING_INTERNAL_BASE64_LIBRARY 1)
-    message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init")
 endif ()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64")
-    message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init")
 else()
     set (BASE64_LIBRARY base64)
     set (USE_BASE64 1)

cmake/find/brotli.cmake

@@ -16,7 +16,7 @@ endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include/brotli/decode.h")
     if (USE_INTERNAL_BROTLI_LIBRARY)
-        message (WARNING "submodule contrib/brotli is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/brotli is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find internal brotli")
         set (USE_INTERNAL_BROTLI_LIBRARY 0)
     endif ()

cmake/find/bzip2.cmake

@@ -6,7 +6,7 @@ if (NOT ENABLE_BZIP2)
 endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/bzip2/bzlib.h")
-    message (WARNING "submodule contrib/bzip2 is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/bzip2 is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal bzip2 library")
     set (USE_NLP 0)
     return()

cmake/find/capnp.cmake

@@ -11,7 +11,7 @@ option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/capnproto/CMakeLists.txt")
     if(USE_INTERNAL_CAPNP_LIBRARY)
-        message(WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init")
         message(${RECONFIGURE_MESSAGE_LEVEL} "cannot find internal capnproto")
         set(USE_INTERNAL_CAPNP_LIBRARY 0)
     endif()

cmake/find/cassandra.cmake

@@ -14,7 +14,7 @@ if (APPLE)
 endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra")
-    message (ERROR "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init --recursive")
+    message (ERROR "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal Cassandra")
     set (USE_CASSANDRA 0)
     return()

cmake/find/cxx.cmake

@@ -17,7 +17,7 @@ option (USE_INTERNAL_LIBCXX_LIBRARY "Disable to use system libcxx and libcxxabi
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/CMakeLists.txt")
     if (USE_INTERNAL_LIBCXX_LIBRARY)
-        message(WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libcxx")
         set(USE_INTERNAL_LIBCXX_LIBRARY 0)
     endif()

cmake/find/cyrus-sasl.cmake

@@ -6,7 +6,7 @@ endif()
 
 OPTION(ENABLE_CYRUS_SASL "Enable cyrus-sasl" ${DEFAULT_ENABLE_CYRUS_SASL})
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cyrus-sasl/README")
-    message (WARNING "submodule contrib/cyrus-sasl is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/cyrus-sasl is missing. to fix try run: \n git submodule update --init")
     set (ENABLE_CYRUS_SASL 0)
 endif ()
 

cmake/find/datasketches.cmake

@@ -6,7 +6,7 @@ option (USE_INTERNAL_DATASKETCHES_LIBRARY "Set to FALSE to use system DataSketch
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/theta/CMakeLists.txt")
     if (USE_INTERNAL_DATASKETCHES_LIBRARY)
-       message(WARNING "submodule contrib/datasketches-cpp is missing. to fix try run: \n git submodule update --init --recursive")
+       message(WARNING "submodule contrib/datasketches-cpp is missing. to fix try run: \n git submodule update --init")
     endif()
     set(MISSING_INTERNAL_DATASKETCHES_LIBRARY 1)
     set(USE_INTERNAL_DATASKETCHES_LIBRARY 0)

cmake/find/fast_float.cmake

@@ -1,5 +1,5 @@
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/fast_float/include/fast_float/fast_float.h")
-    message (FATAL_ERROR "submodule contrib/fast_float is missing. to fix try run: \n git submodule update --init --recursive")
+    message (FATAL_ERROR "submodule contrib/fast_float is missing. to fix try run: \n git submodule update --init")
 endif ()
 
 set(FAST_FLOAT_LIBRARY fast_float)

cmake/find/fastops.cmake

@@ -10,7 +10,7 @@ if(NOT ENABLE_FASTOPS)
 endif()
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/fastops/fastops/fastops.h")
-    message(WARNING "submodule contrib/fastops is missing. to fix try run: \n git submodule update --init --recursive")
+    message(WARNING "submodule contrib/fastops is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal fastops library")
     set(MISSING_INTERNAL_FASTOPS_LIBRARY 1)
 endif()

cmake/find/grpc.cmake

@@ -26,7 +26,7 @@ option(USE_INTERNAL_GRPC_LIBRARY "Set to FALSE to use system gRPC library instea
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/grpc/CMakeLists.txt")
   if(USE_INTERNAL_GRPC_LIBRARY)
-    message(WARNING "submodule contrib/grpc is missing. to fix try run: \n git submodule update --init --recursive")
+    message(WARNING "submodule contrib/grpc is missing. to fix try run: \n git submodule update --init")
     message(${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal grpc")
     set(USE_INTERNAL_GRPC_LIBRARY 0)
   endif()

cmake/find/gtest.cmake

@@ -4,7 +4,7 @@ option (USE_INTERNAL_GTEST_LIBRARY "Set to FALSE to use system Google Test inste
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest/CMakeLists.txt")
    if (USE_INTERNAL_GTEST_LIBRARY)
-       message (WARNING "submodule contrib/googletest is missing. to fix try run: \n git submodule update --init --recursive")
+       message (WARNING "submodule contrib/googletest is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal gtest")
        set (USE_INTERNAL_GTEST_LIBRARY 0)
    endif ()

cmake/find/h3.cmake

@@ -11,7 +11,7 @@ option(USE_INTERNAL_H3_LIBRARY "Set to FALSE to use system h3 library instead of
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib/include/h3Index.h")
     if(USE_INTERNAL_H3_LIBRARY)
-        message(WARNING "submodule contrib/h3 is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/h3 is missing. to fix try run: \n git submodule update --init")
         message(${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal h3 library")
         set(USE_INTERNAL_H3_LIBRARY 0)
     endif()

cmake/find/hdfs3.cmake

@@ -16,7 +16,7 @@ option(USE_INTERNAL_HDFS3_LIBRARY "Set to FALSE to use system HDFS3 instead of b
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/hdfs.h")
     if(USE_INTERNAL_HDFS3_LIBRARY)
-        message(WARNING "submodule contrib/libhdfs3 is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/libhdfs3 is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal HDFS3 library")
         set(USE_INTERNAL_HDFS3_LIBRARY 0)
     endif()

cmake/find/icu.cmake

@@ -16,7 +16,7 @@ option (USE_INTERNAL_ICU_LIBRARY "Set to FALSE to use system ICU library instead
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/icu/icu4c/LICENSE")
     if (USE_INTERNAL_ICU_LIBRARY)
-        message (WARNING "submodule contrib/icu is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/icu is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal ICU")
         set (USE_INTERNAL_ICU_LIBRARY 0)
     endif ()

cmake/find/krb5.cmake

@@ -1,7 +1,7 @@
 OPTION(ENABLE_KRB5 "Enable krb5" ${ENABLE_LIBRARIES})
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/krb5/README")
-    message (WARNING "submodule contrib/krb5 is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/krb5 is missing. to fix try run: \n git submodule update --init")
     set (ENABLE_KRB5 0)
 endif ()
 

cmake/find/ldap.cmake

@@ -15,7 +15,7 @@ option (USE_INTERNAL_LDAP_LIBRARY "Set to FALSE to use system *LDAP library inst
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/openldap/README")
     if (USE_INTERNAL_LDAP_LIBRARY)
-        message (WARNING "Submodule contrib/openldap is missing. To fix try running:\n git submodule update --init --recursive")
+        message (WARNING "Submodule contrib/openldap is missing. To fix try running:\n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal LDAP library")
     endif ()
 

cmake/find/libgsasl.cmake

@@ -16,7 +16,7 @@ endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src/gsasl.h")
     if (USE_INTERNAL_LIBGSASL_LIBRARY)
-        message (WARNING "submodule contrib/libgsasl is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/libgsasl is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libgsasl")
         set (USE_INTERNAL_LIBGSASL_LIBRARY 0)
     endif ()

cmake/find/libpqxx.cmake

@@ -5,14 +5,14 @@ if (NOT ENABLE_LIBPQXX)
 endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libpqxx/CMakeLists.txt")
-    message (WARNING "submodule contrib/libpqxx is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/libpqxx is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libpqxx library")
     set (USE_LIBPQXX 0)
     return()
 endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libpq/include")
-    message (ERROR "submodule contrib/libpq is missing. to fix try run: \n git submodule update --init --recursive")
+    message (ERROR "submodule contrib/libpq is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libpq needed for libpqxx")
     set (USE_LIBPQXX 0)
     return()

cmake/find/libprotobuf-mutator.cmake

@@ -0,0 +1,11 @@
+option(USE_LIBPROTOBUF_MUTATOR "Enable libprotobuf-mutator" ${ENABLE_FUZZING})
+
+if (NOT USE_LIBPROTOBUF_MUTATOR)
+    return()
+endif()
+
+set(LibProtobufMutator_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libprotobuf-mutator")
+
+if (NOT EXISTS "${LibProtobufMutator_SOURCE_DIR}/README.md")
+    message (ERROR "submodule contrib/libprotobuf-mutator is missing. to fix try run: \n git submodule update --init")
+endif()

cmake/find/libuv.cmake

@@ -5,7 +5,7 @@ if (OS_DARWIN AND COMPILER_GCC)
 endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libuv")
-    message (WARNING "submodule contrib/libuv is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/libuv is missing. to fix try run: \n git submodule update --init")
     SET(MISSING_INTERNAL_LIBUV_LIBRARY 1)
     return()
 endif()

cmake/find/libxml2.cmake

@@ -2,7 +2,7 @@ option (USE_INTERNAL_LIBXML2_LIBRARY "Set to FALSE to use system libxml2 library
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libxml2/libxml.h")
     if (USE_INTERNAL_LIBXML2_LIBRARY)
-        message (WARNING "submodule contrib/libxml2 is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/libxml2 is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libxml")
         set (USE_INTERNAL_LIBXML2_LIBRARY 0)
     endif ()

cmake/find/llvm.cmake

@@ -12,7 +12,7 @@ if (NOT ENABLE_EMBEDDED_COMPILER)
 endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/CMakeLists.txt")
-    message (${RECONFIGURE_MESSAGE_LEVEL} "submodule /contrib/llvm is missing. to fix try run: \n git submodule update --init --recursive")
+    message (${RECONFIGURE_MESSAGE_LEVEL} "submodule /contrib/llvm is missing. to fix try run: \n git submodule update --init")
 endif ()
 
 set (USE_EMBEDDED_COMPILER 1)

cmake/find/msgpack.cmake

@@ -11,7 +11,7 @@ option (USE_INTERNAL_MSGPACK_LIBRARY "Set to FALSE to use system msgpack library
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/msgpack-c/include/msgpack.hpp")
     if(USE_INTERNAL_MSGPACK_LIBRARY)
-        message(WARNING "Submodule contrib/msgpack-c is missing. To fix try run: \n git submodule update --init --recursive")
+        message(WARNING "Submodule contrib/msgpack-c is missing. To fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal msgpack")
         set(USE_INTERNAL_MSGPACK_LIBRARY 0)
     endif()

cmake/find/mysqlclient.cmake

@@ -16,7 +16,7 @@ option(USE_INTERNAL_MYSQL_LIBRARY "Set to FALSE to use system mysqlclient librar
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mariadb-connector-c/README")
     if(USE_INTERNAL_MYSQL_LIBRARY)
-        message(WARNING "submodule contrib/mariadb-connector-c is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/mariadb-connector-c is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal mysql library")
         set(USE_INTERNAL_MYSQL_LIBRARY 0)
     endif()

cmake/find/nanodbc.cmake

@@ -7,7 +7,7 @@ if (NOT USE_INTERNAL_NANODBC_LIBRARY)
 endif ()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/nanodbc/CMakeLists.txt")
-    message (FATAL_ERROR "submodule contrib/nanodbc is missing. to fix try run: \n git submodule update --init --recursive")
+    message (FATAL_ERROR "submodule contrib/nanodbc is missing. to fix try run: \n git submodule update --init")
 endif()
 
 set (NANODBC_LIBRARY nanodbc)

cmake/find/nlp.cmake

@@ -7,21 +7,21 @@ if (NOT ENABLE_NLP)
 endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libstemmer_c/Makefile")
-    message (WARNING "submodule contrib/libstemmer_c is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/libstemmer_c is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libstemmer_c library, NLP functions will be disabled")
     set (USE_NLP 0)
     return()
 endif ()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/wordnet-blast/CMakeLists.txt")
-    message (WARNING "submodule contrib/wordnet-blast is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/wordnet-blast is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal wordnet-blast library, NLP functions will be disabled")
     set (USE_NLP 0)
     return()
 endif ()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/lemmagen-c/README.md")
-    message (WARNING "submodule contrib/lemmagen-c is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/lemmagen-c is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal lemmagen-c library, NLP functions will be disabled")
     set (USE_NLP 0)
     return()

cmake/find/nuraft.cmake

@@ -5,7 +5,7 @@ if (NOT ENABLE_NURAFT)
 endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/NuRaft/CMakeLists.txt")
-    message (WARNING "submodule contrib/NuRaft is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/NuRaft is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal NuRaft library")
     set (USE_NURAFT 0)
     return()

cmake/find/orc.cmake

@@ -18,7 +18,7 @@ include(cmake/find/snappy.cmake)
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include/orc/OrcFile.hh")
    if(USE_INTERNAL_ORC_LIBRARY)
-       message(WARNING "submodule contrib/orc is missing. to fix try run: \n git submodule update --init --recursive")
+       message(WARNING "submodule contrib/orc is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal ORC")
        set(USE_INTERNAL_ORC_LIBRARY 0)
    endif()

cmake/find/parquet.cmake

@@ -20,7 +20,7 @@ endif()
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/CMakeLists.txt")
     if(USE_INTERNAL_PARQUET_LIBRARY)
-        message(WARNING "submodule contrib/arrow (required for Parquet) is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/arrow (required for Parquet) is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal parquet library")
         set(USE_INTERNAL_PARQUET_LIBRARY 0)
     endif()

cmake/find/protobuf.cmake

@@ -15,7 +15,7 @@ option(USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instea
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/protobuf/cmake/CMakeLists.txt")
   if(USE_INTERNAL_PROTOBUF_LIBRARY)
-    message(WARNING "submodule contrib/protobuf is missing. to fix try run: \n git submodule update --init --recursive")
+    message(WARNING "submodule contrib/protobuf is missing. to fix try run: \n git submodule update --init")
     message(${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal protobuf")
     set(USE_INTERNAL_PROTOBUF_LIBRARY 0)
   endif()

cmake/find/rapidjson.cmake

@@ -10,7 +10,7 @@ option(USE_INTERNAL_RAPIDJSON_LIBRARY "Set to FALSE to use system rapidjson libr
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/rapidjson/include/rapidjson/rapidjson.h")
     if(USE_INTERNAL_RAPIDJSON_LIBRARY)
-       message(WARNING "submodule contrib/rapidjson is missing. to fix try run: \n git submodule update --init --recursive")
+       message(WARNING "submodule contrib/rapidjson is missing. to fix try run: \n git submodule update --init")
        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal rapidjson library")
        set(USE_INTERNAL_RAPIDJSON_LIBRARY 0)
     endif()

cmake/find/rdkafka.cmake

@@ -11,7 +11,7 @@ option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka inst
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/CMakeLists.txt")
     if(USE_INTERNAL_RDKAFKA_LIBRARY)
-        message (WARNING "submodule contrib/cppkafka is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/cppkafka is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal cppkafka")
         set (USE_INTERNAL_RDKAFKA_LIBRARY 0)
     endif()
@@ -20,7 +20,7 @@ endif ()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/CMakeLists.txt")
     if(USE_INTERNAL_RDKAFKA_LIBRARY OR MISSING_INTERNAL_CPPKAFKA_LIBRARY)
-        message (WARNING "submodule contrib/librdkafka is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/librdkafka is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal rdkafka")
         set (USE_INTERNAL_RDKAFKA_LIBRARY 0)
     endif()

cmake/find/re2.cmake

@@ -2,7 +2,7 @@ option (USE_INTERNAL_RE2_LIBRARY "Set to FALSE to use system re2 library instead
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/re2/CMakeLists.txt")
     if(USE_INTERNAL_RE2_LIBRARY)
-        message(WARNING "submodule contrib/re2 is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/re2 is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal re2 library")
     endif()
     set(USE_INTERNAL_RE2_LIBRARY 0)

cmake/find/rocksdb.cmake

@@ -15,7 +15,7 @@ option(USE_INTERNAL_ROCKSDB_LIBRARY "Set to FALSE to use system ROCKSDB library
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/rocksdb/CMakeLists.txt")
     if (USE_INTERNAL_ROCKSDB_LIBRARY)
-        message (WARNING "submodule contrib is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib is missing. to fix try run: \n git submodule update --init")
         message(${RECONFIGURE_MESSAGE_LEVEL} "cannot find internal rocksdb")
     endif()
     set (MISSING_INTERNAL_ROCKSDB 1)

cmake/find/s2geometry.cmake

@@ -3,7 +3,7 @@ option(ENABLE_S2_GEOMETRY "Enable S2 geometry library" ${ENABLE_LIBRARIES})
 
 if (ENABLE_S2_GEOMETRY)
     if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/s2geometry")
-        message (WARNING "submodule contrib/s2geometry is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/s2geometry is missing. to fix try run: \n git submodule update --init")
         set (ENABLE_S2_GEOMETRY 0)
         set (USE_S2_GEOMETRY 0)
     else()

cmake/find/s3.cmake

@@ -23,7 +23,7 @@ if (NOT USE_INTERNAL_AWS_S3_LIBRARY)
 endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3")
-    message (WARNING "submodule contrib/aws is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/aws is missing. to fix try run: \n git submodule update --init")
     if (USE_INTERNAL_AWS_S3_LIBRARY)
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal S3 library")
     endif ()

cmake/find/sentry.cmake

@@ -2,7 +2,7 @@ set (SENTRY_LIBRARY "sentry")
 
 set (SENTRY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/sentry-native/include")
 if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h")
-    message (WARNING "submodule contrib/sentry-native is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/sentry-native is missing. to fix try run: \n git submodule update --init")
     if (USE_SENTRY)
          message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal sentry library")
     endif()

cmake/find/simdjson.cmake

@@ -1,7 +1,7 @@
 option (USE_SIMDJSON "Use simdjson" ${ENABLE_LIBRARIES})
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/simdjson/include/simdjson.h")
-    message (WARNING "submodule contrib/simdjson is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/simdjson is missing. to fix try run: \n git submodule update --init")
     if (USE_SIMDJSON)
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal simdjson library")
     endif()

cmake/find/sqlite.cmake

@@ -5,7 +5,7 @@ if (NOT ENABLE_SQLITE)
 endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/sqlite-amalgamation/sqlite3.c")
-    message (WARNING "submodule contrib/sqlite3-amalgamation is missing. to fix try run: \n git submodule update --init --recursive")
+    message (WARNING "submodule contrib/sqlite3-amalgamation is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal sqlite library")
     set (USE_SQLITE 0)
     return()

cmake/find/ssl.cmake

@@ -13,7 +13,7 @@ option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boringssl/README.md")
     if(USE_INTERNAL_SSL_LIBRARY)
-        message(WARNING "submodule contrib/boringssl is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/boringssl is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal ssl library")
     endif()
     set(USE_INTERNAL_SSL_LIBRARY 0)

cmake/find/stats.cmake

@@ -2,11 +2,11 @@ option(ENABLE_STATS "Enable StatsLib library" ${ENABLE_LIBRARIES})
 
 if (ENABLE_STATS)
     if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/stats")
-        message (WARNING "submodule contrib/stats is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/stats is missing. to fix try run: \n git submodule update --init")
         set (ENABLE_STATS 0)
         set (USE_STATS 0)
     elseif (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/gcem")
-        message (WARNING "submodule contrib/gcem is missing. to fix try run: \n git submodule update --init --recursive")
+        message (WARNING "submodule contrib/gcem is missing. to fix try run: \n git submodule update --init")
         set (ENABLE_STATS 0)
         set (USE_STATS 0)
     else()

cmake/find/xz.cmake

@@ -2,7 +2,7 @@ option (USE_INTERNAL_XZ_LIBRARY "Set to OFF to use system xz (lzma) library inst
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/xz/src/liblzma/api/lzma.h")
     if(USE_INTERNAL_XZ_LIBRARY)
-        message(WARNING "submodule contrib/xz is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/xz is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal xz (lzma) library")
         set(USE_INTERNAL_XZ_LIBRARY 0)
     endif()

cmake/find/yaml-cpp.cmake

@@ -5,5 +5,5 @@ if (NOT USE_YAML_CPP)
 endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/yaml-cpp/README.md")
-    message (ERROR "submodule contrib/yaml-cpp is missing. to fix try run: \n git submodule update --init --recursive")
+    message (ERROR "submodule contrib/yaml-cpp is missing. to fix try run: \n git submodule update --init")
 endif()

cmake/find/zlib.cmake

@@ -12,7 +12,7 @@ endif ()
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}/zlib.h")
     if(USE_INTERNAL_ZLIB_LIBRARY)
-        message(WARNING "submodule contrib/${INTERNAL_ZLIB_NAME} is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/${INTERNAL_ZLIB_NAME} is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal zlib library")
     endif()
     set(USE_INTERNAL_ZLIB_LIBRARY 0)

cmake/find/zstd.cmake

@@ -2,7 +2,7 @@ option (USE_INTERNAL_ZSTD_LIBRARY "Set to FALSE to use system zstd library inste
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/zstd/lib/zstd.h")
     if(USE_INTERNAL_ZSTD_LIBRARY)
-        message(WARNING "submodule contrib/zstd is missing. to fix try run: \n git submodule update --init --recursive")
+        message(WARNING "submodule contrib/zstd is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal zstd library")
         set(USE_INTERNAL_ZSTD_LIBRARY 0)
     endif()

cmake/linux/default_libs.cmake

@@ -14,6 +14,8 @@ endif ()
 if (OS_ANDROID)
     # pthread and rt are included in libc
     set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -ldl")
+elseif (USE_MUSL)
+    set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -static -lc")
 else ()
     set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread -ldl")
 endif ()
@@ -26,7 +28,7 @@ set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
 # glibc-compatibility library relies to constant version of libc headers
 # (because minor changes in function attributes between different glibc versions will introduce incompatibilities)
 # This is for x86_64. For other architectures we have separate toolchains.
-if (ARCH_AMD64 AND NOT_UNBUNDLED)
+if (ARCH_AMD64 AND NOT_UNBUNDLED AND NOT CMAKE_CROSSCOMPILING)
     set(CMAKE_C_STANDARD_INCLUDE_DIRECTORIES ${ClickHouse_SOURCE_DIR}/contrib/libc-headers/x86_64-linux-gnu ${ClickHouse_SOURCE_DIR}/contrib/libc-headers)
     set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES ${ClickHouse_SOURCE_DIR}/contrib/libc-headers/x86_64-linux-gnu ${ClickHouse_SOURCE_DIR}/contrib/libc-headers)
 endif ()
@@ -37,8 +39,10 @@ set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
 if (NOT OS_ANDROID)
-    # Our compatibility layer doesn't build under Android, many errors in musl.
-    add_subdirectory(base/glibc-compatibility)
+    if (NOT USE_MUSL)
+        # Our compatibility layer doesn't build under Android, many errors in musl.
+        add_subdirectory(base/glibc-compatibility)
+    endif ()
     add_subdirectory(base/harmful)
 endif ()
 

cmake/linux/toolchain-riscv64.cmake

@@ -0,0 +1,32 @@
+set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+
+set (CMAKE_SYSTEM_NAME "Linux")
+set (CMAKE_SYSTEM_PROCESSOR "riscv64")
+set (CMAKE_C_COMPILER_TARGET "riscv64-linux-gnu")
+set (CMAKE_CXX_COMPILER_TARGET "riscv64-linux-gnu")
+set (CMAKE_ASM_COMPILER_TARGET "riscv64-linux-gnu")
+
+set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-riscv64")
+
+set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}")
+
+find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
+find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
+
+set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
+set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
+
+set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+
+set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
+
+set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
+set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
+
+set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
+
+set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

cmake/linux/toolchain-x86_64-musl.cmake

@@ -0,0 +1,35 @@
+set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+
+set (CMAKE_SYSTEM_NAME "Linux")
+set (CMAKE_SYSTEM_PROCESSOR "x86_64")
+set (CMAKE_C_COMPILER_TARGET "x86_64-linux-musl")
+set (CMAKE_CXX_COMPILER_TARGET "x86_64-linux-musl")
+set (CMAKE_ASM_COMPILER_TARGET "x86_64-linux-musl")
+
+set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-x86_64-musl")
+
+set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}")
+
+find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
+find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9")
+
+set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE)
+set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE)
+
+set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+
+set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
+
+set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
+set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
+
+set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
+
+set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
+
+set (USE_MUSL 1)
+add_definitions(-DUSE_MUSL=1)

contrib/CMakeLists.txt

@@ -49,6 +49,10 @@ add_subdirectory (replxx-cmake)
 add_subdirectory (unixodbc-cmake)
 add_subdirectory (nanodbc-cmake)
 
+if (ENABLE_FUZZING)
+    add_subdirectory (libprotobuf-mutator-cmake)
+endif()
+
 if (USE_YAML_CPP)
     add_subdirectory (yaml-cpp-cmake)
 endif()

contrib/boringssl

@@ -1 +1 @@
-Subproject commit a6a2e2ab3e44d97ce98e51c558e989f211de7eb3
+Subproject commit c1e01a441d6db234f4f12e63a7657d1f9e6db9c1

contrib/boringssl-cmake/CMakeLists.txt

@@ -4,7 +4,7 @@
 
 # This file is created by generate_build_files.py and edited accordingly.
 
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.5)
 
 project(BoringSSL LANGUAGES C CXX)
 
@@ -20,12 +20,7 @@ if(CMAKE_COMPILER_IS_GNUCXX OR CLANG)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
   endif()
 
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-common")
-  if((CMAKE_C_COMPILER_VERSION VERSION_GREATER "4.8.99") OR CLANG)
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11")
-  else()
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
-  endif()
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-common -std=c11")
 endif()
 
 # pthread_rwlock_t requires a feature flag.
@@ -55,7 +50,7 @@ add_definitions(-DBORINGSSL_IMPLEMENTATION)
 # builds.
 if(NOT OPENSSL_NO_ASM AND CMAKE_OSX_ARCHITECTURES)
   list(LENGTH CMAKE_OSX_ARCHITECTURES NUM_ARCHES)
-  if(NOT ${NUM_ARCHES} EQUAL 1)
+  if(NOT NUM_ARCHES EQUAL 1)
     message(FATAL_ERROR "Universal binaries not supported.")
   endif()
   list(GET CMAKE_OSX_ARCHITECTURES 0 CMAKE_SYSTEM_PROCESSOR)
@@ -78,7 +73,13 @@ elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "AMD64")
 elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86")
   set(ARCH "x86")
 elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i386")
-  set(ARCH "x86")
+  # cmake uses `uname -p` to set the system processor, but Solaris
+  # systems support multiple architectures.
+  if((${CMAKE_SYSTEM_NAME} STREQUAL "SunOS") AND CMAKE_SIZEOF_VOID_P EQUAL 8)
+    set(ARCH "x86_64")
+  else()
+    set(ARCH "x86")
+  endif()
 elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "i686")
   set(ARCH "x86")
 elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64")
@@ -289,6 +290,21 @@ set(
   mac-x86_64/crypto/test/trampoline-x86_64.S
 )
 
+set(
+  CRYPTO_win_aarch64_SOURCES
+
+  win-aarch64/crypto/chacha/chacha-armv8.S
+  win-aarch64/crypto/fipsmodule/aesv8-armx64.S
+  win-aarch64/crypto/fipsmodule/armv8-mont.S
+  win-aarch64/crypto/fipsmodule/ghash-neon-armv8.S
+  win-aarch64/crypto/fipsmodule/ghashv8-armx64.S
+  win-aarch64/crypto/fipsmodule/sha1-armv8.S
+  win-aarch64/crypto/fipsmodule/sha256-armv8.S
+  win-aarch64/crypto/fipsmodule/sha512-armv8.S
+  win-aarch64/crypto/fipsmodule/vpaes-armv8.S
+  win-aarch64/crypto/test/trampoline-armv8.S
+)
+
 set(
   CRYPTO_win_x86_SOURCES
 
@@ -331,9 +347,9 @@ set(
   win-x86_64/crypto/test/trampoline-x86_64.asm
 )
 
-if(APPLE AND ${ARCH} STREQUAL "aarch64")
+if(APPLE AND ARCH STREQUAL "aarch64")
   set(CRYPTO_ARCH_SOURCES ${CRYPTO_ios_aarch64_SOURCES})
-elseif(APPLE AND ${ARCH} STREQUAL "arm")
+elseif(APPLE AND ARCH STREQUAL "arm")
   set(CRYPTO_ARCH_SOURCES ${CRYPTO_ios_arm_SOURCES})
 elseif(APPLE)
   set(CRYPTO_ARCH_SOURCES ${CRYPTO_mac_${ARCH}_SOURCES})
@@ -360,6 +376,7 @@ add_library(
   "${BORINGSSL_SOURCE_DIR}/crypto/asn1/a_object.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/asn1/a_octet.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/asn1/a_print.c"
+  "${BORINGSSL_SOURCE_DIR}/crypto/asn1/a_strex.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/asn1/a_strnid.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/asn1/a_time.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/asn1/a_type.c"
@@ -389,6 +406,7 @@ add_library(
   "${BORINGSSL_SOURCE_DIR}/crypto/bio/printf.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/bio/socket.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/bio/socket_helper.c"
+  "${BORINGSSL_SOURCE_DIR}/crypto/blake2/blake2.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/bn_extra/bn_asn1.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/bn_extra/convert.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/buf/buf.c"
@@ -413,6 +431,7 @@ add_library(
   "${BORINGSSL_SOURCE_DIR}/crypto/conf/conf.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/cpu-aarch64-fuchsia.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/cpu-aarch64-linux.c"
+  "${BORINGSSL_SOURCE_DIR}/crypto/cpu-aarch64-win.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/cpu-arm-linux.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/cpu-arm.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/cpu-intel.c"
@@ -452,7 +471,6 @@ add_library(
   "${BORINGSSL_SOURCE_DIR}/crypto/ex_data.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/fipsmodule/bcm.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/fipsmodule/fips_shared_support.c"
-  "${BORINGSSL_SOURCE_DIR}/crypto/fipsmodule/is_fips.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/hkdf/hkdf.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/hpke/hpke.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/hrss/hrss.c"
@@ -499,13 +517,13 @@ add_library(
   "${BORINGSSL_SOURCE_DIR}/crypto/trust_token/voprf.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/a_digest.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/a_sign.c"
-  "${BORINGSSL_SOURCE_DIR}/crypto/x509/a_strex.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/a_verify.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/algorithm.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/asn1_gen.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/by_dir.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/by_file.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/i2d_pr.c"
+  "${BORINGSSL_SOURCE_DIR}/crypto/x509/name_print.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/rsa_pss.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/t_crl.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/t_req.c"
@@ -519,7 +537,6 @@ add_library(
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/x509_ext.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/x509_lu.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/x509_obj.c"
-  "${BORINGSSL_SOURCE_DIR}/crypto/x509/x509_r2x.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/x509_req.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/x509_set.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509/x509_trs.c"
@@ -589,6 +606,8 @@ add_library(
   "${BORINGSSL_SOURCE_DIR}/ssl/d1_srtp.cc"
   "${BORINGSSL_SOURCE_DIR}/ssl/dtls_method.cc"
   "${BORINGSSL_SOURCE_DIR}/ssl/dtls_record.cc"
+  "${BORINGSSL_SOURCE_DIR}/ssl/encrypted_client_hello.cc"
+  "${BORINGSSL_SOURCE_DIR}/ssl/extensions.cc"
   "${BORINGSSL_SOURCE_DIR}/ssl/handoff.cc"
   "${BORINGSSL_SOURCE_DIR}/ssl/handshake.cc"
   "${BORINGSSL_SOURCE_DIR}/ssl/handshake_client.cc"
@@ -611,7 +630,6 @@ add_library(
   "${BORINGSSL_SOURCE_DIR}/ssl/ssl_versions.cc"
   "${BORINGSSL_SOURCE_DIR}/ssl/ssl_x509.cc"
   "${BORINGSSL_SOURCE_DIR}/ssl/t1_enc.cc"
-  "${BORINGSSL_SOURCE_DIR}/ssl/t1_lib.cc"
   "${BORINGSSL_SOURCE_DIR}/ssl/tls13_both.cc"
   "${BORINGSSL_SOURCE_DIR}/ssl/tls13_client.cc"
   "${BORINGSSL_SOURCE_DIR}/ssl/tls13_enc.cc"
@@ -633,6 +651,7 @@ add_executable(
   "${BORINGSSL_SOURCE_DIR}/tool/digest.cc"
   "${BORINGSSL_SOURCE_DIR}/tool/fd.cc"
   "${BORINGSSL_SOURCE_DIR}/tool/file.cc"
+  "${BORINGSSL_SOURCE_DIR}/tool/generate_ech.cc"
   "${BORINGSSL_SOURCE_DIR}/tool/generate_ed25519.cc"
   "${BORINGSSL_SOURCE_DIR}/tool/genrsa.cc"
   "${BORINGSSL_SOURCE_DIR}/tool/pkcs12.cc"

contrib/fastops

@@ -1 +1 @@
-Subproject commit 012b777df9e2d145a24800a6c8c3d4a0249bb09e
+Subproject commit 1460583af7d13c0e980ce46aec8ee9400314669a

contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h

@@ -18,8 +18,10 @@
  * Define overrides for non-standard allocator-related functions if they are
  * present on the system.
  */
-#define JEMALLOC_OVERRIDE_MEMALIGN
-#define JEMALLOC_OVERRIDE_VALLOC
+#if !defined(USE_MUSL)
+    #define JEMALLOC_OVERRIDE_MEMALIGN
+    #define JEMALLOC_OVERRIDE_VALLOC
+#endif
 
 /*
  * At least Linux omits the "const" in:

contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h

@@ -1,6 +1,6 @@
 // OSX does not have this for system alloc functions, so you will get
 // "exception specification in declaration" error.
-#if defined(__APPLE__) || defined(__FreeBSD__)
+#if defined(__APPLE__) || defined(__FreeBSD__) || defined(USE_MUSL)
 # undef JEMALLOC_NOTHROW
 # define JEMALLOC_NOTHROW
 

contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -13,12 +13,14 @@
  * Define overrides for non-standard allocator-related functions if they are
  * present on the system.
  */
-#define JEMALLOC_OVERRIDE___LIBC_CALLOC
-#define JEMALLOC_OVERRIDE___LIBC_FREE
-#define JEMALLOC_OVERRIDE___LIBC_MALLOC
-#define JEMALLOC_OVERRIDE___LIBC_MEMALIGN
-#define JEMALLOC_OVERRIDE___LIBC_REALLOC
-#define JEMALLOC_OVERRIDE___LIBC_VALLOC
+#if !defined(USE_MUSL)
+    #define JEMALLOC_OVERRIDE___LIBC_CALLOC
+    #define JEMALLOC_OVERRIDE___LIBC_FREE
+    #define JEMALLOC_OVERRIDE___LIBC_MALLOC
+    #define JEMALLOC_OVERRIDE___LIBC_MEMALIGN
+    #define JEMALLOC_OVERRIDE___LIBC_REALLOC
+    #define JEMALLOC_OVERRIDE___LIBC_VALLOC
+#endif
 /* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */
 
 /*

contrib/libcxx-cmake/CMakeLists.txt

@@ -56,6 +56,10 @@ if (USE_UNWIND)
     target_compile_definitions(cxx PUBLIC -DSTD_EXCEPTION_HAS_STACK_TRACE=1)
 endif ()
 
+if (USE_MUSL)
+    target_compile_definitions(cxx PUBLIC -D_LIBCPP_HAS_MUSL_LIBC=1)
+endif ()
+
 # Override the deduced attribute support that causes error.
 if (OS_DARWIN AND COMPILER_GCC)
     add_compile_definitions(_LIBCPP_INIT_PRIORITY_MAX)

contrib/libprotobuf-mutator

@@ -0,0 +1 @@
+Subproject commit ffd86a32874e5c08a143019aad1aaf0907294c9f

contrib/libprotobuf-mutator-cmake/CMakeLists.txt

@@ -0,0 +1,14 @@
+set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/libprotobuf-mutator)
+
+add_library(protobuf-mutator
+    ${LIBRARY_DIR}/src/libfuzzer/libfuzzer_macro.cc
+    ${LIBRARY_DIR}/src/libfuzzer/libfuzzer_mutator.cc
+    ${LIBRARY_DIR}/src/binary_format.cc
+    ${LIBRARY_DIR}/src/mutator.cc
+    ${LIBRARY_DIR}/src/text_format.cc
+    ${LIBRARY_DIR}/src/utf8_fix.cc)
+
+target_include_directories(protobuf-mutator BEFORE PRIVATE "${LIBRARY_DIR}")
+target_include_directories(protobuf-mutator BEFORE PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/protobuf/src")
+
+target_link_libraries(protobuf-mutator ${Protobuf_LIBRARY})

contrib/openldap-cmake/linux_x86_64/include/portable.h

@@ -98,7 +98,9 @@
 #define HAVE_BCOPY 1
 
 /* Define to 1 if you have the <bits/types.h> header file. */
-#define HAVE_BITS_TYPES_H 1
+#if !defined(USE_MUSL)
+    #define HAVE_BITS_TYPES_H 1
+#endif
 
 /* Define to 1 if you have the `chroot' function. */
 #define HAVE_CHROOT 1

contrib/sysroot

@@ -1 +1 @@
-Subproject commit 002415524b5d14124bb8a61a3ce7ac65774f5479
+Subproject commit 6172893931e19b028f9cabb7095a44361be863df

docker/test/fasttest/run.sh

@@ -189,7 +189,7 @@ function clone_submodules
         )
 
         git submodule sync
-        git submodule update --depth 1 --init --recursive "${SUBMODULES_TO_UPDATE[@]}"
+        git submodule update --depth 1 --init "${SUBMODULES_TO_UPDATE[@]}"
         git submodule foreach git reset --hard
         git submodule foreach git checkout @ -f
         git submodule foreach git clean -xfd

docs/_includes/install/universal.sh

@@ -47,13 +47,17 @@ then
 fi
 
 URL="https://builds.clickhouse.com/master/${DIR}/clickhouse"
+echo
 echo "Will download ${URL}"
+echo
 curl -O "${URL}" && chmod a+x clickhouse &&
+echo
 echo "Successfully downloaded the ClickHouse binary, you can run it as:
     ./clickhouse"
 
 if [ "${OS}" = "Linux" ]
 then
+    echo
     echo "You can also install it:
     sudo ./clickhouse install"
 fi

docs/en/development/developer-instruction.md

@@ -37,7 +37,7 @@ Next, you need to download the source files onto your working machine. This is c
 
 In the command line terminal run:
 
-    git clone --recursive git@github.com:your_github_username/ClickHouse.git
+    git clone git@github.com:your_github_username/ClickHouse.git
     cd ClickHouse
 
 Note: please, substitute *your_github_username* with what is appropriate!
@@ -79,7 +79,7 @@ After successfully running this command you will be able to pull updates from th
 
 Working with submodules in git could be painful. Next commands will help to manage it:
 
-    # ! each command accepts --recursive
+    # ! each command accepts
     # Update remote URLs for submodules. Barely rare case
     git submodule sync
     # Add new submodules
@@ -92,16 +92,16 @@ Working with submodules in git could be painful. Next commands will help to mana
 The next commands would help you to reset all submodules to the initial state (!WARNING! - any changes inside will be deleted):
 
     # Synchronizes submodules' remote URL with .gitmodules
-    git submodule sync --recursive
+    git submodule sync
     # Update the registered submodules with initialize not yet initialized
-    git submodule update --init --recursive
+    git submodule update --init
     # Reset all changes done after HEAD
     git submodule foreach git reset --hard
     # Clean files from .gitignore
     git submodule foreach git clean -xfd
     # Repeat last 4 commands for all submodule
-    git submodule foreach git submodule sync --recursive
-    git submodule foreach git submodule update --init --recursive
+    git submodule foreach git submodule sync
+    git submodule foreach git submodule update --init
     git submodule foreach git submodule foreach git reset --hard
     git submodule foreach git submodule foreach git clean -xfd
 

docs/en/getting-started/install.md

@@ -27,10 +27,11 @@ It is recommended to use official pre-compiled `deb` packages for Debian or Ubun
 {% include 'install/deb.sh' %}
 ```
 
-If you want to use the most recent version, replace `stable` with `testing` (this is recommended for your testing environments).
+You can replace `stable` with `lts` or `testing` to use different [“release trains”](../faq/operations/production.md) based on your needs.
 
 You can also download and install packages manually from [here](https://repo.clickhouse.com/deb/stable/main/).
 
+
 #### Packages {#packages}
 
 -   `clickhouse-common-static` — Installs ClickHouse compiled binary files.

docs/en/interfaces/cli.md

@@ -127,6 +127,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
 -   `--secure` – If specified, will connect to server over secure connection.
 -   `--history_file` — Path to a file containing command history.
 -   `--param_<name>` — Value for a [query with parameters](#cli-queries-with-parameters).
+-   `--hardware-utilization` — Print hardware utilization information in progress bar.
 
 Since version 20.5, `clickhouse-client` has automatic syntax highlighting (always enabled).
 

docs/en/operations/server-configuration-parameters/settings.md

@@ -71,13 +71,13 @@ If no conditions met for a data part, ClickHouse uses the `lz4` compression.
 
 ## encryption {#server-settings-encryption}
 
-Configures a command to obtain a key to be used by [encryption codecs](../../sql-reference/statements/create/table.md#create-query-encryption-codecs). Key (or keys) should be written in enviroment variables or be set in configuration file.
+Configures a command to obtain a key to be used by [encryption codecs](../../sql-reference/statements/create/table.md#create-query-encryption-codecs). Key (or keys) should be written in environment variables or set in the configuration file.
 
-Keys can be hex or string. Their length must be equal to 16.
+Keys can be hex or string with a length equal to 16 bytes.
 
 **Example**
 
-Load from config:
+Loading from config:
 
 ```xml
 <encryption_codecs>
@@ -88,9 +88,9 @@ Load from config:
 ```
 
 !!! note "NOTE"
-    Storing keys in configuration file is not recommended. It isn't secure. You can move the keys into a separate config file on a secure disk and put a symlink to that config file to `config.d/` folder.
+    Storing keys in the configuration file is not recommended. It isn't secure. You can move the keys into a separate config file on a secure disk and put a symlink to that config file to `config.d/` folder.
 
-Load from config, when key is in hex:
+Loading from config, when the key is in hex:
 
 ```xml
 <encryption_codecs>
@@ -100,7 +100,7 @@ Load from config, when key is in hex:
 </encryption_codecs>
 ```
 
-Load key from environment variable:
+Loading key from the environment variable:
 
 ```xml
 <encryption_codecs>
@@ -110,9 +110,9 @@ Load key from environment variable:
 </encryption_codecs>
 ```
 
-Where `current_key_id` sets the current key for encryption, and all specified keys can be used for decryption.
+Here `current_key_id` sets the current key for encryption, and all specified keys can be used for decryption.
 
-All this methods can be applied for multiple keys:
+Each of these methods can be applied for multiple keys:
 
 ```xml
 <encryption_codecs>
@@ -124,9 +124,9 @@ All this methods can be applied for multiple keys:
 </encryption_codecs>
 ```
 
-Where `current_key_id` shows current key for encryption.
+Here `current_key_id` shows current key for encryption.
 
-Also user can add nonce that must be 12 bytes long (by default encryption and decryption will use nonce consisting of zero bytes):
+Also, users can add nonce that must be 12 bytes long (by default encryption and decryption processes use nonce that consists of zero bytes):
 
 ```xml
 <encryption_codecs>
@@ -146,7 +146,7 @@ Or it can be set in hex:
 </encryption_codecs>
 ```
 
-Everything above can be applied for `aes_256_gcm_siv` (but key must be 32 bytes length).
+Everything mentioned above can be applied for `aes_256_gcm_siv` (but the key must be 32 bytes long).
 
 ## custom_settings_prefixes {#custom_settings_prefixes}
 

docs/en/operations/settings/settings.md

@@ -1808,7 +1808,7 @@ Default value: 1000
 
 ## optimize_skip_unused_shards {#optimize-skip-unused-shards}
 
-Enables or disables skipping of unused shards for [SELECT](../../sql-reference/statements/select/index.md) queries that have sharding key condition in `WHERE/PREWHERE` (assuming that the data is distributed by sharding key, otherwise does nothing).
+Enables or disables skipping of unused shards for [SELECT](../../sql-reference/statements/select/index.md) queries that have sharding key condition in `WHERE/PREWHERE` (assuming that the data is distributed by sharding key, otherwise a query yields incorrect result).
 
 Possible values:
 

docs/en/operations/system-tables/quotas.md

@@ -10,7 +10,7 @@ Columns:
     - `[]` — All users share the same quota.
     - `['user_name']` — Connections with the same user name share the same quota.
     - `['ip_address']` — Connections from the same IP share the same quota.
-    - `['client_key']` — Connections with the same key share the same quota. A key must be explicitly provided by a client. When using [clickhouse-client](../../interfaces/cli.md), pass a key value in the `--quota-key` parameter, or use the `quota_key` parameter in the client configuration file. When using HTTP interface, use the `X-ClickHouse-Quota` header.
+    - `['client_key']` — Connections with the same key share the same quota. A key must be explicitly provided by a client. When using [clickhouse-client](../../interfaces/cli.md), pass a key value in the `--quota_key` parameter, or use the `quota_key` parameter in the client configuration file. When using HTTP interface, use the `X-ClickHouse-Quota` header.
     - `['user_name', 'client_key']` — Connections with the same `client_key` share the same quota. If a key isn’t provided by a client, the qouta is tracked for `user_name`.
     - `['client_key', 'ip_address']` — Connections with the same `client_key` share the same quota. If a key isn’t provided by a client, the qouta is tracked for `ip_address`.
 - `durations` ([Array](../../sql-reference/data-types/array.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Time interval lengths in seconds.

docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md

@@ -155,6 +155,60 @@ Configuration example:
 LAYOUT(COMPLEX_KEY_HASHED())
 ```
 
+### complex_key_sparse_hashed {#complex-key-sparse-hashed}
+
+This type of storage is for use with composite [keys](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md). Similar to `sparse_hashed`.
+
+Configuration example:
+
+``` xml
+<layout>
+  <complex_key_sparse_hashed />
+</layout>
+```
+
+``` sql
+LAYOUT(COMPLEX_KEY_SPARSE_HASHED())
+```
+
+### hashed_array {#dicts-external_dicts_dict_layout-hashed-array}
+
+The dictionary is completely stored in memory. Each attribute is stored in array. Key attribute is stored in the form of hashed table where value is index in attributes array. The dictionary can contain any number of elements with any identifiers In practice, the number of keys can reach tens of millions of items.
+
+All types of sources are supported. When updating, data (from a file or from a table) is read in its entirety.
+
+Configuration example:
+
+``` xml
+<layout>
+  <hashed_array>
+  </hashed_array>
+</layout>
+```
+
+or
+
+``` sql
+LAYOUT(HASHED_ARRAY())
+```
+
+### complex_key_hashed_array {#complex-key-hashed-array}
+
+This type of storage is for use with composite [keys](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md). Similar to `hashed_array`.
+
+Configuration example:
+
+``` xml
+<layout>
+  <complex_key_hashed_array />
+</layout>
+```
+
+``` sql
+LAYOUT(COMPLEX_KEY_HASHED_ARRAY())
+```
+
+
 ### range_hashed {#range-hashed}
 
 The dictionary is stored in memory in the form of a hash table with an ordered array of ranges and their corresponding values.
@@ -368,7 +422,7 @@ Similar to `cache`, but stores data on SSD and index in RAM. All cache dictionar
         <!-- Size of RAM buffer in bytes for aggregating elements before flushing to SSD. -->
         <write_buffer_size>1048576</write_buffer_size>
         <!-- Path where cache file will be stored. -->
-        <path>/var/lib/clickhouse/clickhouse_dictionaries/test_dict</path>
+        <path>/var/lib/clickhouse/user_files/test_dict</path>
     </ssd_cache>
 </layout>
 ```
@@ -377,7 +431,7 @@ or
 
 ``` sql
 LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 16777216 READ_BUFFER_SIZE 1048576
-    PATH ./user_files/test_dict))
+    PATH '/var/lib/clickhouse/user_files/test_dict'))
 ```
 
 ### complex_key_ssd_cache {#complex-key-ssd-cache}

docs/en/sql-reference/functions/string-functions.md

@@ -313,6 +313,32 @@ SELECT toValidUTF8('\x61\xF0\x80\x80\x80b');
 └───────────────────────┘
 ```
 
+## tokens {#tokens}
+
+Split string into tokens using non-alpha numeric ASCII characters as separators.
+
+**Arguments**
+
+-   `input_string` — Any set of bytes represented as the [String](../../sql-reference/data-types/string.md) data type object.
+
+**Returned value**
+
+-   The resulting array of tokens from input string.
+
+Type: [Array](../data-types/array.md).
+
+**Example**
+
+``` sql
+SELECT tokens('test1,;\\ test2,;\\ test3,;\\   test4') AS tokens;
+```
+
+``` text
+┌─tokens────────────────────────────┐
+│ ['test1','test2','test3','test4'] │
+└───────────────────────────────────┘
+```
+
 ## repeat {#repeat}
 
 Repeats a string as many times as specified and concatenates the replicated values as a single string.

docs/en/sql-reference/functions/type-conversion-functions.md

@@ -279,24 +279,26 @@ Result:
 └──────────────────────────────┴────────────────────┘
 ```
 
-## toDate32OrDefault {#todate32-or-null}
+## toDate32OrDefault {#todate32-or-default}
 
-The same as [toDate32](#todate32) but returns default value if invalid argument is received.
+Converts the argument to the [Date32](../../sql-reference/data-types/date32.md) data type. If the value is outside the range returns the lower border value supported by `Date32`. If the argument has [Date](../../sql-reference/data-types/date.md) type, borders of `Date` are taken into account. Returns default value if invalid argument is received.
 
 **Example**
 
 Query:
 
 ``` sql
-SELECT toDate32OrDefault('1955-01-01'), toDate32OrDefault('');
+SELECT
+    toDate32OrDefault('1930-01-01', toDate32('2020-01-01')),
+    toDate32OrDefault('xx1930-01-01', toDate32('2020-01-01'));
 ```
 
 Result:
 
 ``` text
-┌─toDate32OrDefault('1955-01-01')─┬─toDate32OrDefault('')─┐
-│                      1955-01-01 │            1970-01-01 │
-└─────────────────────────────────┴───────────────────────┘
+┌─toDate32OrDefault('1930-01-01', toDate32('2020-01-01'))─┬─toDate32OrDefault('xx1930-01-01', toDate32('2020-01-01'))─┐
+│                                              1930-01-01 │                                                2020-01-01 │
+└─────────────────────────────────────────────────────────┴───────────────────────────────────────────────────────────┘
 ```
 
 ## toDecimal(32\|64\|128\|256) {#todecimal3264128256}
@@ -362,7 +364,7 @@ Result:
 ```
 
 
-## toDecimal(32\|64\|128\|256)OrDefault {#todecimal3264128256ornull}
+## toDecimal(32\|64\|128\|256)OrDefault {#todecimal3264128256ordefault}
 
 Converts an input string to a [Decimal(P,S)](../../sql-reference/data-types/decimal.md) data type value. This family of functions include:
 
@@ -855,25 +857,25 @@ Result:
 ```
 
 
-## accurateCastOrDefault(x, T[, default_value]) {#type_conversion_function-accurate-cast_or_null}
+## accurateCastOrDefault(x, T[, default_value]) {#type_conversion_function-accurate-cast_or_default}
 
 Converts input value `x` to the specified data type `T`. Returns default type value or `default_value` if specified if the casted value is not representable in the target type.
 
 **Syntax**
 
 ```sql
-accurateCastOrNull(x, T)
+accurateCastOrDefault(x, T)
 ```
 
 **Parameters**
 
 -   `x` — Input value.
 -   `T` — The name of the returned data type.
--   `default_value` - Default value of returned data type.
+-   `default_value` — Default value of returned data type.
 
 **Returned value**
 
--   The value, converted to the specified data type `T`.
+-   The value converted to the specified data type `T`.
 
 **Example**
 

docs/ja/development/developer-instruction.md

@@ -41,7 +41,7 @@ Ubuntuでこれを行うには、コマンドラインターミナルで実行
 
 コマンドラインターミナルで実行:
 
-    git clone --recursive git@github.com:your_github_username/ClickHouse.git
+    git clone git@github.com:your_github_username/ClickHouse.git
     cd ClickHouse
 
 注:、代理して下さい *your_github_username* 適切なもので！
@@ -83,7 +83,7 @@ ClickHouseリポジトリは以下を使用します `submodules`. That is what
 
 Gitでサブモジュールを操作するのは苦痛です。 次のコマンドは管理に役立ちます:
 
-    # ! each command accepts --recursive
+    # ! each command accepts
     # Update remote URLs for submodules. Barely rare case
     git submodule sync
     # Add new submodules
@@ -96,16 +96,16 @@ Gitでサブモジュールを操作するのは苦痛です。 次のコマン
 次のコマンドは、すべてのサブモジュールを初期状態にリセットするのに役立ちます（！ﾂづﾂつｷﾂ。 -内部の変更は削除されます):
 
     # Synchronizes submodules' remote URL with .gitmodules
-    git submodule sync --recursive
+    git submodule sync
     # Update the registered submodules with initialize not yet initialized
-    git submodule update --init --recursive
+    git submodule update --init
     # Reset all changes done after HEAD
     git submodule foreach git reset --hard
     # Clean files from .gitignore
     git submodule foreach git clean -xfd
     # Repeat last 4 commands for all submodule
-    git submodule foreach git submodule sync --recursive
-    git submodule foreach git submodule update --init --recursive
+    git submodule foreach git submodule sync
+    git submodule foreach git submodule update --init
     git submodule foreach git submodule foreach git reset --hard
     git submodule foreach git submodule foreach git clean -xfd
 

docs/ru/development/developer-instruction.md

@@ -40,7 +40,7 @@ ClickHouse не работает и не собирается на 32-битны
 
 Выполните в терминале:
 
-    git clone --recursive git@github.com:ClickHouse/ClickHouse.git
+    git clone git@github.com:ClickHouse/ClickHouse.git
     cd ClickHouse
 
 Замените первое вхождение слова `ClickHouse` в команде для git на имя вашего аккаунта на GitHub.
@@ -82,7 +82,7 @@ ClickHouse не работает и не собирается на 32-битны
 
 Работа с сабмодулями git может быть достаточно болезненной. Следующие команды позволят содержать их в порядке:
 
-    # ! Каждая команда принимает аргумент --recursive
+    # ! Каждая команда принимает аргумент
     # Обновить URLs удалённого репозитория для каждого сабмодуля, используется относительно редко
     git submodule sync
     # Добавить новые сабмодули
@@ -96,16 +96,16 @@ ClickHouse не работает и не собирается на 32-битны
 
     # Synchronizes submodules' remote URL with .gitmodules
     # Обновить URLs удалённого репозитория для каждого сабмодуля
-    git submodule sync --recursive
+    git submodule sync
     # Обновить существующие модули и добавить отсутствующие
-    git submodule update --init --recursive
+    git submodule update --init
     # Удалить все изменения в сабмодуле относительно HEAD
     git submodule foreach git reset --hard
     # Очистить игнорируемые файлы
     git submodule foreach git clean -xfd
     # Повторить последние 4 команды для каждого из сабмодулей
-    git submodule foreach git submodule sync --recursive
-    git submodule foreach git submodule update --init --recursive
+    git submodule foreach git submodule sync
+    git submodule foreach git submodule update --init
     git submodule foreach git submodule foreach git reset --hard
     git submodule foreach git submodule foreach git clean -xfd
 

docs/ru/interfaces/formats.md

@@ -73,7 +73,7 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT
 Формат `TabSeparated` поддерживает вывод тотальных значений (при использовании WITH TOTALS) и экстремальных значений (при настройке extremes выставленной в 1). В этих случаях, после основных данных выводятся тотальные значения, и экстремальные значения. Основной результат, тотальные значения и экстремальные значения, отделяются друг от друга пустой строкой. Пример:
 
 ``` sql
-SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT TabSeparated``
+SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT TabSeparated
 ```
 
 ``` text

docs/ru/operations/server-configuration-parameters/settings.md

@@ -71,9 +71,9 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
 
 ## encryption {#server-settings-encryption}
 
-Настраивает комманду для получения ключа используемого [кодеками шифрования](../../sql-reference/statements/create/table.md#create-query-encryption-codecs). Ключ (или несколько ключей) должны быть записаны в переменные окружения или установлены в конфигурационном файле .
+Настраивает команду для получения ключа, используемого [кодеками шифрования](../../sql-reference/statements/create/table.md#create-query-encryption-codecs). Ключ (или несколько ключей) должен быть записан в переменные окружения или установлен в конфигурационном файле.
 
-Ключи могут быть представлены в шестнадцатеричной или строковой форме. Их длинна должна быть равна 16.
+Ключи могут быть представлены в шестнадцатеричной или строковой форме. Их длина должна быть равна 16 байтам.
 
 **Пример**
 
@@ -146,7 +146,7 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
 </encryption_codecs>
 ```
 
-Все вышеперечисленное можно применить также для алгоритма `aes_256_gcm_siv` (но ключ должен быть длинной 32 байта).
+Всё вышеперечисленное также применимо для алгоритма `aes_256_gcm_siv` (но ключ должен быть длиной 32 байта).
 
 ## custom_settings_prefixes {#custom_settings_prefixes}
 

docs/ru/operations/settings/settings.md

@@ -1707,7 +1707,7 @@ ClickHouse генерирует исключение
 
 ## optimize_skip_unused_shards {#optimize-skip-unused-shards}
 
-Включает или отключает пропуск неиспользуемых шардов для запросов [SELECT](../../sql-reference/statements/select/index.md) , в которых условие ключа шардирования задано в секции `WHERE/PREWHERE`. Предполагается, что данные распределены с помощью ключа шардирования, в противном случае настройка ничего не делает.
+Включает или отключает пропуск неиспользуемых шардов для запросов [SELECT](../../sql-reference/statements/select/index.md) , в которых условие ключа шардирования задано в секции `WHERE/PREWHERE`. Предполагается, что данные распределены с помощью ключа шардирования, в противном случае запрос выдаст неверный результат.
 
 Возможные значения:
 

docs/ru/operations/system-tables/quotas.md

@@ -11,7 +11,7 @@
     -   `[]` — Все пользователи используют одну и ту же квоту.
     -   `['user_name']` — Соединения с одинаковым именем пользователя используют одну и ту же квоту.
     -   `['ip_address']` — Соединения с одинаковым IP-адресом используют одну и ту же квоту.
-    -   `['client_key']` — Соединения с одинаковым ключом используют одну и ту же квоту. Ключ может быть явно задан клиентом. При использовании [clickhouse-client](../../interfaces/cli.md), передайте ключевое значение в параметре `--quota-key`, или используйте параметр `quota_key` файле настроек клиента. В случае использования HTTP интерфейса, используйте заголовок `X-ClickHouse-Quota`.
+    -   `['client_key']` — Соединения с одинаковым ключом используют одну и ту же квоту. Ключ может быть явно задан клиентом. При использовании [clickhouse-client](../../interfaces/cli.md), передайте ключевое значение в параметре `--quota_key`, или используйте параметр `quota_key` файле настроек клиента. В случае использования HTTP интерфейса, используйте заголовок `X-ClickHouse-Quota`.
     -   `['user_name', 'client_key']` — Соединения с одинаковым ключом используют одну и ту же квоту. Если ключ не предоставлен клиентом, то квота отслеживается для `user_name`.
     -   `['client_key', 'ip_address']` — Соединения с одинаковым ключом используют одну и ту же квоту. Если ключ не предоставлен клиентом, то квота отслеживается для `ip_address`.
 -   `durations` ([Array](../../sql-reference/data-types/array.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Длины временных интервалов для расчета потребления ресурсов, в секундах.

docs/ru/sql-reference/data-types/simpleaggregatefunction.md

@@ -21,8 +21,6 @@
 -   [`sumMap`](../../sql-reference/aggregate-functions/reference/summap.md#agg_functions-summap)
 -   [`minMap`](../../sql-reference/aggregate-functions/reference/minmap.md#agg_functions-minmap)
 -   [`maxMap`](../../sql-reference/aggregate-functions/reference/maxmap.md#agg_functions-maxmap)
--   [`argMin`](../../sql-reference/aggregate-functions/reference/argmin.md)
--   [`argMax`](../../sql-reference/aggregate-functions/reference/argmax.md)
 
 !!! note "Примечание"
     Значения `SimpleAggregateFunction(func, Type)` отображаются и хранятся так же, как и `Type`, поэтому комбинаторы [-Merge](../../sql-reference/aggregate-functions/combinators.md#aggregate_functions_combinators-merge) и [-State](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-state) не требуются.

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md

@@ -352,7 +352,7 @@ LAYOUT(CACHE(SIZE_IN_CELLS 1000000000))
         <!-- Size of RAM buffer in bytes for aggregating elements before flushing to SSD. -->
         <write_buffer_size>1048576</write_buffer_size>
         <!-- Path where cache file will be stored. -->
-        <path>/var/lib/clickhouse/clickhouse_dictionaries/test_dict</path>
+        <path>/var/lib/clickhouse/user_files/test_dict</path>
     </ssd_cache>
 </layout>
 ```
@@ -361,7 +361,7 @@ LAYOUT(CACHE(SIZE_IN_CELLS 1000000000))
 
 ``` sql
 LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 16777216 READ_BUFFER_SIZE 1048576
-    PATH ./user_files/test_dict))
+    PATH '/var/lib/clickhouse/user_files/test_dict'))
 ```
 
 ### complex_key_ssd_cache {#complex-key-ssd-cache}

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md

@@ -251,7 +251,7 @@ ClickHouse получает от ODBC-драйвера информацию о
 
 ### Выявленная уязвимость в функционировании ODBC словарей {#vyiavlennaia-uiazvimost-v-funktsionirovanii-odbc-slovarei}
 
-!!! attention "Attention"
+!!! attention "Внимание"
     При соединении с базой данных через ODBC можно заменить параметр соединения `Servername`. В этом случае, значения `USERNAME` и `PASSWORD` из `odbc.ini` отправляются на удаленный сервер и могут быть скомпрометированы.
 
 **Пример небезопасного использования**
@@ -764,7 +764,7 @@ Setting fields:
 -   `port` – Порт для соединения с PostgreSQL. Вы можете указать его для всех реплик или задать индивидуально для каждой релпики (внутри `<replica>`).
 -   `user` – Имя пользователя для соединения с PostgreSQL. Вы можете указать его для всех реплик или задать индивидуально для каждой релпики (внутри `<replica>`).
 -   `password` – Пароль для пользователя PostgreSQL.
--   `replica` – Section of replica configurations. There can be multiple sections.
+-   `replica` – Раздел конфигурации реплик. Может быть несколько.
     - `replica/host` – хост PostgreSQL.
     - `replica/port` – порт PostgreSQL .
     - `replica/priority` – Приоритет реплики. Во время попытки соединения, ClickHouse будет перебирать реплики в порядке приоритет. Меньшее значение означает более высокий приоритет.

docs/ru/sql-reference/functions/date-time-functions.md

@@ -335,7 +335,7 @@ SELECT toStartOfISOYear(toDate('2017-01-01')) AS ISOYear20170101;
 **Синтаксис**
 
 ``` sql
-toStartOfSecond(value[, timezone])
+toStartOfSecond(value, [timezone])
 ```
 
 **Аргументы**

docs/ru/sql-reference/functions/string-functions.md

@@ -34,15 +34,15 @@ empty(x)
 Запрос:
 
 ```sql
-SELECT notempty('text');
+SELECT empty('text');
 ```
 
 Результат:
 
 ```text
-┌─empty('')─┐
-│         1 │
-└───────────┘
+┌─empty('text')─┐
+│             0 │
+└───────────────┘
 ```
 
 ## notEmpty {#notempty}

docs/ru/sql-reference/functions/type-conversion-functions.md

@@ -90,6 +90,26 @@ SELECT toInt64OrNull('123123'), toInt8OrNull('123qwe123');
 └─────────────────────────┴───────────────────────────┘
 ```
 
+## toInt(8\|16\|32\|64\|128\|256)OrDefault {#toint8163264128256orDefault}
+
+Принимает аргумент типа String и пытается его распарсить в Int(8\|16\|32\|64\|128\|256). Если не удалось —  возвращает значение по умолчанию.
+
+**Пример**
+
+Запрос:
+
+``` sql
+SELECT toInt64OrDefault('123123', cast('-1' as Int64)), toInt8OrDefault('123qwe123', cast('-1' as Int8));
+```
+
+Результат:
+
+``` text
+┌─toInt64OrDefault('123123', CAST('-1', 'Int64'))─┬─toInt8OrDefault('123qwe123', CAST('-1', 'Int8'))─┐
+│                                          123123 │                                               -1 │
+└─────────────────────────────────────────────────┴──────────────────────────────────────────────────┘
+```
+
 ## toUInt(8\|16\|32\|64\|256) {#touint8163264}
 
 Преобраует входное значение к типу [UInt](../../sql-reference/functions/type-conversion-functions.md). Семейство функций включает:
@@ -132,12 +152,16 @@ SELECT toUInt64(nan), toUInt32(-32), toUInt16('16'), toUInt8(8.8);
 
 ## toUInt(8\|16\|32\|64\|256)OrNull {#touint8163264ornull}
 
+## toUInt(8\|16\|32\|64\|256)OrDefault {#touint8163264256ordefault}
+
 ## toFloat(32\|64) {#tofloat3264}
 
 ## toFloat(32\|64)OrZero {#tofloat3264orzero}
 
 ## toFloat(32\|64)OrNull {#tofloat3264ornull}
 
+## toFloat(32\|64)OrDefault {#tofloat3264ordefault}
+
 ## toDate {#todate}
 
 Cиноним: `DATE`.
@@ -146,12 +170,16 @@ Cиноним: `DATE`.
 
 ## toDateOrNull {#todateornull}
 
+## toDateOrDefault {#todateordefault}
+
 ## toDateTime {#todatetime}
 
 ## toDateTimeOrZero {#todatetimeorzero}
 
 ## toDateTimeOrNull {#todatetimeornull}
 
+## toDateTimeOrDefault {#todatetimeordefault}
+
 ## toDate32 {#todate32}
 
 Конвертирует аргумент в значение типа [Date32](../../sql-reference/data-types/date32.md). Если значение выходит за границы диапазона, возвращается пограничное значение `Date32`. Если аргумент имеет тип [Date](../../sql-reference/data-types/date.md), учитываются границы типа `Date`.
@@ -250,6 +278,28 @@ SELECT toDate32OrNull('1955-01-01'), toDate32OrNull('');
 └──────────────────────────────┴────────────────────┘
 ```
 
+## toDate32OrDefault {#todate32-or-default}
+
+Конвертирует аргумент в значение типа [Date32](../../sql-reference/data-types/date32.md). Если значение выходит за границы диапазона, возвращается нижнее пограничное значение `Date32`. Если аргумент имеет тип [Date](../../sql-reference/data-types/date.md), учитываются границы типа `Date`. Возвращает значение по умолчанию, если получен недопустимый аргумент.
+
+**Пример**
+
+Запрос:
+
+``` sql
+SELECT
+    toDate32OrDefault('1930-01-01', toDate32('2020-01-01')),
+    toDate32OrDefault('xx1930-01-01', toDate32('2020-01-01'));
+```
+
+Результат:
+
+``` text
+┌─toDate32OrDefault('1930-01-01', toDate32('2020-01-01'))─┬─toDate32OrDefault('xx1930-01-01', toDate32('2020-01-01'))─┐
+│                                              1930-01-01 │                                                2020-01-01 │
+└─────────────────────────────────────────────────────────┴───────────────────────────────────────────────────────────┘
+```
+
 ## toDecimal(32\|64\|128\|256) {#todecimal3264128}
 
 Преобразует `value` к типу данных [Decimal](../../sql-reference/functions/type-conversion-functions.md) с точностью `S`. `value` может быть числом или строкой. Параметр `S` (scale) задаёт число десятичных знаков.
@@ -312,6 +362,59 @@ SELECT toDecimal32OrNull(toString(-1.111), 2) AS val, toTypeName(val);
 └──────┴────────────────────────────────────────────────────┘
 ```
 
+## toDecimal(32\|64\|128\|256)OrDefault {#todecimal3264128256ordefault}
+
+Преобразует входную строку в значение с типом данных [Decimal(P,S)](../../sql-reference/data-types/decimal.md). Семейство функций включает в себя:
+
+-   `toDecimal32OrDefault(expr, S)` — возвращает значение типа `Decimal32(S)`.
+-   `toDecimal64OrDefault(expr, S)` — возвращает значение типа `Decimal64(S)`.
+-   `toDecimal128OrDefault(expr, S)` — возвращает значение типа `Decimal128(S)`.
+-   `toDecimal256OrDefault(expr, S)` — возвращает значение типа `Decimal256(S)`.
+
+Эти функции следует использовать вместо функций `toDecimal*()`, если при ошибке обработки входного значения вы хотите получать значение по умолчанию вместо исключения.
+
+**Аргументы**
+
+-   `expr` — [выражение](../syntax.md#syntax-expressions), возвращающее значение типа [String](../../sql-reference/functions/type-conversion-functions.md). ClickHouse ожидает текстовое представление десятичного числа. Например, `'1.111'`.
+-   `S` — количество десятичных знаков в результирующем значении.
+
+**Возвращаемое значение**
+
+Значение типа `Decimal(P,S)`. Значение содержит:
+
+-   Число с `S` десятичными знаками, если ClickHouse распознал число во входной строке.
+-   Значение по умолчанию типа `Decimal(P,S)`, если ClickHouse не смог распознать число во входной строке или входное число содержит больше чем `S` десятичных знаков.
+
+**Примеры**
+
+Запрос:
+
+``` sql
+SELECT toDecimal32OrDefault(toString(-1.111), 5) AS val, toTypeName(val);
+```
+
+Результат:
+
+``` text
+┌────val─┬─toTypeName(toDecimal32OrDefault(toString(-1.111), 5))─┐
+│ -1.111 │ Decimal(9, 5)                                         │
+└────────┴───────────────────────────────────────────────────────┘
+```
+
+Запрос:
+
+``` sql
+SELECT toDecimal32OrDefault(toString(-1.111), 2) AS val, toTypeName(val);
+```
+
+Результат:
+
+``` text
+┌─val─┬─toTypeName(toDecimal32OrDefault(toString(-1.111), 2))─┐
+│   0 │ Decimal(9, 2)                                         │
+└─────┴───────────────────────────────────────────────────────┘
+```
+
 ## toDecimal(32\|64\|128\|256)OrZero {#todecimal3264128orzero}
 
 Преобразует тип входного значения в [Decimal (P, S)](../../sql-reference/functions/type-conversion-functions.md). Семейство функций включает в себя:
@@ -750,6 +853,63 @@ SELECT
 └───────┴──────┴──────────────┘
 ```
 
+
+## accurateCastOrDefault(x, T[, default_value]) {#type_conversion_function-accurate-cast_or_default}
+
+Преобразует входное значение `x` в указанный тип данных `T`. Если исходное значение не может быть преобразовано к целевому типу, возвращает значение по умолчанию или `default_value`, если оно указано.
+
+**Синтаксис**
+
+```sql
+accurateCastOrDefault(x, T)
+```
+
+**Аргументы**
+
+-   `x` — входное значение.
+-   `T` — имя возвращаемого типа данных.
+-   `default_value` — значение по умолчанию возвращаемого типа данных.
+
+**Возвращаемое значение**
+
+-   Значение, преобразованное в указанный тип `T`.
+
+**Пример**
+
+Запрос:
+
+``` sql
+SELECT toTypeName(accurateCastOrDefault(5, 'UInt8'));
+```
+
+Результат:
+
+``` text
+┌─toTypeName(accurateCastOrDefault(5, 'UInt8'))─┐
+│ UInt8                                         │
+└───────────────────────────────────────────────┘
+```
+
+Запрос:
+
+``` sql
+SELECT
+    accurateCastOrDefault(-1, 'UInt8') as uint8,
+    accurateCastOrDefault(-1, 'UInt8', 5) as uint8_default,
+    accurateCastOrDefault(128, 'Int8') as int8,
+    accurateCastOrDefault(128, 'Int8', 5) as int8_default,
+    accurateCastOrDefault('Test', 'FixedString(2)') as fixed_string,
+    accurateCastOrDefault('Test', 'FixedString(2)', 'Te') as fixed_string_default;
+```
+
+Результат:
+
+``` text
+┌─uint8─┬─uint8_default─┬─int8─┬─int8_default─┬─fixed_string─┬─fixed_string_default─┐
+│     0 │             5 │    0 │            5 │              │ Te                   │
+└───────┴───────────────┴──────┴──────────────┴──────────────┴──────────────────────┘
+```
+
 ## toInterval(Year\|Quarter\|Month\|Week\|Day\|Hour\|Minute\|Second) {#function-tointerval}
 
 Приводит аргумент из числового типа данных к типу данных [IntervalType](../../sql-reference/data-types/special-data-types/interval.md).

docs/zh/development/developer-instruction.md

@@ -29,7 +29,7 @@ ClickHose支持Linux,FreeBSD 及 Mac OS X 系统。
 
 在终端命令行输入下列指令：
 
-    git clone --recursive git@guthub.com:your_github_username/ClickHouse.git
+    git clone git@guthub.com:your_github_username/ClickHouse.git
     cd ClickHouse
 
 请注意，您需要将*your_github_username* 替换成实际使用的账户名!
@@ -71,7 +71,7 @@ ClickHose支持Linux,FreeBSD 及 Mac OS X 系统。
 
 在git中使用子模块可能会很痛苦。 接下来的命令将有助于管理它:
 
-    # ! each command accepts --recursive
+    # ! each command accepts
     # Update remote URLs for submodules. Barely rare case
     git submodule sync
     # Add new submodules
@@ -84,16 +84,16 @@ ClickHose支持Linux,FreeBSD 及 Mac OS X 系统。
 接下来的命令将帮助您将所有子模块重置为初始状态（！华林！ -里面的任何chenges将被删除):
 
     # Synchronizes submodules' remote URL with .gitmodules
-    git submodule sync --recursive
+    git submodule sync
     # Update the registered submodules with initialize not yet initialized
-    git submodule update --init --recursive
+    git submodule update --init
     # Reset all changes done after HEAD
     git submodule foreach git reset --hard
     # Clean files from .gitignore
     git submodule foreach git clean -xfd
     # Repeat last 4 commands for all submodule
-    git submodule foreach git submodule sync --recursive
-    git submodule foreach git submodule update --init --recursive
+    git submodule foreach git submodule sync
+    git submodule foreach git submodule update --init
     git submodule foreach git submodule foreach git reset --hard
     git submodule foreach git submodule foreach git clean -xfd
 

programs/benchmark/Benchmark.cpp

@@ -28,7 +28,7 @@
 #include <IO/ConnectionTimeouts.h>
 #include <IO/ConnectionTimeoutsContext.h>
 #include <IO/UseSSL.h>
-#include <DataStreams/RemoteQueryExecutor.h>
+#include <QueryPipeline/RemoteQueryExecutor.h>
 #include <Interpreters/Context.h>
 #include <Client/Connection.h>
 #include <Common/InterruptListener.h>
@@ -432,7 +432,7 @@ private:
         Progress progress;
         executor.setProgressCallback([&progress](const Progress & value) { progress.incrementPiecewiseAtomically(value); });
 
-        BlockStreamProfileInfo info;
+        ProfileInfo info;
         while (Block block = executor.read())
             info.update(block);
 

programs/client/Client.cpp

@@ -45,8 +45,6 @@
 #include <IO/WriteBufferFromOStream.h>
 #include <IO/UseSSL.h>
 
-#include <DataStreams/NullBlockOutputStream.h>
-
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTDropQuery.h>
 #include <Parsers/ASTSetQuery.h>
@@ -62,7 +60,6 @@
 #include <Functions/registerFunctions.h>
 #include <AggregateFunctions/registerAggregateFunctions.h>
 #include <Formats/registerFormats.h>
-#include <Formats/FormatFactory.h>
 #include "TestTags.h"
 
 #ifndef __clang__