Merge branch 'master' into chenziliang/feature/wildcard-dynamic-columns

2024-11-25 09:02:00 +00:00 · 2021-01-10 04:04:47 +03:00 · 2021-01-10 04:04:47 +03:00 · 7e9120b34f
commit 7e9120b34f
parent aa8d4eec69 8387ac86ef
352 changed files with 5889 additions and 1967 deletions
--- a/.github/ISSUE_TEMPLATE/10_question.md
+++ b/.github/ISSUE_TEMPLATE/10_question.md
@ -1,6 +1,6 @@
 ---
 name: Question
-about: Ask question about ClickHouse
+about: Ask a question about ClickHouse
 title: ''
 labels: question
 assignees: ''
--- a/.github/ISSUE_TEMPLATE/20_feature-request.md
+++ b/.github/ISSUE_TEMPLATE/20_feature-request.md
--- a/.github/ISSUE_TEMPLATE/30_unexpected-behaviour.md
+++ b/.github/ISSUE_TEMPLATE/30_unexpected-behaviour.md
@ -1,6 +1,6 @@
 ---
 name: Unexpected behaviour
-about: Create a report to help us improve ClickHouse
+about: Some feature is working in non-obvious way
 title: ''
 labels: unexpected behaviour
 assignees: ''
--- a/.github/ISSUE_TEMPLATE/35_incomplete_implementation.md
+++ b/.github/ISSUE_TEMPLATE/35_incomplete_implementation.md
@ -0,0 +1,30 @@
+---
+name: Incomplete implementation
+about: Implementation of existing feature is not finished
+title: ''
+labels: unfinished code
+assignees: ''
+
+---
+
+(you don't have to strictly follow this form)
+
+**Describe the unexpected behaviour**
+A clear and concise description of what works not as it is supposed to.
+
+**How to reproduce**
+* Which ClickHouse server version to use
+* Which interface to use, if matters
+* Non-default settings, if any
+* `CREATE TABLE` statements for all tables involved
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Queries to run that lead to unexpected result
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Error message and/or stacktrace**
+If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.
--- a/.github/ISSUE_TEMPLATE/40_bug-report.md
+++ b/.github/ISSUE_TEMPLATE/40_bug-report.md
--- a/.github/ISSUE_TEMPLATE/45_usability-issue.md
+++ b/.github/ISSUE_TEMPLATE/45_usability-issue.md
@ -1,6 +1,6 @@
 ---
 name: Usability issue
-about: Create a report to help us improve ClickHouse
+about: Report something can be made more convenient to use
 title: ''
 labels: usability
 assignees: ''
--- a/.github/ISSUE_TEMPLATE/50_build-issue.md
+++ b/.github/ISSUE_TEMPLATE/50_build-issue.md
--- a/.github/ISSUE_TEMPLATE/60_documentation-issue.md
+++ b/.github/ISSUE_TEMPLATE/60_documentation-issue.md
--- a/.github/ISSUE_TEMPLATE/70_performance-issue.md
+++ b/.github/ISSUE_TEMPLATE/70_performance-issue.md
--- a/.github/ISSUE_TEMPLATE/80_backward-compatibility.md
+++ b/.github/ISSUE_TEMPLATE/80_backward-compatibility.md
@ -1,6 +1,6 @@
 ---
 name: Backward compatibility issue
-about: Create a report to help us improve ClickHouse
+about: Report the case when the behaviour of a new version can break existing use cases
 title: ''
 labels: backward compatibility
 assignees: ''
--- a/.github/ISSUE_TEMPLATE/90_fuzzing-report.md
+++ b/.github/ISSUE_TEMPLATE/90_fuzzing-report.md
@ -0,0 +1,16 @@
+---
+name: Assertion found via fuzzing
+about: Potential issue has been found via Fuzzer or Stress tests
+title: ''
+labels: fuzz
+assignees: ''
+
+---
+
+(you don't have to strictly follow this form)
+
+**Describe the bug**
+A link to the report
+
+**How to reproduce**
+Try to reproduce the report and copy the tables and queries involved.
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -195,13 +195,14 @@ endif ()
 option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker.")

 if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
-    if (LINKER_NAME STREQUAL "lld")
+    # Can be lld or ld-lld.
+    if (LINKER_NAME MATCHES "lld$")
        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index")
        set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gdb-index")
        message (STATUS "Adding .gdb-index via --gdb-index linker option.")
    # we use another tool for gdb-index, because gold linker removes section .debug_aranges, which used inside clickhouse stacktraces
    # http://sourceware-org.1504.n7.nabble.com/gold-No-debug-aranges-section-when-linking-with-gdb-index-td540965.html#a556932
-    elseif (LINKER_NAME STREQUAL "gold" AND ADD_GDB_INDEX_FOR_GOLD)
+    elseif (LINKER_NAME MATCHES "gold$" AND ADD_GDB_INDEX_FOR_GOLD)
        find_program (GDB_ADD_INDEX_EXE NAMES "gdb-add-index" DOC "Path to gdb-add-index executable")
        if (NOT GDB_ADD_INDEX_EXE)
            set (USE_GDB_ADD_INDEX 0)
@ -213,6 +214,19 @@ if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
    endif ()
 endif()

+# Create BuildID when using lld. For other linkers it is created by default.
+if (LINKER_NAME MATCHES "lld$")
+    # SHA1 is not cryptographically secure but it is the best what lld is offering.
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id=sha1")
+endif ()
+
+# Add a section with the hash of the compiled machine code for integrity checks.
+# Only for official builds, because adding a section can be time consuming (rewrite of several GB).
+# And cross compiled binaries are not supported (since you cannot execute clickhouse hash-binary)
+if (OBJCOPY_PATH AND YANDEX_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE))
+    set (USE_BINARY_HASH 1)
+endif ()
+
 cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd


--- a/PreLoad.cmake
+++ b/PreLoad.cmake
@ -12,7 +12,7 @@
 # https://youtrack.jetbrains.com/issue/CPP-2659
 # https://youtrack.jetbrains.com/issue/CPP-870

-if (NOT DEFINED ENV{CLION_IDE})
+if (NOT DEFINED ENV{CLION_IDE} AND NOT DEFINED ENV{XCODE_IDE})
    find_program(NINJA_PATH ninja)
    if (NINJA_PATH)
        set(CMAKE_GENERATOR "Ninja" CACHE INTERNAL "" FORCE)
--- a/base/daemon/BaseDaemon.cpp
+++ b/base/daemon/BaseDaemon.cpp
@ -56,6 +56,9 @@
 #include <Common/Config/ConfigProcessor.h>
 #include <Common/MemorySanitizer.h>
 #include <Common/SymbolIndex.h>
+#include <Common/getExecutablePath.h>
+#include <Common/getHashOfLoadedBinary.h>
+#include <Common/Elf.h>

 #if !defined(ARCADIA_BUILD)
 #   include <Common/config_version.h>
@ -80,16 +83,6 @@ static void call_default_signal_handler(int sig)
    raise(sig);
 }

-const char * msan_strsignal(int sig)
-{
-    // Apparently strsignal is not instrumented by MemorySanitizer, so we
-    // have to unpoison it to avoid msan reports inside fmt library when we
-    // print it.
-    const char * signal_name = strsignal(sig);
-    __msan_unpoison_string(signal_name);
-    return signal_name;
-}
-
 static constexpr size_t max_query_id_size = 127;

 static const size_t signal_pipe_buf_size =
@ -294,13 +287,13 @@ private:
        {
            LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (no query) Received signal {} ({})",
                VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info,
-                thread_num, msan_strsignal(sig), sig);
+                thread_num, strsignal(sig), sig);
        }
        else
        {
            LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (query_id: {}) Received signal {} ({})",
                VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info,
-                thread_num, query_id, msan_strsignal(sig), sig);
+                thread_num, query_id, strsignal(sig), sig);
        }

        String error_message;
@ -328,6 +321,32 @@ private:
        /// Write symbolized stack trace line by line for better grep-ability.
        stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); });

+#if defined(__linux__)
+        /// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace.
+        String calculated_binary_hash = getHashOfLoadedBinaryHex();
+        if (daemon.stored_binary_hash.empty())
+        {
+            LOG_FATAL(log, "Calculated checksum of the binary: {}."
+                " There is no information about the reference checksum.", calculated_binary_hash);
+        }
+        else if (calculated_binary_hash == daemon.stored_binary_hash)
+        {
+            LOG_FATAL(log, "Checksum of the binary: {}, integrity check passed.", calculated_binary_hash);
+        }
+        else
+        {
+            LOG_FATAL(log, "Calculated checksum of the ClickHouse binary ({0}) does not correspond"
+                " to the reference checksum stored in the binary ({1})."
+                " It may indicate one of the following:"
+                " - the file was changed just after startup;"
+                " - the file is damaged on disk due to faulty hardware;"
+                " - the loaded executable is damaged in memory due to faulty hardware;"
+                " - the file was intentionally modified;"
+                " - logical error in code."
+                , calculated_binary_hash, daemon.stored_binary_hash);
+        }
+#endif
+
        /// Write crash to system.crash_log table if available.
        if (collectCrashLog)
            collectCrashLog(sig, thread_num, query_id, stack_trace);
@ -481,8 +500,9 @@ void BaseDaemon::kill()
 {
    dumpCoverageReportIfPossible();
    pid_file.reset();
-    if (::raise(SIGKILL) != 0)
-        throw Poco::SystemException("cannot kill process");
+    /// Exit with the same code as it is usually set by shell when process is terminated by SIGKILL.
+    /// It's better than doing 'raise' or 'kill', because they have no effect for 'init' process (with pid = 0, usually in Docker).
+    _exit(128 + SIGKILL);
 }

 std::string BaseDaemon::getDefaultCorePath() const
@ -787,6 +807,13 @@ void BaseDaemon::initializeTerminationAndSignalProcessing()
 #else
    build_id_info = "no build id";
 #endif
+
+#if defined(__linux__)
+    std::string executable_path = getExecutablePath();
+
+    if (!executable_path.empty())
+        stored_binary_hash = DB::Elf(executable_path).getBinaryHash();
+#endif
 }

 void BaseDaemon::logRevision() const
@ -846,13 +873,13 @@ void BaseDaemon::handleSignal(int signal_id)
        onInterruptSignals(signal_id);
    }
    else
-        throw DB::Exception(std::string("Unsupported signal: ") + msan_strsignal(signal_id), 0);
+        throw DB::Exception(std::string("Unsupported signal: ") + strsignal(signal_id), 0);
 }

 void BaseDaemon::onInterruptSignals(int signal_id)
 {
    is_cancelled = true;
-    LOG_INFO(&logger(), "Received termination signal ({})", msan_strsignal(signal_id));
+    LOG_INFO(&logger(), "Received termination signal ({})", strsignal(signal_id));

    if (sigint_signals_counter >= 2)
    {
@ -998,3 +1025,9 @@ void BaseDaemon::setupWatchdog()
 #endif
    }
 }
+
+
+String BaseDaemon::getStoredBinaryHash() const
+{
+    return stored_binary_hash;
+}
--- a/base/daemon/BaseDaemon.h
+++ b/base/daemon/BaseDaemon.h
@ -60,7 +60,7 @@ public:
    static void terminate();

    /// Forceful shutdown
-    void kill();
+    [[noreturn]] void kill();

    /// Cancellation request has been received.
    bool isCancelled() const
@ -121,6 +121,9 @@ public:
    /// argv0 is needed to change process name (consequently, it is needed for scripts involving "pgrep", "pidof" to work correctly).
    void shouldSetupWatchdog(char * argv0_);

+    /// Hash of the binary for integrity checks.
+    String getStoredBinaryHash() const;
+
 protected:
    virtual void logRevision() const;

@ -168,6 +171,7 @@ protected:
    Poco::Util::AbstractConfiguration * last_configuration = nullptr;

    String build_id_info;
+    String stored_binary_hash;

    std::vector<int> handled_signals;

--- a/base/glibc-compatibility/musl/lgamma.c
+++ b/base/glibc-compatibility/musl/lgamma.c
@ -258,11 +258,3 @@ double lgamma_r(double x, int *signgamp)
 		r = nadj - r;
 	return r;
 }
-
-
-int signgam;
-
-double lgamma(double x)
-{
-	return lgamma_r(x, &signgam);
-}
--- a/base/glibc-compatibility/musl/lgammal.c
+++ b/base/glibc-compatibility/musl/lgammal.c
@ -328,12 +328,3 @@ long double lgammal_r(long double x, int *sg)
 	return lgamma_r(x, sg);
 }
 #endif
-
-
-int signgam_lgammal;
-
-long double lgammal(long double x)
-{
-	return lgammal_r(x, &signgam_lgammal);
-}
-
--- a/base/glibc-compatibility/musl/strsignal.c
+++ b/base/glibc-compatibility/musl/strsignal.c
@ -0,0 +1,125 @@
+#include <signal.h>
+#include <string.h>
+
+#if (SIGHUP == 1) && (SIGINT == 2) && (SIGQUIT == 3) && (SIGILL == 4) \
+ && (SIGTRAP == 5) && (SIGABRT == 6) && (SIGBUS == 7) && (SIGFPE == 8) \
+ && (SIGKILL == 9) && (SIGUSR1 == 10) && (SIGSEGV == 11) && (SIGUSR2 == 12) \
+ && (SIGPIPE == 13) && (SIGALRM == 14) && (SIGTERM == 15) && (SIGSTKFLT == 16) \
+ && (SIGCHLD == 17) && (SIGCONT == 18) && (SIGSTOP == 19) && (SIGTSTP == 20) \
+ && (SIGTTIN == 21) && (SIGTTOU == 22) && (SIGURG == 23) && (SIGXCPU == 24) \
+ && (SIGXFSZ == 25) && (SIGVTALRM == 26) && (SIGPROF == 27) && (SIGWINCH == 28) \
+ && (SIGPOLL == 29) && (SIGPWR == 30) && (SIGSYS == 31)
+
+#define sigmap(x) x
+
+#else
+
+static const char map[] = {
+	[SIGHUP]    = 1,
+	[SIGINT]    = 2,
+	[SIGQUIT]   = 3,
+	[SIGILL]    = 4,
+	[SIGTRAP]   = 5,
+	[SIGABRT]   = 6,
+	[SIGBUS]    = 7,
+	[SIGFPE]    = 8,
+	[SIGKILL]   = 9,
+	[SIGUSR1]   = 10,
+	[SIGSEGV]   = 11,
+	[SIGUSR2]   = 12,
+	[SIGPIPE]   = 13,
+	[SIGALRM]   = 14,
+	[SIGTERM]   = 15,
+#if defined(SIGSTKFLT)
+	[SIGSTKFLT] = 16,
+#elif defined(SIGEMT)
+	[SIGEMT]    = 16,
+#endif
+	[SIGCHLD]   = 17,
+	[SIGCONT]   = 18,
+	[SIGSTOP]   = 19,
+	[SIGTSTP]   = 20,
+	[SIGTTIN]   = 21,
+	[SIGTTOU]   = 22,
+	[SIGURG]    = 23,
+	[SIGXCPU]   = 24,
+	[SIGXFSZ]   = 25,
+	[SIGVTALRM] = 26,
+	[SIGPROF]   = 27,
+	[SIGWINCH]  = 28,
+	[SIGPOLL]   = 29,
+	[SIGPWR]    = 30,
+	[SIGSYS]    = 31
+};
+
+#define sigmap(x) ((x) >= sizeof map ? (x) : map[(x)])
+
+#endif
+
+static const char strings[] =
+	"Unknown signal\0"
+	"Hangup\0"
+	"Interrupt\0"
+	"Quit\0"
+	"Illegal instruction\0"
+	"Trace/breakpoint trap\0"
+	"Aborted\0"
+	"Bus error\0"
+	"Arithmetic exception\0"
+	"Killed\0"
+	"User defined signal 1\0"
+	"Segmentation fault\0"
+	"User defined signal 2\0"
+	"Broken pipe\0"
+	"Alarm clock\0"
+	"Terminated\0"
+#if defined(SIGSTKFLT)
+	"Stack fault\0"
+#elif defined(SIGEMT)
+	"Emulator trap\0"
+#else
+	"Unknown signal\0"
+#endif
+	"Child process status\0"
+	"Continued\0"
+	"Stopped (signal)\0"
+	"Stopped\0"
+	"Stopped (tty input)\0"
+	"Stopped (tty output)\0"
+	"Urgent I/O condition\0"
+	"CPU time limit exceeded\0"
+	"File size limit exceeded\0"
+	"Virtual timer expired\0"
+	"Profiling timer expired\0"
+	"Window changed\0"
+	"I/O possible\0"
+	"Power failure\0"
+	"Bad system call\0"
+	"RT32"
+	"\0RT33\0RT34\0RT35\0RT36\0RT37\0RT38\0RT39\0RT40"
+	"\0RT41\0RT42\0RT43\0RT44\0RT45\0RT46\0RT47\0RT48"
+	"\0RT49\0RT50\0RT51\0RT52\0RT53\0RT54\0RT55\0RT56"
+	"\0RT57\0RT58\0RT59\0RT60\0RT61\0RT62\0RT63\0RT64"
+#if _NSIG > 65
+	"\0RT65\0RT66\0RT67\0RT68\0RT69\0RT70\0RT71\0RT72"
+	"\0RT73\0RT74\0RT75\0RT76\0RT77\0RT78\0RT79\0RT80"
+	"\0RT81\0RT82\0RT83\0RT84\0RT85\0RT86\0RT87\0RT88"
+	"\0RT89\0RT90\0RT91\0RT92\0RT93\0RT94\0RT95\0RT96"
+	"\0RT97\0RT98\0RT99\0RT100\0RT101\0RT102\0RT103\0RT104"
+	"\0RT105\0RT106\0RT107\0RT108\0RT109\0RT110\0RT111\0RT112"
+	"\0RT113\0RT114\0RT115\0RT116\0RT117\0RT118\0RT119\0RT120"
+	"\0RT121\0RT122\0RT123\0RT124\0RT125\0RT126\0RT127\0RT128"
+#endif
+	"";
+
+char *strsignal(int signum)
+{
+	const char *s = strings;
+
+	signum = sigmap(signum);
+	if (signum - 1U >= _NSIG-1) signum = 0;
+
+	for (; signum--; s++) for (; *s; s++);
+
+	return (char *)s;
+}
--- a/base/harmful/CMakeLists.txt
+++ b/base/harmful/CMakeLists.txt
@ -0,0 +1,2 @@
+add_library(harmful harmful.c)
+install(TARGETS harmful EXPORT global ARCHIVE DESTINATION lib)
--- a/base/harmful/README.md
+++ b/base/harmful/README.md
@ -0,0 +1 @@
+A library that traps whenever harmful functions from libc are called.
--- a/base/harmful/harmful.c
+++ b/base/harmful/harmful.c
@ -0,0 +1,244 @@
+/** This library provides runtime instrumentation (hardening)
+  * that ensures no "harmful" functions from libc are called
+  * (by terminating the program immediately).
+  */
+
+/// It is only enabled in debug build (its intended use is for CI checks).
+#if !defined(NDEBUG)
+
+#if defined(__clang__)
+    #pragma clang diagnostic ignored "-Wincompatible-library-redeclaration"
+#else
+    #pragma GCC diagnostic ignored "-Wbuiltin-declaration-mismatch"
+#endif
+
+/// We cannot use libc headers here.
+long write(int, const void *, unsigned long);
+#define TRAP(func) void func() { write(2, #func "\n", __builtin_strlen(#func) + 1); __builtin_trap(); }
+
+/// Trap all non thread-safe functions:
+/// nm -D /lib/x86_64-linux-gnu/{libc.so.6,libdl.so.2,libm.so.6,libpthread.so.0,librt.so.1,libnss_dns.so.2,libresolv.so.2} | grep -P '_r@?$' | awk '{ print $3 }' | sed -r -e 's/_r//' | grep -vP '^_'
+
+/// See also https://reviews.llvm.org/D90944
+
+/// You can edit this list and even comment out some functions.
+/// The only purpose of the library is to force you to pay attention.
+
+TRAP(argp_error)
+TRAP(argp_help)
+TRAP(argp_parse)
+TRAP(argp_state_help)
+TRAP(argp_usage)
+TRAP(asctime)
+TRAP(clearenv)
+TRAP(crypt)
+TRAP(ctime)
+TRAP(cuserid)
+TRAP(drand48)
+TRAP(ecvt)
+TRAP(encrypt)
+TRAP(endfsent)
+TRAP(endgrent)
+TRAP(endhostent)
+TRAP(endnetent)
+TRAP(endnetgrent)
+TRAP(endprotoent)
+TRAP(endpwent)
+TRAP(endservent)
+TRAP(endutent)
+TRAP(endutxent)
+TRAP(erand48)
+TRAP(error_at_line)
+///TRAP(exit)
+TRAP(fcloseall)
+TRAP(fcvt)
+TRAP(fgetgrent)
+TRAP(fgetpwent)
+TRAP(gammal)
+TRAP(getchar_unlocked)
+TRAP(getdate)
+TRAP(getfsent)
+TRAP(getfsfile)
+TRAP(getfsspec)
+TRAP(getgrent)
+TRAP(getgrent_r)
+TRAP(getgrgid)
+TRAP(getgrnam)
+TRAP(gethostbyaddr)
+TRAP(gethostbyname)
+TRAP(gethostbyname2)
+TRAP(gethostent)
+TRAP(getlogin)
+TRAP(getmntent)
+TRAP(getnetbyaddr)
+TRAP(getnetbyname)
+TRAP(getnetent)
+TRAP(getnetgrent)
+TRAP(getnetgrent_r)
+TRAP(getopt)
+TRAP(getopt_long)
+TRAP(getopt_long_only)
+TRAP(getpass)
+TRAP(getprotobyname)
+TRAP(getprotobynumber)
+TRAP(getprotoent)
+TRAP(getpwent)
+TRAP(getpwent_r)
+TRAP(getpwnam)
+TRAP(getpwuid)
+TRAP(getservbyname)
+TRAP(getservbyport)
+TRAP(getservent)
+TRAP(getutent)
+TRAP(getutent_r)
+TRAP(getutid)
+TRAP(getutid_r)
+TRAP(getutline)
+TRAP(getutline_r)
+TRAP(getutxent)
+TRAP(getutxid)
+TRAP(getutxline)
+TRAP(getwchar_unlocked)
+//TRAP(glob)
+//TRAP(glob64)
+TRAP(gmtime)
+TRAP(hcreate)
+TRAP(hdestroy)
+TRAP(hsearch)
+TRAP(innetgr)
+TRAP(jrand48)
+TRAP(l64a)
+TRAP(lcong48)
+TRAP(lgammafNx)
+TRAP(localeconv)
+TRAP(localtime)
+TRAP(login)
+TRAP(login_tty)
+TRAP(logout)
+TRAP(logwtmp)
+TRAP(lrand48)
+TRAP(mallinfo)
+TRAP(mallopt)
+TRAP(mblen)
+TRAP(mbrlen)
+TRAP(mbrtowc)
+TRAP(mbsnrtowcs)
+TRAP(mbsrtowcs)
+//TRAP(mbtowc) // Used by Standard C++ library
+TRAP(mcheck)
+TRAP(mprobe)
+TRAP(mrand48)
+TRAP(mtrace)
+TRAP(muntrace)
+TRAP(nrand48)
+TRAP(__ppc_get_timebase_freq)
+TRAP(ptsname)
+TRAP(putchar_unlocked)
+TRAP(putenv)
+TRAP(pututline)
+TRAP(pututxline)
+TRAP(putwchar_unlocked)
+TRAP(qecvt)
+TRAP(qfcvt)
+TRAP(register_printf_function)
+TRAP(seed48)
+//TRAP(setenv)
+TRAP(setfsent)
+TRAP(setgrent)
+TRAP(sethostent)
+TRAP(sethostid)
+TRAP(setkey)
+//TRAP(setlocale) // Used by replxx at startup
+TRAP(setlogmask)
+TRAP(setnetent)
+TRAP(setnetgrent)
+TRAP(setprotoent)
+TRAP(setpwent)
+TRAP(setservent)
+TRAP(setutent)
+TRAP(setutxent)
+TRAP(siginterrupt)
+TRAP(sigpause)
+//TRAP(sigprocmask)
+TRAP(sigsuspend)
+TRAP(sleep)
+TRAP(srand48)
+//TRAP(strerror) // Used by RocksDB and many other libraries, unfortunately.
+//TRAP(strsignal) // This function is imported from Musl and is thread safe.
+TRAP(strtok)
+TRAP(tcflow)
+TRAP(tcsendbreak)
+TRAP(tmpnam)
+TRAP(ttyname)
+TRAP(unsetenv)
+TRAP(updwtmp)
+TRAP(utmpname)
+TRAP(utmpxname)
+//TRAP(valloc)
+TRAP(vlimit)
+//TRAP(wcrtomb) // Used by Standard C++ library
+TRAP(wcsnrtombs)
+TRAP(wcsrtombs)
+TRAP(wctomb)
+TRAP(wordexp)
+TRAP(basename)
+TRAP(catgets)
+TRAP(dbm_clearerr)
+TRAP(dbm_close)
+TRAP(dbm_delete)
+TRAP(dbm_error)
+TRAP(dbm_fetch)
+TRAP(dbm_firstkey)
+TRAP(dbm_nextkey)
+TRAP(dbm_open)
+TRAP(dbm_store)
+TRAP(dirname)
+TRAP(dlerror)
+TRAP(ftw)
+TRAP(getc_unlocked)
+//TRAP(getenv) // Ok at program startup
+TRAP(inet_ntoa)
+TRAP(lgamma)
+TRAP(lgammaf)
+TRAP(lgammal)
+TRAP(nftw)
+TRAP(nl_langinfo)
+TRAP(putc_unlocked)
+TRAP(rand)
+/** In  the current POSIX.1 specification (POSIX.1-2008), readdir() is not required to be thread-safe.  However, in modern
+  * implementations (including the glibc implementation), concurrent calls to readdir() that specify different directory streams
+  * are thread-safe.  In cases where multiple threads must read from the same directory stream, using readdir() with external
+  * synchronization is still preferable to the use of the deprecated readdir_r(3)  function. It is expected that a future
+  * version of POSIX.1 will require that readdir() be thread-safe when concurrently employed on different directory streams.
+  * - man readdir
+  */
+//TRAP(readdir)
+TRAP(system)
+TRAP(wcstombs)
+TRAP(ether_aton)
+TRAP(ether_ntoa)
+TRAP(fgetsgent)
+TRAP(fgetspent)
+TRAP(getaliasbyname)
+TRAP(getaliasent)
+TRAP(getrpcbyname)
+TRAP(getrpcbynumber)
+TRAP(getrpcent)
+TRAP(getsgent)
+TRAP(getsgnam)
+TRAP(getspent)
+TRAP(getspnam)
+TRAP(initstate)
+TRAP(random)
+TRAP(setstate)
+TRAP(sgetsgent)
+TRAP(sgetspent)
+TRAP(srandom)
+TRAP(twalk)
+TRAP(lgammaf128)
+TRAP(lgammaf32)
+TRAP(lgammaf32x)
+TRAP(lgammaf64)
+TRAP(lgammaf64x)
+
+#endif
--- a/cmake/linux/default_libs.cmake
+++ b/cmake/linux/default_libs.cmake
@ -39,6 +39,7 @@ find_package(Threads REQUIRED)
 if (NOT OS_ANDROID)
    # Our compatibility layer doesn't build under Android, many errors in musl.
    add_subdirectory(base/glibc-compatibility)
+    add_subdirectory(base/harmful)
 endif ()

 include (cmake/find/unwind.cmake)
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@ -18,7 +18,11 @@ if (WITH_COVERAGE)
  set (WITHOUT_COVERAGE_LIST ${WITHOUT_COVERAGE})
  separate_arguments(WITHOUT_COVERAGE_LIST)
  # disable coverage for contib files and build with optimisations
-  add_compile_options(-O3 -DNDEBUG -finline-functions -finline-hint-functions ${WITHOUT_COVERAGE_LIST})
+  if (COMPILER_CLANG)
+      add_compile_options(-O3 -DNDEBUG -finline-functions -finline-hint-functions ${WITHOUT_COVERAGE_LIST})
+  else()
+      add_compile_options(-O3 -DNDEBUG -finline-functions ${WITHOUT_COVERAGE_LIST})
+  endif()
 endif()

 if (SANITIZE STREQUAL "undefined")
--- a/contrib/mariadb-connector-c
+++ b/contrib/mariadb-connector-c
@ -1 +1 @@
-Subproject commit e05523ca7c1fb8d095b612a1b1cfe96e199ffb17
+Subproject commit 21f451d4d3157ffed31ec60a8b76c407190e66bd
--- a/contrib/replxx
+++ b/contrib/replxx
@ -1 +1 @@
-Subproject commit 254be98ae7f2fd92d6db768f8e11ea5a5226cbf5
+Subproject commit cdb6e3f2ce4464225daf9c8beeae7db98d590bdc
--- a/contrib/rocksdb
+++ b/contrib/rocksdb
@ -1 +1 @@
-Subproject commit 8b966f0ca298fc1475bd09d9775f32dff0fdce0a
+Subproject commit 54a0decabbcf4c0bb5cf7befa9c597f28289bff5
--- a/debian/control
+++ b/debian/control
@ -40,7 +40,7 @@ Description: Common files for ClickHouse
 Package: clickhouse-server
 Architecture: all
 Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-common-static (= ${binary:Version}), adduser
-Recommends: libcap2-bin, krb5-user
+Recommends: libcap2-bin
 Replaces: clickhouse-server-common, clickhouse-server-base
 Provides: clickhouse-server-common
 Description: Server binary for ClickHouse
--- a/docker/packager/binary/build.sh
+++ b/docker/packager/binary/build.sh
@ -31,7 +31,7 @@ find . -name '*.so.*' -print -exec mv '{}' /output \;
 if [ "performance" == "$COMBINED_OUTPUT" ]
 then
    cp -r ../tests/performance /output
-    cp -r ../tests/config/top_level_domains  /
+    cp -r ../tests/config/top_level_domains  /output
    cp -r ../docker/test/performance-comparison/config /output ||:
    rm /output/unit_tests_dbms ||:
    rm /output/clickhouse-odbc-bridge ||:
--- a/docker/test/fuzzer/query-fuzzer-tweaks-users.xml
+++ b/docker/test/fuzzer/query-fuzzer-tweaks-users.xml
@ -10,6 +10,11 @@
                <max_execution_time>
                    <max>10</max>
                </max_execution_time>
+
+                <!-- Not ready for production -->
+                <compile_expressions>
+                    <readonly />
+                </compile_expressions>
            </constraints>
        </default>
    </profiles>
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@ -81,12 +81,11 @@ function fuzz
    echo Server started

    fuzzer_exit_code=0
-    # SC2012: Use find instead of ls to better handle non-alphanumeric filenames.
-    # They are all alphanumeric.
-    # shellcheck disable=SC2012
-    ./clickhouse-client --query-fuzzer-runs=1000 \
-        < <(for f in $(ls ch/tests/queries/0_stateless/*.sql | sort -R); do cat "$f"; echo ';'; done) \
-        > >(tail -10000 > fuzzer.log) \
+    # SC2012: Use find instead of ls to better handle non-alphanumeric filenames. They are all alphanumeric.
+    # SC2046: Quote this to prevent word splitting. Actually I need word splitting.
+    # shellcheck disable=SC2012,SC2046
+    ./clickhouse-client --query-fuzzer-runs=1000 --queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) \
+        > >(tail -n 10000 > fuzzer.log) \
        2>&1 \
        || fuzzer_exit_code=$?

--- a/docker/test/performance-comparison/compare.sh
+++ b/docker/test/performance-comparison/compare.sh
@ -36,6 +36,22 @@ function wait_for_server # port, pid
    fi
 }

+function left_or_right()
+{
+    local from=$1 && shift
+    local basename=$1 && shift
+
+    if [ -e "$from/$basename" ]; then
+        echo "$from/$basename"
+        return
+    fi
+
+    case "$from" in
+        left) echo "right/$basename" ;;
+        right) echo "left/$basename" ;;
+    esac
+}
+
 function configure
 {
    # Use the new config for both servers, so that we can change it in a PR.
@ -55,7 +71,7 @@ function configure
        # server *config* directives overrides
        --path db0
        --user_files_path db0/user_files
-        --top_level_domains_path /top_level_domains
+        --top_level_domains_path "$(left_or_right right top_level_domains)"
        --tcp_port $LEFT_SERVER_PORT
    )
    left/clickhouse-server "${setup_left_server_opts[@]}" &> setup-server-log.log &
@ -103,7 +119,7 @@ function restart
        # server *config* directives overrides
        --path left/db
        --user_files_path left/db/user_files
-        --top_level_domains_path /top_level_domains
+        --top_level_domains_path "$(left_or_right left top_level_domains)"
        --tcp_port $LEFT_SERVER_PORT
    )
    left/clickhouse-server "${left_server_opts[@]}" &>> left-server-log.log &
@ -118,7 +134,7 @@ function restart
        # server *config* directives overrides
        --path right/db
        --user_files_path right/db/user_files
-        --top_level_domains_path /top_level_domains
+        --top_level_domains_path "$(left_or_right right top_level_domains)"
        --tcp_port $RIGHT_SERVER_PORT
    )
    right/clickhouse-server "${right_server_opts[@]}" &>> right-server-log.log &
--- a/docker/test/stateful/run.sh
+++ b/docker/test/stateful/run.sh
@ -37,7 +37,15 @@ chmod 777 -R /var/lib/clickhouse
 clickhouse-client --query "SHOW DATABASES"
 clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary"
 clickhouse-client --query "CREATE DATABASE test"
-service clickhouse-server restart && sleep 5
+
+service clickhouse-server restart
+
+# Wait for server to start accepting connections
+for _ in {1..120}; do
+    clickhouse-client --query "SELECT 1" && break
+    sleep 1
+done
+
 clickhouse-client --query "SHOW TABLES FROM datasets"
 clickhouse-client --query "SHOW TABLES FROM test"
 clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
--- a/docs/en/development/tests.md
+++ b/docs/en/development/tests.md
@ -11,7 +11,7 @@ Functional tests are the most simple and convenient to use. Most of ClickHouse f

 Each functional test sends one or multiple queries to the running ClickHouse server and compares the result with reference.

-Tests are located in `queries` directory. There are two subdirectories: `stateless` and `stateful`. Stateless tests run queries without any preloaded test data - they often create small synthetic datasets on the fly, within the test itself. Stateful tests require preloaded test data from Yandex.Metrica and not available to general public. We tend to use only `stateless` tests and avoid adding new `stateful` tests.
+Tests are located in `queries` directory. There are two subdirectories: `stateless` and `stateful`. Stateless tests run queries without any preloaded test data - they often create small synthetic datasets on the fly, within the test itself. Stateful tests require preloaded test data from Yandex.Metrica and it is available to general public.

 Each test can be one of two types: `.sql` and `.sh`. `.sql` test is the simple SQL script that is piped to `clickhouse-client --multiquery --testmode`. `.sh` test is a script that is run by itself. SQL tests are generally preferable to `.sh` tests. You should use `.sh` tests only when you have to test some feature that cannot be exercised from pure SQL, such as piping some input data into `clickhouse-client` or testing `clickhouse-local`.

@ -84,11 +84,9 @@ If you want to improve performance of ClickHouse in some scenario, and if improv

 Some programs in `tests` directory are not prepared tests, but are test tools. For example, for `Lexer` there is a tool `src/Parsers/tests/lexer` that just do tokenization of stdin and writes colorized result to stdout. You can use these kind of tools as a code examples and for exploration and manual testing.

-You can also place pair of files `.sh` and `.reference` along with the tool to run it on some predefined input - then script result can be compared to `.reference` file. These kind of tests are not automated.
-
 ## Miscellaneous Tests {#miscellaneous-tests}

-There are tests for external dictionaries located at `tests/external_dictionaries` and for machine learned models in `tests/external_models`. These tests are not updated and must be transferred to integration tests.
+There are tests for machine learned models in `tests/external_models`. These tests are not updated and must be transferred to integration tests.

 There is separate test for quorum inserts. This test run ClickHouse cluster on separate servers and emulate various failure cases: network split, packet drop (between ClickHouse nodes, between ClickHouse and ZooKeeper, between ClickHouse server and client, etc.), `kill -9`, `kill -STOP` and `kill -CONT` , like [Jepsen](https://aphyr.com/tags/Jepsen). Then the test checks that all acknowledged inserts was written and all rejected inserts was not.

@ -169,53 +167,55 @@ Precise query execution timings are not recorded and not compared due to high va

 ## Build Tests {#build-tests}

-Build tests allow to check that build is not broken on various alternative configurations and on some foreign systems. Tests are located at `ci` directory. They run build from source inside Docker, Vagrant, and sometimes with `qemu-user-static` inside Docker. These tests are under development and test runs are not automated.
+Build tests allow to check that build is not broken on various alternative configurations and on some foreign systems. These tests are automated as well.

-Motivation:
-
-Normally we release and run all tests on a single variant of ClickHouse build. But there are alternative build variants that are not thoroughly tested. Examples:
-
-   build on FreeBSD
-   build on Debian with libraries from system packages
-   build with shared linking of libraries
-   build on AArch64 platform
-   build on PowerPc platform
+Examples:
+-   cross-compile for Darwin x86_64 (Mac OS X)
+-   cross-compile for FreeBSD x86_64
+-   cross-compile for Linux AArch64
+-   build on Ubuntu with libraries from system packages (discouraged)
+-   build with shared linking of libraries (discouraged)

 For example, build with system packages is bad practice, because we cannot guarantee what exact version of packages a system will have. But this is really needed by Debian maintainers. For this reason we at least have to support this variant of build. Another example: shared linking is a common source of trouble, but it is needed for some enthusiasts.

 Though we cannot run all tests on all variant of builds, we want to check at least that various build variants are not broken. For this purpose we use build tests.

+We also test that there are no translation units that are too long to compile or require too much RAM.
+
+We also test that there are no too large stack frames.
+
 ## Testing for Protocol Compatibility {#testing-for-protocol-compatibility}

 When we extend ClickHouse network protocol, we test manually that old clickhouse-client works with new clickhouse-server and new clickhouse-client works with old clickhouse-server (simply by running binaries from corresponding packages).

+We also test some cases automatically with integrational tests:
+- if data written by old version of ClickHouse can be successfully read by the new version;
+- do distributed queries work in a cluster with different ClickHouse versions.
+
 ## Help from the Compiler {#help-from-the-compiler}

 Main ClickHouse code (that is located in `dbms` directory) is built with `-Wall -Wextra -Werror` and with some additional enabled warnings. Although these options are not enabled for third-party libraries.

 Clang has even more useful warnings - you can look for them with `-Weverything` and pick something to default build.

-For production builds, gcc is used (it still generates slightly more efficient code than clang). For development, clang is usually more convenient to use. You can build on your own machine with debug mode (to save battery of your laptop), but please note that compiler is able to generate more warnings with `-O3` due to better control flow and inter-procedure analysis. When building with clang in debug mode, debug version of `libc++` is used that allows to catch more errors at runtime.
+For production builds, clang is used, but we also test make gcc builds. For development, clang is usually more convenient to use. You can build on your own machine with debug mode (to save battery of your laptop), but please note that compiler is able to generate more warnings with `-O3` due to better control flow and inter-procedure analysis. When building with clang in debug mode, debug version of `libc++` is used that allows to catch more errors at runtime.

 ## Sanitizers {#sanitizers}

 ### Address sanitizer
-We run functional and integration tests under ASan on per-commit basis.
-
-### Valgrind (Memcheck)
-We run functional tests under Valgrind overnight. It takes multiple hours. Currently there is one known false positive in `re2` library, see [this article](https://research.swtch.com/sparse).
-
-### Undefined behaviour sanitizer
-We run functional and integration tests under ASan on per-commit basis.
+We run functional, integration, stress and unit tests under ASan on per-commit basis.

 ### Thread sanitizer
-We run functional tests under TSan on per-commit basis. We still don’t run integration tests under TSan on per-commit basis.
+We run functional, integration, stress and unit tests under TSan on per-commit basis.

 ### Memory sanitizer
-Currently we still don’t use MSan.
+We run functional, integration, stress and unit tests under MSan on per-commit basis.

-### Debug allocator
-Debug version of `jemalloc` is used for debug build.
+### Undefined behaviour sanitizer
+We run functional, integration, stress and unit tests under UBSan on per-commit basis. The code of some third-party libraries is not sanitized for UB.
+
+### Valgrind (Memcheck)
+We used to run functional tests under Valgrind overnight, but don't do it anymore. It takes multiple hours. Currently there is one known false positive in `re2` library, see [this article](https://research.swtch.com/sparse).

 ## Fuzzing {#fuzzing}

@ -233,19 +233,62 @@ Google OSS-Fuzz can be found at `docker/fuzz`.
 We also use simple fuzz test to generate random SQL queries and to check that the server doesn’t die executing them.
 You can find it in `00746_sql_fuzzy.pl`. This test should be run continuously (overnight and longer).

+We also use sophisticated AST-based query fuzzer that is able to find huge amount of corner cases. It does random permutations and substitutions in queries AST. It remembers AST nodes from previous tests to use them for fuzzing of subsequent tests while processing them in random order.
+
+## Stress test
+
+Stress tests are another case of fuzzing. It runs all functional tests in parallel in random order with a single server. Results of the tests are not checked.
+
+It is checked that:
+- server does not crash, no debug or sanitizer traps are triggered;
+- there are no deadlocks;
+- the database structure is consistent;
+- server can successfully stop after the test and start again without exceptions.
+
+There are five variants (Debug, ASan, TSan, MSan, UBSan).
+
+## Thread Fuzzer
+
+Thread Fuzzer (please don't mix up with Thread Sanitizer) is another kind of fuzzing that allows to randomize thread order of execution. It helps to find even more special cases.
+
 ## Security Audit {#security-audit}

 People from Yandex Security Team do some basic overview of ClickHouse capabilities from the security standpoint.

 ## Static Analyzers {#static-analyzers}

-We run `PVS-Studio` on per-commit basis. We have evaluated `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`. You will find instructions for usage in `tests/instructions/` directory. Also you can read [the article in russian](https://habr.com/company/yandex/blog/342018/).
+We run `clang-tidy` and `PVS-Studio` on per-commit basis. `clang-static-analyzer` checks are also enabled. `clang-tidy` is also used for some style checks.
+
+We have evaluated `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`, `CodeQL`. You will find instructions for usage in `tests/instructions/` directory. Also you can read [the article in russian](https://habr.com/company/yandex/blog/342018/).

 If you use `CLion` as an IDE, you can leverage some `clang-tidy` checks out of the box.

+We also use `shellcheck` for static analysis of shell scripts.
+
 ## Hardening {#hardening}

-`FORTIFY_SOURCE` is used by default. It is almost useless, but still makes sense in rare cases and we don’t disable it.
+In debug build we are using custom allocator that does ASLR of user-level allocations.
+
+We also manually protect memory regions that are expected to be readonly after allocation.
+
+In debug build we also involve a customization of libc that ensures that no "harmful" (obsolete, insecure, not thread-safe) functions are called.
+
+Debug assertions are used extensively.
+
+In debug build, if exception with "logical error" code (implies a bug) is being thrown, the program is terminated prematurally. It allows to use exceptions in release build but make it an assertion in debug build.
+
+Debug version of jemalloc is used for debug builds.
+Debug version of libc++ is used for debug builds.
+
+## Runtime Integrity Checks
+
+Data stored on disk is checksummed. Data in MergeTree tables is checksummed in three ways simultaneously* (compressed data blocks, uncompressed data blocks, the total checksum across blocks). Data transferred over network between client and server or between servers is also checksummed. Replication ensures bit-identical data on replicas.
+
+It is required to protect from faulty hardware (bit rot on storage media, bit flips in RAM on server, bit flips in RAM of network controller, bit flips in RAM of network switch, bit flips in RAM of client, bit flips on the wire). Note that bit flips are common and likely to occur even for ECC RAM and in presense of TCP checksums (if you manage to run thousands of servers processing petabytes of data each day). [See the video (russian)](https://www.youtube.com/watch?v=ooBAQIe0KlQ).
+
+ClickHouse provides diagnostics that will help ops engineers to find faulty hardware.
+
+\* and it is not slow.

 ## Code Style {#code-style}

@ -259,6 +302,8 @@ Alternatively you can try `uncrustify` tool to reformat your code. Configuration

 `CLion` has its own code formatter that has to be tuned for our code style.

+We also use `codespell` to find typos in code. It is automated as well.
+
 ## Metrica B2B Tests {#metrica-b2b-tests}

 Each ClickHouse release is tested with Yandex Metrica and AppMetrica engines. Testing and stable versions of ClickHouse are deployed on VMs and run with a small copy of Metrica engine that is processing fixed sample of input data. Then results of two instances of Metrica engine are compared together.
@ -267,13 +312,25 @@ These tests are automated by separate team. Due to high number of moving parts,

 ## Test Coverage {#test-coverage}

-As of July 2018 we don’t track test coverage.
+We also track test coverage but only for functional tests and only for clickhouse-server. It is performed on daily basis.
+
+## Tests for Tests
+
+There is automated check for flaky tests. It runs all new tests 100 times (for functional tests) or 10 times (for integration tests). If at least single time the test failed, it is considered flaky.
+
+## Testflows
+
+[Testflows](https://testflows.com/) is an enterprise-grade testing framework. It is used by Altinity for some of the tests and we run these tests in our CI.
+
+## Yandex Checks (only for Yandex employees)
+
+These checks are importing ClickHouse code into Yandex internal monorepository, so ClickHouse codebase can be used as a library by other products at Yandex (YT and YDB). Note that clickhouse-server itself is not being build from internal repo and unmodified open-source build is used for Yandex applications.

 ## Test Automation {#test-automation}

 We run tests with Yandex internal CI and job automation system named “Sandbox”.

-Build jobs and tests are run in Sandbox on per commit basis. Resulting packages and test results are published in GitHub and can be downloaded by direct links. Artifacts are stored eternally. When you send a pull request on GitHub, we tag it as “can be tested” and our CI system will build ClickHouse packages (release, debug, with address sanitizer, etc) for you.
+Build jobs and tests are run in Sandbox on per commit basis. Resulting packages and test results are published in GitHub and can be downloaded by direct links. Artifacts are stored for several months. When you send a pull request on GitHub, we tag it as “can be tested” and our CI system will build ClickHouse packages (release, debug, with address sanitizer, etc) for you.

 We don’t use Travis CI due to the limit on time and computational power.
 We don’t use Jenkins. It was used before and now we are happy we are not using Jenkins.
--- a/docs/en/engines/table-engines/mergetree-family/replication.md
+++ b/docs/en/engines/table-engines/mergetree-family/replication.md
@ -38,15 +38,15 @@ Example of setting the addresses of the ZooKeeper cluster:

 ``` xml
 <zookeeper>
-    <node index="1">
+    <node>
        <host>example1</host>
        <port>2181</port>
    </node>
-    <node index="2">
+    <node>
        <host>example2</host>
        <port>2181</port>
    </node>
-    <node index="3">
+    <node>
        <host>example3</host>
        <port>2181</port>
    </node>
@ -61,21 +61,21 @@ Example of setting the addresses of the auxiliary ZooKeeper cluster:
 ``` xml
 <auxiliary_zookeepers>
    <zookeeper2>
-        <node index="1">
+        <node>
            <host>example_2_1</host>
            <port>2181</port>
        </node>
-        <node index="2">
+        <node>
            <host>example_2_2</host>
            <port>2181</port>
        </node>
-        <node index="3">
+        <node>
            <host>example_2_3</host>
            <port>2181</port>
        </node>
    </zookeeper2>
    <zookeeper3>
-        <node index="1">
+        <node>
            <host>example_3_1</host>
            <port>2181</port>
        </node>
--- a/docs/en/getting-started/example-datasets/brown-benchmark.md
+++ b/docs/en/getting-started/example-datasets/brown-benchmark.md
@ -0,0 +1,414 @@
+---
+toc_priority: 20
+toc_title: Brown University Benchmark
+---
+
+# Brown University Benchmark
+
+MgBench - A new analytical benchmark for machine-generated log data, [Andrew Crotty](http://cs.brown.edu/people/acrotty/).
+
+Download the data:
+```
+wget https://datasets.clickhouse.tech/mgbench{1..3}.csv.xz
+```
+
+Unpack the data:
+```
+xz -v -d mgbench{1..3}.csv.xz
+```
+
+Create tables:
+```
+CREATE DATABASE mgbench;
+
+
+CREATE TABLE mgbench.logs1 (
+  log_time      DateTime,
+  machine_name  LowCardinality(String),
+  machine_group LowCardinality(String),
+  cpu_idle      Nullable(Float32),
+  cpu_nice      Nullable(Float32),
+  cpu_system    Nullable(Float32),
+  cpu_user      Nullable(Float32),
+  cpu_wio       Nullable(Float32),
+  disk_free     Nullable(Float32),
+  disk_total    Nullable(Float32),
+  part_max_used Nullable(Float32),
+  load_fifteen  Nullable(Float32),
+  load_five     Nullable(Float32),
+  load_one      Nullable(Float32),
+  mem_buffers   Nullable(Float32),
+  mem_cached    Nullable(Float32),
+  mem_free      Nullable(Float32),
+  mem_shared    Nullable(Float32),
+  swap_free     Nullable(Float32),
+  bytes_in      Nullable(Float32),
+  bytes_out     Nullable(Float32)
+)
+ENGINE = MergeTree()
+ORDER BY (machine_group, machine_name, log_time);
+
+
+CREATE TABLE mgbench.logs2 (
+  log_time    DateTime,
+  client_ip   IPv4,
+  request     String,
+  status_code UInt16,
+  object_size UInt64
+)
+ENGINE = MergeTree()
+ORDER BY log_time;
+
+
+CREATE TABLE mgbench.logs3 (
+  log_time     DateTime64,
+  device_id    FixedString(15),
+  device_name  LowCardinality(String),
+  device_type  LowCardinality(String),
+  device_floor UInt8,
+  event_type   LowCardinality(String),
+  event_unit   FixedString(1),
+  event_value  Nullable(Float32)
+)
+ENGINE = MergeTree()
+ORDER BY (event_type, log_time);
+```
+
+Insert data:
+
+```
+clickhouse-client --query "INSERT INTO mgbench.logs1 FORMAT CSVWithNames" < mgbench1.csv
+clickhouse-client --query "INSERT INTO mgbench.logs2 FORMAT CSVWithNames" < mgbench2.csv
+clickhouse-client --query "INSERT INTO mgbench.logs3 FORMAT CSVWithNames" < mgbench3.csv
+```
+
+Run benchmark queries:
+```
+-- Q1.1: What is the CPU/network utilization for each web server since midnight?
+
+SELECT machine_name,
+       MIN(cpu) AS cpu_min,
+       MAX(cpu) AS cpu_max,
+       AVG(cpu) AS cpu_avg,
+       MIN(net_in) AS net_in_min,
+       MAX(net_in) AS net_in_max,
+       AVG(net_in) AS net_in_avg,
+       MIN(net_out) AS net_out_min,
+       MAX(net_out) AS net_out_max,
+       AVG(net_out) AS net_out_avg
+FROM (
+  SELECT machine_name,
+         COALESCE(cpu_user, 0.0) AS cpu,
+         COALESCE(bytes_in, 0.0) AS net_in,
+         COALESCE(bytes_out, 0.0) AS net_out
+  FROM logs1
+  WHERE machine_name IN ('anansi','aragog','urd')
+    AND log_time >= TIMESTAMP '2017-01-11 00:00:00'
+) AS r
+GROUP BY machine_name;
+
+
+-- Q1.2: Which computer lab machines have been offline in the past day?
+
+SELECT machine_name,
+       log_time
+FROM logs1
+WHERE (machine_name LIKE 'cslab%' OR
+       machine_name LIKE 'mslab%')
+  AND load_one IS NULL
+  AND log_time >= TIMESTAMP '2017-01-10 00:00:00'
+ORDER BY machine_name,
+         log_time;
+
+
+-- Q1.3: What are the hourly average metrics during the past 10 days for a specific workstation?
+
+SELECT dt,
+       hr,
+       AVG(load_fifteen) AS load_fifteen_avg,
+       AVG(load_five) AS load_five_avg,
+       AVG(load_one) AS load_one_avg,
+       AVG(mem_free) AS mem_free_avg,
+       AVG(swap_free) AS swap_free_avg
+FROM (
+  SELECT CAST(log_time AS DATE) AS dt,
+         EXTRACT(HOUR FROM log_time) AS hr,
+         load_fifteen,
+         load_five,
+         load_one,
+         mem_free,
+         swap_free
+  FROM logs1
+  WHERE machine_name = 'babbage'
+    AND load_fifteen IS NOT NULL
+    AND load_five IS NOT NULL
+    AND load_one IS NOT NULL
+    AND mem_free IS NOT NULL
+    AND swap_free IS NOT NULL
+    AND log_time >= TIMESTAMP '2017-01-01 00:00:00'
+) AS r
+GROUP BY dt,
+         hr
+ORDER BY dt,
+         hr;
+
+
+-- Q1.4: Over a 1-month period, how often was each server blocked on disk I/O?
+
+SELECT machine_name,
+       COUNT(*) AS spikes
+FROM logs1
+WHERE machine_group = 'Servers'
+  AND cpu_wio > 0.99
+  AND log_time >= TIMESTAMP '2016-12-01 00:00:00'
+  AND log_time < TIMESTAMP '2017-01-01 00:00:00'
+GROUP BY machine_name
+ORDER BY spikes DESC
+LIMIT 10;
+
+
+-- Q1.5: Which externally reachable VMs have run low on memory?
+
+SELECT machine_name,
+       dt,
+       MIN(mem_free) AS mem_free_min
+FROM (
+  SELECT machine_name,
+         CAST(log_time AS DATE) AS dt,
+         mem_free
+  FROM logs1
+  WHERE machine_group = 'DMZ'
+    AND mem_free IS NOT NULL
+) AS r
+GROUP BY machine_name,
+         dt
+HAVING MIN(mem_free) < 10000
+ORDER BY machine_name,
+         dt;
+
+
+-- Q1.6: What is the total hourly network traffic across all file servers?
+
+SELECT dt,
+       hr,
+       SUM(net_in) AS net_in_sum,
+       SUM(net_out) AS net_out_sum,
+       SUM(net_in) + SUM(net_out) AS both_sum
+FROM (
+  SELECT CAST(log_time AS DATE) AS dt,
+         EXTRACT(HOUR FROM log_time) AS hr,
+         COALESCE(bytes_in, 0.0) / 1000000000.0 AS net_in,
+         COALESCE(bytes_out, 0.0) / 1000000000.0 AS net_out
+  FROM logs1
+  WHERE machine_name IN ('allsorts','andes','bigred','blackjack','bonbon',
+      'cadbury','chiclets','cotton','crows','dove','fireball','hearts','huey',
+      'lindt','milkduds','milkyway','mnm','necco','nerds','orbit','peeps',
+      'poprocks','razzles','runts','smarties','smuggler','spree','stride',
+      'tootsie','trident','wrigley','york')
+) AS r
+GROUP BY dt,
+         hr
+ORDER BY both_sum DESC
+LIMIT 10;
+
+
+-- Q2.1: Which requests have caused server errors within the past 2 weeks?
+
+SELECT *
+FROM logs2
+WHERE status_code >= 500
+  AND log_time >= TIMESTAMP '2012-12-18 00:00:00'
+ORDER BY log_time;
+
+
+-- Q2.2: During a specific 2-week period, was the user password file leaked?
+
+SELECT *
+FROM logs2
+WHERE status_code >= 200
+  AND status_code < 300
+  AND request LIKE '%/etc/passwd%'
+  AND log_time >= TIMESTAMP '2012-05-06 00:00:00'
+  AND log_time < TIMESTAMP '2012-05-20 00:00:00';
+
+
+-- Q2.3: What was the average path depth for top-level requests in the past month?
+
+SELECT top_level,
+       AVG(LENGTH(request) - LENGTH(REPLACE(request, '/', ''))) AS depth_avg
+FROM (
+  SELECT SUBSTRING(request FROM 1 FOR len) AS top_level,
+         request
+  FROM (
+    SELECT POSITION(SUBSTRING(request FROM 2), '/') AS len,
+           request
+    FROM logs2
+    WHERE status_code >= 200
+      AND status_code < 300
+      AND log_time >= TIMESTAMP '2012-12-01 00:00:00'
+  ) AS r
+  WHERE len > 0
+) AS s
+WHERE top_level IN ('/about','/courses','/degrees','/events',
+                    '/grad','/industry','/news','/people',
+                    '/publications','/research','/teaching','/ugrad')
+GROUP BY top_level
+ORDER BY top_level;
+
+
+-- Q2.4: During the last 3 months, which clients have made an excessive number of requests?
+
+SELECT client_ip,
+       COUNT(*) AS num_requests
+FROM logs2
+WHERE log_time >= TIMESTAMP '2012-10-01 00:00:00'
+GROUP BY client_ip
+HAVING COUNT(*) >= 100000
+ORDER BY num_requests DESC;
+
+
+-- Q2.5: What are the daily unique visitors?
+
+SELECT dt,
+       COUNT(DISTINCT client_ip)
+FROM (
+  SELECT CAST(log_time AS DATE) AS dt,
+         client_ip
+  FROM logs2
+) AS r
+GROUP BY dt
+ORDER BY dt;
+
+
+-- Q2.6: What are the average and maximum data transfer rates (Gbps)?
+
+SELECT AVG(transfer) / 125000000.0 AS transfer_avg,
+       MAX(transfer) / 125000000.0 AS transfer_max
+FROM (
+  SELECT log_time,
+         SUM(object_size) AS transfer
+  FROM logs2
+  GROUP BY log_time
+) AS r;
+
+
+-- Q3.1: Did the indoor temperature reach freezing over the weekend?
+
+SELECT *
+FROM logs3
+WHERE event_type = 'temperature'
+  AND event_value <= 32.0
+  AND log_time >= '2019-11-29 17:00:00.000';
+
+
+-- Q3.4: Over the past 6 months, how frequently was each door opened?
+
+SELECT device_name,
+       device_floor,
+       COUNT(*) AS ct
+FROM logs3
+WHERE event_type = 'door_open'
+  AND log_time >= '2019-06-01 00:00:00.000'
+GROUP BY device_name,
+         device_floor
+ORDER BY ct DESC;
+
+
+-- Q3.5: Where in the building do large temperature variations occur in winter and summer?
+
+WITH temperature AS (
+  SELECT dt,
+         device_name,
+         device_type,
+         device_floor
+  FROM (
+    SELECT dt,
+           hr,
+           device_name,
+           device_type,
+           device_floor,
+           AVG(event_value) AS temperature_hourly_avg
+    FROM (
+      SELECT CAST(log_time AS DATE) AS dt,
+             EXTRACT(HOUR FROM log_time) AS hr,
+             device_name,
+             device_type,
+             device_floor,
+             event_value
+      FROM logs3
+      WHERE event_type = 'temperature'
+    ) AS r
+    GROUP BY dt,
+             hr,
+             device_name,
+             device_type,
+             device_floor
+  ) AS s
+  GROUP BY dt,
+           device_name,
+           device_type,
+           device_floor
+  HAVING MAX(temperature_hourly_avg) - MIN(temperature_hourly_avg) >= 25.0
+)
+SELECT DISTINCT device_name,
+       device_type,
+       device_floor,
+       'WINTER'
+FROM temperature
+WHERE dt >= DATE '2018-12-01'
+  AND dt < DATE '2019-03-01'
+UNION
+SELECT DISTINCT device_name,
+       device_type,
+       device_floor,
+       'SUMMER'
+FROM temperature
+WHERE dt >= DATE '2019-06-01'
+  AND dt < DATE '2019-09-01';
+
+
+-- Q3.6: For each device category, what are the monthly power consumption metrics?
+
+SELECT yr,
+       mo,
+       SUM(coffee_hourly_avg) AS coffee_monthly_sum,
+       AVG(coffee_hourly_avg) AS coffee_monthly_avg,
+       SUM(printer_hourly_avg) AS printer_monthly_sum,
+       AVG(printer_hourly_avg) AS printer_monthly_avg,
+       SUM(projector_hourly_avg) AS projector_monthly_sum,
+       AVG(projector_hourly_avg) AS projector_monthly_avg,
+       SUM(vending_hourly_avg) AS vending_monthly_sum,
+       AVG(vending_hourly_avg) AS vending_monthly_avg
+FROM (
+  SELECT dt,
+         yr,
+         mo,
+         hr,
+         AVG(coffee) AS coffee_hourly_avg,
+         AVG(printer) AS printer_hourly_avg,
+         AVG(projector) AS projector_hourly_avg,
+         AVG(vending) AS vending_hourly_avg
+  FROM (
+    SELECT CAST(log_time AS DATE) AS dt,
+           EXTRACT(YEAR FROM log_time) AS yr,
+           EXTRACT(MONTH FROM log_time) AS mo,
+           EXTRACT(HOUR FROM log_time) AS hr,
+           CASE WHEN device_name LIKE 'coffee%' THEN event_value END AS coffee,
+           CASE WHEN device_name LIKE 'printer%' THEN event_value END AS printer,
+           CASE WHEN device_name LIKE 'projector%' THEN event_value END AS projector,
+           CASE WHEN device_name LIKE 'vending%' THEN event_value END AS vending
+    FROM logs3
+    WHERE device_type = 'meter'
+  ) AS r
+  GROUP BY dt,
+           yr,
+           mo,
+           hr
+) AS s
+GROUP BY yr,
+         mo
+ORDER BY yr,
+         mo;
+```
+
+The data is also available for interactive queries in the [Playground](https://gh-api.clickhouse.tech/play?user=play), [example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIG1hY2hpbmVfbmFtZSwKICAgICAgIE1JTihjcHUpIEFTIGNwdV9taW4sCiAgICAgICBNQVgoY3B1KSBBUyBjcHVfbWF4LAogICAgICAgQVZHKGNwdSkgQVMgY3B1X2F2ZywKICAgICAgIE1JTihuZXRfaW4pIEFTIG5ldF9pbl9taW4sCiAgICAgICBNQVgobmV0X2luKSBBUyBuZXRfaW5fbWF4LAogICAgICAgQVZHKG5ldF9pbikgQVMgbmV0X2luX2F2ZywKICAgICAgIE1JTihuZXRfb3V0KSBBUyBuZXRfb3V0X21pbiwKICAgICAgIE1BWChuZXRfb3V0KSBBUyBuZXRfb3V0X21heCwKICAgICAgIEFWRyhuZXRfb3V0KSBBUyBuZXRfb3V0X2F2ZwpGUk9NICgKICBTRUxFQ1QgbWFjaGluZV9uYW1lLAogICAgICAgICBDT0FMRVNDRShjcHVfdXNlciwgMC4wKSBBUyBjcHUsCiAgICAgICAgIENPQUxFU0NFKGJ5dGVzX2luLCAwLjApIEFTIG5ldF9pbiwKICAgICAgICAgQ09BTEVTQ0UoYnl0ZXNfb3V0LCAwLjApIEFTIG5ldF9vdXQKICBGUk9NIG1nYmVuY2gubG9nczEKICBXSEVSRSBtYWNoaW5lX25hbWUgSU4gKCdhbmFuc2knLCdhcmFnb2cnLCd1cmQnKQogICAgQU5EIGxvZ190aW1lID49IFRJTUVTVEFNUCAnMjAxNy0wMS0xMSAwMDowMDowMCcKKSBBUyByCkdST1VQIEJZIG1hY2hpbmVfbmFtZQ==).
--- a/docs/en/getting-started/example-datasets/index.md
+++ b/docs/en/getting-started/example-datasets/index.md
@ -13,11 +13,12 @@ The list of documented datasets:
 -   [GitHub Events](../../getting-started/example-datasets/github-events.md)
 -   [Anonymized Yandex.Metrica Dataset](../../getting-started/example-datasets/metrica.md)
 -   [Recipes](../../getting-started/example-datasets/recipes.md)
+-   [OnTime](../../getting-started/example-datasets/ontime.md)
+-   [New York Taxi Data](../../getting-started/example-datasets/nyc-taxi.md)
 -   [Star Schema Benchmark](../../getting-started/example-datasets/star-schema.md)
 -   [WikiStat](../../getting-started/example-datasets/wikistat.md)
 -   [Terabyte of Click Logs from Criteo](../../getting-started/example-datasets/criteo.md)
 -   [AMPLab Big Data Benchmark](../../getting-started/example-datasets/amplab-benchmark.md)
-   [New York Taxi Data](../../getting-started/example-datasets/nyc-taxi.md)
-   [OnTime](../../getting-started/example-datasets/ontime.md)
+-   [Brown University Benchmark](../../getting-started/example-datasets/brown-benchmark.md)

 [Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets) <!--hide-->
--- a/docs/en/getting-started/example-datasets/metrica.md
+++ b/docs/en/getting-started/example-datasets/metrica.md
@ -71,4 +71,4 @@ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1"

 [ClickHouse tutorial](../../getting-started/tutorial.md) is based on Yandex.Metrica dataset and the recommended way to get started with this dataset is to just go through tutorial.

-Additional examples of queries to these tables can be found among [stateful tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) of ClickHouse (they are named `test.hists` and `test.visits` there).
+Additional examples of queries to these tables can be found among [stateful tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) of ClickHouse (they are named `test.hits` and `test.visits` there).
--- a/docs/en/getting-started/example-datasets/ontime.md
+++ b/docs/en/getting-started/example-datasets/ontime.md
@ -398,6 +398,8 @@ ORDER BY c DESC
 LIMIT 10;
 ```

+You can also play with the data in Playground, [example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIERheU9mV2VlaywgY291bnQoKikgQVMgYwpGUk9NIG9udGltZQpXSEVSRSBZZWFyPj0yMDAwIEFORCBZZWFyPD0yMDA4CkdST1VQIEJZIERheU9mV2VlawpPUkRFUiBCWSBjIERFU0M7Cg==).
+
 This performance test was created by Vadim Tkachenko. See:

 -   https://www.percona.com/blog/2009/10/02/analyzing-air-traffic-performance-with-infobright-and-monetdb/
--- a/docs/en/interfaces/third-party/client-libraries.md
+++ b/docs/en/interfaces/third-party/client-libraries.md
@ -13,6 +13,7 @@ toc_title: Client Libraries
    -   [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver)
    -   [clickhouse-client](https://github.com/yurial/clickhouse-client)
    -   [aiochclient](https://github.com/maximdanilchenko/aiochclient)
+    -   [asynch](https://github.com/long2ice/asynch)
 -   PHP
    -   [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
    -   [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client)
--- a/docs/en/operations/system-tables/index.md
+++ b/docs/en/operations/system-tables/index.md
@ -20,7 +20,33 @@ System tables:

 Most of system tables store their data in RAM. A ClickHouse server creates such system tables at the start.

-Unlike other system tables, the system tables [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one.
+Unlike other system tables, the system log tables [metric_log](../../operations/system-tables/metric_log.md), [query_log](../../operations/system-tables/query_log.md), [query_thread_log](../../operations/system-tables/query_thread_log.md), [trace_log](../../operations/system-tables/trace_log.md), [part_log](../../operations/system-tables/part_log.md), crash_log and [text_log](../../operations/system-tables/text_log.md) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem by default. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one.
+
+System log tables can be customized by creating a config file with the same name as the table under `/etc/clickhouse-server/config.d/`, or setting corresponding elements in `/etc/clickhouse-server/config.xml`. Elements can be customized are:
+
+-   `database`: database the system log table belongs to. This option is deprecated now. All system log tables are under database `system`.
+-   `table`: table to insert data.
+-   `partition_by`: specify [PARTITION BY](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) expression.
+-   `ttl`: specify table [TTL](../../sql-reference/statements/alter/ttl.md) expression.
+-   `flush_interval_milliseconds`: interval of flushing data to disk.
+-   `engine`: provide full engine expression (starting with `ENGINE =` ) with parameters. This option is contradict with `partition_by` and `ttl`. If set together, the server would raise an exception and exit.
+
+An example:
+
+```
+<yandex>
+    <query_log>
+        <database>system</database>
+        <table>query_log</table>
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
+        <!--
+        <engine>ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024</engine>
+        -->
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </query_log>
+</yandex>
+```

 By default, table growth is unlimited. To control a size of a table, you can use [TTL](../../sql-reference/statements/alter/ttl.md#manipulations-with-table-ttl) settings for removing outdated log records. Also you can use the partitioning feature of `MergeTree`-engine tables.

--- a/docs/en/operations/system-tables/table_engines.md
+++ b/docs/en/operations/system-tables/table_engines.md
@ -11,6 +11,7 @@ This table contains the following columns (the column type is shown in brackets)
 -   `supports_sort_order` (UInt8) — Flag that indicates if table engine supports clauses `PARTITION_BY`, `PRIMARY_KEY`, `ORDER_BY` and `SAMPLE_BY`.
 -   `supports_replication` (UInt8) — Flag that indicates if table engine supports [data replication](../../engines/table-engines/mergetree-family/replication.md).
 -   `supports_duduplication` (UInt8) — Flag that indicates if table engine supports data deduplication.
+-   `supports_parallel_insert` (UInt8) — Flag that indicates if table engine supports parallel insert (see [`max_insert_threads`](../../operations/settings/settings.md#settings-max-insert-threads) setting).

 Example:

@ -21,11 +22,11 @@ WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree')
 ```

 ``` text
-┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┐
-│ Kafka                         │                 1 │                         0 │                   0 │            0 │                    0 │                      0 │
-│ MergeTree                     │                 1 │                         1 │                   1 │            1 │                    0 │                      0 │
-│ ReplicatedCollapsingMergeTree │                 1 │                         1 │                   1 │            1 │                    1 │                      1 │
-└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┘
+┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┬─supports_parallel_insert─┐
+│ MergeTree                     │                 1 │                         1 │                   1 │            1 │                    0 │                      0 │                        1 │
+│ Kafka                         │                 1 │                         0 │                   0 │            0 │                    0 │                      0 │                        0 │
+│ ReplicatedCollapsingMergeTree │                 1 │                         1 │                   1 │            1 │                    1 │                      1 │                        1 │
+└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┴──────────────────────────┘
 ```

 **See also**
--- a/docs/en/sql-reference/functions/array-functions.md
+++ b/docs/en/sql-reference/functions/array-functions.md
@ -1290,25 +1290,68 @@ Note that the `arrayFirstIndex` is a [higher-order function](../../sql-reference

 ## arrayMin(\[func,\] arr1, …) {#array-min}

-Returns the sum of the `func` values. If the function is omitted, it just returns the min of the array elements.
+Returns the min of the `func` values. If the function is omitted, it just returns the min of the array elements.

 Note that the `arrayMin` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.

+Examples:
+```sql
+SELECT arrayMin([1, 2, 4]) AS res
+┌─res─┐
+│   1 │
+└─────┘
+
+
+SELECT arrayMin(x -> (-x), [1, 2, 4]) AS res
+┌─res─┐
+│  -4 │
+└─────┘
+```
+
 ## arrayMax(\[func,\] arr1, …) {#array-max}

-Returns the sum of the `func` values. If the function is omitted, it just returns the min of the array elements.
+Returns the max of the `func` values. If the function is omitted, it just returns the max of the array elements.

 Note that the `arrayMax` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.

+Examples:
+```sql
+SELECT arrayMax([1, 2, 4]) AS res
+┌─res─┐
+│   4 │
+└─────┘
+
+
+SELECT arrayMax(x -> (-x), [1, 2, 4]) AS res
+┌─res─┐
+│  -1 │
+└─────┘
+```
+
 ## arraySum(\[func,\] arr1, …) {#array-sum}

 Returns the sum of the `func` values. If the function is omitted, it just returns the sum of the array elements.

 Note that the `arraySum` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.

+Examples:
+```sql
+SELECT arraySum([2,3]) AS res
+┌─res─┐
+│   5 │
+└─────┘
+
+
+SELECT arraySum(x -> x*x, [2, 3]) AS res
+┌─res─┐
+│  13 │
+└─────┘
+```
+
+
 ## arrayAvg(\[func,\] arr1, …) {#array-avg}

-Returns the sum of the `func` values. If the function is omitted, it just returns the average of the array elements.
+Returns the average of the `func` values. If the function is omitted, it just returns the average of the array elements.

 Note that the `arrayAvg` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.

--- a/docs/en/sql-reference/functions/other-functions.md
+++ b/docs/en/sql-reference/functions/other-functions.md
@ -182,6 +182,14 @@ If `NULL` is passed to the function as input, then it returns the `Nullable(Noth
 Gets the size of the block.
 In ClickHouse, queries are always run on blocks (sets of column parts). This function allows getting the size of the block that you called it for.

+## byteSize(...) {#function-bytesize}
+
+Get an estimate of uncompressed byte size of its arguments in memory.
+E.g. for UInt32 argument it will return constant 4, for String argument - the string length + 9 (terminating zero + length).
+The function can take multiple arguments. The typical application is byteSize(*).
+
+Use case: Suppose you have a service that stores data for multiple clients in one table. Users will pay per data volume. So, you need to implement accounting of users data volume. The function will allow to calculate the data size on per-row basis.
+
 ## materialize(x) {#materializex}

 Turns a constant into a full column containing just one value.
--- a/docs/es/interfaces/third-party/client-libraries.md
+++ b/docs/es/interfaces/third-party/client-libraries.md
@ -13,6 +13,7 @@ toc_title: Client Libraries
    -   [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver)
    -   [clickhouse-client](https://github.com/yurial/clickhouse-client)
    -   [aiochclient](https://github.com/maximdanilchenko/aiochclient)
+    -   [asynch](https://github.com/long2ice/asynch)
 -   PHP
    -   [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
    -   [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client)
--- a/docs/fr/interfaces/third-party/client-libraries.md
+++ b/docs/fr/interfaces/third-party/client-libraries.md
@ -15,6 +15,7 @@ toc_title: "Biblioth\xE8ques Clientes"
    -   [clickhouse-chauffeur](https://github.com/mymarilyn/clickhouse-driver)
    -   [clickhouse-client](https://github.com/yurial/clickhouse-client)
    -   [aiochclient](https://github.com/maximdanilchenko/aiochclient)
+    -   [asynch](https://github.com/long2ice/asynch)
 -   PHP
    -   [smi2 / phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
    -   [8bitov / clickhouse-PHP-client](https://packagist.org/packages/8bitov/clickhouse-php-client)
--- a/docs/ja/interfaces/third-party/client-libraries.md
+++ b/docs/ja/interfaces/third-party/client-libraries.md
@ -15,6 +15,7 @@ toc_title: "\u30AF\u30E9\u30A4\u30A2\u30F3\u30C8"
    -   [clickhouse-ドライバ](https://github.com/mymarilyn/clickhouse-driver)
    -   [clickhouse-クライアント](https://github.com/yurial/clickhouse-client)
    -   [aiochclient](https://github.com/maximdanilchenko/aiochclient)
+    -   [asynch](https://github.com/long2ice/asynch)
 -   PHP
    -   [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
    -   [8bitov/clickhouse-php-クライアント](https://packagist.org/packages/8bitov/clickhouse-php-client)
--- a/docs/ru/interfaces/third-party/client-libraries.md
+++ b/docs/ru/interfaces/third-party/client-libraries.md
@ -13,6 +13,7 @@ toc_title: "\u041a\u043b\u0438\u0435\u043d\u0442\u0441\u043a\u0438\u0435\u0020\u
    -   [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver)
    -   [clickhouse-client](https://github.com/yurial/clickhouse-client)
    -   [aiochclient](https://github.com/maximdanilchenko/aiochclient)
+    -   [asynch](https://github.com/long2ice/asynch)
 -   PHP
    -   [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
    -   [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client)
--- a/docs/tools/release.sh
+++ b/docs/tools/release.sh
@ -31,9 +31,11 @@ then
    git add *
    git add ".nojekyll"

-    # Push to GitHub rewriting the existing contents.
    git commit --quiet -m "Add new release at $(date)"
-    git push --force origin master
+
+    # Push to GitHub rewriting the existing contents.
+    # Sometimes it does not work with error message "! [remote rejected] master -> master (cannot lock ref 'refs/heads/master': is at 42a0f6b6b6c7be56a469441b4bf29685c1cebac3 but expected 520e9b02c0d4678a2a5f41d2f561e6532fb98cc1)"
+    for _ in {1..10}; do git push --force origin master && break; sleep 5; done

    if [[ ! -z "${CLOUDFLARE_TOKEN}" ]]
    then
--- a/docs/zh/development/style.md
+++ b/docs/zh/development/style.md
@ -267,7 +267,7 @@ void executeQuery(

 **9.** 多行注释的开头和结尾不得有空行（关闭多行注释的行除外）。

-**10.** 要注释掉代码，请使用基本注释，而不是«记录»注释。
+**10.** 要注释掉代码，请使用基本注释，而不是“文档”注释。

 **11.** 在提交之前删除代码的无效注释部分。

@ -335,7 +335,7 @@ template <bool without_www>
 struct ExtractDomain
 ```

-**7.** 对于抽象类型（接口），用 `I` 前缀。
+**7.** 对于抽象类（接口），用 `I` 前缀。

 ``` cpp
 class IBlockInputStream
@ -349,7 +349,7 @@ class IBlockInputStream
 bool info_successfully_loaded = false;
 ```

-**9.** `define` 和全局常量的名称使用带下划线的 `ALL_CAPS`。
+**9.** `define` 和全局常量的名称使用全大写带下划线的形式，如 `ALL_CAPS`。

 ``` cpp
 #define MAX_SRC_TABLE_NAMES_TO_STORE 1000
@ -357,14 +357,14 @@ bool info_successfully_loaded = false;

 **10.** 文件名应使用与其内容相同的样式。

-如果文件包含单个类，则以与该类名称相同的方式命名该文件。
+如果文件包含单个类，则以与该类名称相同的方式命名该文件（CamelCase）。

-如果文件包含单个函数，则以与函数名称相同的方式命名文件。
+如果文件包含单个函数，则以与函数名称相同的方式命名文件（camelCase）。

 **11.** 如果名称包含缩写，则：

 -   对于变量名，缩写应使用小写字母 `mysql_connection`（不是 `mySQL_connection` ）。
-   对于类和函数的名称，请将大写字母保留在缩写 `MySQLConnection`（不是 `MySqlConnection` 。
+-   对于类和函数的名称，请将大写字母保留在缩写 `MySQLConnection`（不是 `MySqlConnection`）。

 **12.** 仅用于初始化类成员的构造方法参数的命名方式应与类成员相同，但最后使用下划线。

@ -411,7 +411,7 @@ enum class CompressionMethod

 如果缩短版本是常用的，则可以接受不完整的单词。

-如果注释中旁边包含全名，您也可以使用缩写。
+如果旁边有注释包含全名，您也可以使用缩写。

 **17.** C++ 源码文件名称必须为 `.cpp` 拓展名。 头文件必须为 `.h` 拓展名。

@ -441,7 +441,7 @@ enum class CompressionMethod

 在离线数据处理应用程序中，通常可以接受不捕获异常。

-在处理用户请求的服务器中，通常足以捕获连接处理程序顶层的异常。
+在处理用户请求的服务器中，捕获连接处理程序顶层的异常通常就足够了。

 在线程函数中，你应该在 `join` 之后捕获并保留所有异常以在主线程中重新抛出它们。

@ -548,7 +548,7 @@ Fork不用于并行化。

 **10.** 常量。

-使用 const 引用，指向常量的指针，`const_iterator`和 const 指针。
+使用 const 引用、指针，指向常量、`const_iterator`和 const 方法。

 将 `const` 视为默认值，仅在必要时使用非 `const`。

@ -560,7 +560,7 @@ Fork不用于并行化。

 **12.** 数值类型。

-使用 `UInt8`， `UInt16`， `UInt32`， `UInt64`， `Int8`， `Int16`， `Int32`， 以及 `Int64`， `size_t`， `ssize_t` 还有 `ptrdiff_t`。
+使用 `UInt8`， `UInt16`， `UInt32`， `UInt64`， `Int8`， `Int16`， `Int32` 和 `Int64`，同样还有 `size_t`， `ssize_t` 和 `ptrdiff_t`。

 不要使用这些类型：`signed / unsigned long`，`long long`，`short`，`signed / unsigned char`，`char`。

@ -732,11 +732,11 @@ CPU指令集是我们服务器中支持的最小集合。 目前，它是SSE 4.2

 **8.** 尽可能经常地进行提交，即使代码只是部分准备好了。

-目的明确的功能，使用分支。
+为了这种目的可以创建分支。

-如果 `master` 分支中的代码尚不可构建，请在 `push` 之前将其从构建中排除。您需要在几天内完成或删除它。
+如果您的代码在 `master` 分支中尚不可构建，在 `push` 之前需要将其从构建中排除。您需要在几天内完成或删除它。

-**9.** 对于不重要的更改，请使用分支并在服务器上发布它们。
+**9.** 对于非一般的更改，请使用分支并在服务器上发布它们。

 **10.** 未使用的代码将从 repo 中删除。

--- a/docs/zh/interfaces/third-party/client-libraries.md
+++ b/docs/zh/interfaces/third-party/client-libraries.md
@ -13,6 +13,7 @@ Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试
    -   [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver)
    -   [clickhouse-client](https://github.com/yurial/clickhouse-client)
    -   [aiochclient](https://github.com/maximdanilchenko/aiochclient)
+    -   [asynch](https://github.com/long2ice/asynch)
 -   PHP
    -   [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
    -   [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client)
--- a/docs/zh/operations/system-tables/index.md
+++ b/docs/zh/operations/system-tables/index.md
@ -22,9 +22,35 @@ toc_title: "\u7CFB\u7EDF\u8868"

 大多数系统表将数据存储在RAM中。 ClickHouse服务器在开始时创建此类系统表。

-与其他系统表不同，系统表 [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) 由 [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 表引擎并将其数据存储在存储文件系统中。 如果从文件系统中删除表，ClickHouse服务器会在下一次写入数据时再次创建空表。 如果系统表架构在新版本中发生更改，则ClickHouse会重命名当前表并创建一个新表。
+与其他系统表不同，系统日志表 [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log), [part_log](../../operations/system-tables/part_log.md#system.part_log), crash_log and text_log 默认采用[MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 引擎并将其数据存储在存储文件系统中。 如果从文件系统中删除表，ClickHouse服务器会在下一次写入数据时再次创建空表。 如果系统表架构在新版本中发生更改，则ClickHouse会重命名当前表并创建一个新表。

-默认情况下，表增长是无限的。 要控制表的大小，可以使用 [TTL](../../sql-reference/statements/alter.md#manipulations-with-table-ttl) 删除过期日志记录的设置。 你也可以使用分区功能 `MergeTree`-发动机表。
+用户可以通过在`/etc/clickhouse-server/config.d/`下创建与系统表同名的配置文件, 或者在`/etc/clickhouse-server/config.xml`中设置相应配置项，来自定义系统日志表的结构。可以自定义的配置项如下:
+
+-   `database`: 系统日志表所在的数据库。这个选项目前已经废弃。所有的系统日表都位于`system`库中。
+-   `table`: 系统日志表名。
+-   `partition_by`: 指定[PARTITION BY](../../engines/table-engines/mergetree-family/custom-partitioning-key.md)表达式。
+-   `ttl`: 指定系统日志表TTL选项。
+-   `flush_interval_milliseconds`: 指定系统日志表数据落盘时间。
+-   `engine`: 指定完整的表引擎定义。(以`ENGINE = `开始)。 这个选项与`partition_by`以及`ttl`冲突。如果两者一起设置，服务启动时会抛出异常并且退出。
+
+一个配置定义的例子如下：
+
+```
+<yandex>
+    <query_log>
+        <database>system</database>
+        <table>query_log</table>
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
+        <!--
+        <engine>ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024</engine>
+        -->
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </query_log>
+</yandex>
+```
+
+默认情况下，表增长是无限的。 要控制表的大小，可以使用 TTL 删除过期日志记录的设置。 你也可以使用分区功能 `MergeTree`-发动机表。

 ## 系统指标的来源 {#system-tables-sources-of-system-metrics}

--- a/programs/CMakeLists.txt
+++ b/programs/CMakeLists.txt
@ -216,7 +216,13 @@ if (CLICKHOUSE_SPLIT_BINARY)
    install(PROGRAMS clickhouse-split-helper DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME clickhouse COMPONENT clickhouse)
 else ()
    add_executable (clickhouse main.cpp)
-    target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils)
+
+    # A library that prevent usage of several functions from libc.
+    if (ARCH_AMD64 AND OS_LINUX AND NOT OS_ANDROID)
+        set (HARMFUL_LIB harmful)
+    endif ()
+
+    target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils ${HARMFUL_LIB})
    target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR})

    if (ENABLE_CLICKHOUSE_SERVER)
@ -312,6 +318,10 @@ else ()
    if (USE_GDB_ADD_INDEX)
        add_custom_command(TARGET clickhouse POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} clickhouse COMMENT "Adding .gdb-index to clickhouse" VERBATIM)
    endif()
+
+    if (USE_BINARY_HASH)
+        add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .note.ClickHouse.hash=hash clickhouse COMMENT "Adding .note.ClickHouse.hash to clickhouse" VERBATIM)
+    endif()
 endif ()

 if (ENABLE_TESTS AND USE_GTEST)
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@ -59,7 +59,9 @@
 #include <DataStreams/AsynchronousBlockInputStream.h>
 #include <DataStreams/AddingDefaultsBlockInputStream.h>
 #include <DataStreams/InternalTextLogsRowOutputStream.h>
+#include <DataStreams/NullBlockOutputStream.h>
 #include <Parsers/ASTCreateQuery.h>
+#include <Parsers/ASTDropQuery.h>
 #include <Parsers/ASTSetQuery.h>
 #include <Parsers/ASTUseQuery.h>
 #include <Parsers/ASTInsertQuery.h>
@ -110,6 +112,7 @@ namespace ErrorCodes
    extern const int INVALID_USAGE_OF_INPUT;
    extern const int DEADLOCK_AVOIDED;
    extern const int UNRECOGNIZED_ARGUMENTS;
+    extern const int SYNTAX_ERROR;
 }


@ -136,6 +139,9 @@ private:
    bool stdin_is_a_tty = false;         /// stdin is a terminal.
    bool stdout_is_a_tty = false;        /// stdout is a terminal.

+    /// If not empty, queries will be read from these files
+    std::vector<std::string> queries_files;
+
    std::unique_ptr<Connection> connection;    /// Connection to DB.
    String full_query; /// Current query as it was given to the client.

@ -478,10 +484,10 @@ private:
        /// - stdin is not a terminal. In this case queries are read from it.
        /// - -qf (--queries-file) command line option is present.
        ///   The value of the option is used as file with query (or of multiple queries) to execute.
-        if (!stdin_is_a_tty || config().has("query") || config().has("queries-file"))
+        if (!stdin_is_a_tty || config().has("query") || !queries_files.empty())
            is_interactive = false;

-        if (config().has("query") && config().has("queries-file"))
+        if (config().has("query") && !queries_files.empty())
        {
            throw Exception("Specify either `query` or `queries-file` option", ErrorCodes::BAD_ARGUMENTS);
        }
@ -696,14 +702,8 @@ private:
            auto query_id = config().getString("query_id", "");
            if (!query_id.empty())
                context.setCurrentQueryId(query_id);
-            if (query_fuzzer_runs)
-            {
-                nonInteractiveWithFuzzing();
-            }
-            else
-            {
-                nonInteractive();
-            }
+
+            nonInteractive();

            /// If exception code isn't zero, we should return non-zero return code anyway.
            if (last_exception_received_from_server)
@ -794,15 +794,22 @@ private:
    {
        String text;

-        if (config().has("queries-file"))
+        if (!queries_files.empty())
        {
-            ReadBufferFromFile in(config().getString("queries-file"));
-            readStringUntilEOF(text, in);
-            processMultiQuery(text);
+            for (const auto & queries_file : queries_files)
+            {
+                connection->setDefaultDatabase(connection_parameters.default_database);
+                ReadBufferFromFile in(queries_file);
+                readStringUntilEOF(text, in);
+                if (!processMultiQuery(text))
+                    break;
+            }
            return;
        }
        else if (config().has("query"))
+        {
            text = config().getRawString("query"); /// Poco configuration should not process substitutions in form of ${...} inside query.
+        }
        else
        {
            /// If 'query' parameter is not set, read a query from stdin.
@ -811,113 +818,10 @@ private:
            readStringUntilEOF(text, in);
        }

-        processQueryText(text);
-    }
-
-    void nonInteractiveWithFuzzing()
-    {
-        if (config().has("query"))
-        {
-            // Poco configuration should not process substitutions in form of
-            // ${...} inside query
-            processWithFuzzing(config().getRawString("query"));
-            return;
-        }
-
-        // Try to stream the queries from stdin, without reading all of them
-        // into memory. The interface of the parser does not support streaming,
-        // in particular, it can't distinguish the end of partial input buffer
-        // and the final end of input file. This means we have to try to split
-        // the input into separate queries here. Two patterns of input are
-        // especially interesting:
-        // 1) multiline query:
-        //      select 1
-        //      from system.numbers;
-        //
-        // 2) csv insert with in-place data:
-        //      insert into t format CSV 1;2
-        //
-        // (1) means we can't split on new line, and (2) means we can't split on
-        // semicolon. Solution: split on ';\n'. This sequence is frequent enough
-        // in the SQL tests which are our principal input for fuzzing. Now we
-        // have another interesting case:
-        // 3) escaped semicolon followed by newline, e.g.
-        //      select ';
-        //          '
-        //
-        // To handle (3), parse until we can, and read more data if the parser
-        // complains. Hopefully this should be enough...
-        ReadBufferFromFileDescriptor in(STDIN_FILENO);
-        std::string text;
-        while (!in.eof())
-        {
-            // Read until separator.
-            while (!in.eof())
-            {
-                char * next_separator = find_first_symbols<';'>(in.position(),
-                    in.buffer().end());
-
-                if (next_separator < in.buffer().end())
-                {
-                    next_separator++;
-                    if (next_separator < in.buffer().end()
-                        && *next_separator == '\n')
-                    {
-                        // Found ';\n', append it to the query text and try to
-                        // parse.
-                        next_separator++;
-                        text.append(in.position(), next_separator - in.position());
-                        in.position() = next_separator;
-                        break;
-                    }
-                }
-
-                // Didn't find the semicolon and reached the end of buffer.
-                text.append(in.position(), next_separator - in.position());
-                in.position() = next_separator;
-
-                if (text.size() > 1024 * 1024)
-                {
-                    // We've read a lot of text and still haven't seen a separator.
-                    // Likely some pathological input, just fall through to prevent
-                    // too long loops.
-                    break;
-                }
-            }
-
-            // Parse and execute what we've read.
-            const auto * new_end = processWithFuzzing(text);
-
-            if (new_end > &text[0])
-            {
-                const auto rest_size = text.size() - (new_end - &text[0]);
-
-                memcpy(&text[0], new_end, rest_size);
-                text.resize(rest_size);
-            }
-            else
-            {
-                // We didn't read enough text to parse a query. Will read more.
-            }
-
-            // Ensure that we're still connected to the server. If the server died,
-            // the reconnect is going to fail with an exception, and the fuzzer
-            // will exit. The ping() would be the best match here, but it's
-            // private, probably for a good reason that the protocol doesn't allow
-            // pings at any possible moment.
-            // Don't forget to reset the default database which might have changed.
-            connection->setDefaultDatabase("");
-            connection->forceConnected(connection_parameters.timeouts);
-
-            if (text.size() > 4 * 1024)
-            {
-                // Some pathological situation where the text is larger than 4kB
-                // and we still cannot parse a single query in it. Abort.
-                std::cerr << "Read too much text and still can't parse a query."
-                     " Aborting." << std::endl;
-                exit(1);
-            }
-        }
+        if (query_fuzzer_runs)
+            processWithFuzzing(text);
+        else
+            processQueryText(text);
    }

    bool processQueryText(const String & text)
@ -945,7 +849,8 @@ private:
    {
        const bool test_mode = config().has("testmode");

-        {   /// disable logs if expects errors
+        {
+            /// disable logs if expects errors
            TestHint test_hint(test_mode, all_queries_text);
            if (test_hint.clientError() || test_hint.serverError())
                processTextAsSingleQuery("SET send_logs_level = 'fatal'");
@ -1019,7 +924,7 @@ private:
                if (hint.clientError() != e.code())
                {
                    if (hint.clientError())
-                        e.addMessage("\nExpected clinet error: " + std::to_string(hint.clientError()));
+                        e.addMessage("\nExpected client error: " + std::to_string(hint.clientError()));
                    throw;
                }

@ -1078,40 +983,51 @@ private:
            expected_client_error = test_hint.clientError();
            expected_server_error = test_hint.serverError();

-            try
+            if (query_fuzzer_runs)
            {
-                processParsedSingleQuery();
-
-                if (insert_ast && insert_ast->data)
-                {
-                    // For VALUES format: use the end of inline data as reported
-                    // by the format parser (it is saved in sendData()). This
-                    // allows us to handle queries like:
-                    //   insert into t values (1); select 1
-                    //, where the inline data is delimited by semicolon and not
-                    // by a newline.
-                    this_query_end = parsed_query->as<ASTInsertQuery>()->end;
-                }
-            }
-            catch (...)
-            {
-                last_exception_received_from_server = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
-                actual_client_error = last_exception_received_from_server->code();
-                if (!ignore_error && (!actual_client_error || actual_client_error != expected_client_error))
-                    std::cerr << "Error on processing query: " << full_query << std::endl << last_exception_received_from_server->message();
-                received_exception_from_server = true;
-            }
-
-            if (!test_hint.checkActual(actual_server_error, actual_client_error, received_exception_from_server, last_exception_received_from_server))
-                connection->forceConnected(connection_parameters.timeouts);
-
-            if (received_exception_from_server && !ignore_error)
-            {
-                if (is_interactive)
-                    break;
-                else
+                if (!processWithFuzzing(full_query))
                    return false;
            }
+            else
+            {
+                try
+                {
+                    processParsedSingleQuery();
+
+                    if (insert_ast && insert_ast->data)
+                    {
+                        // For VALUES format: use the end of inline data as reported
+                        // by the format parser (it is saved in sendData()). This
+                        // allows us to handle queries like:
+                        //   insert into t values (1); select 1
+                        //, where the inline data is delimited by semicolon and not
+                        // by a newline.
+                        this_query_end = parsed_query->as<ASTInsertQuery>()->end;
+                    }
+                }
+                catch (...)
+                {
+                    last_exception_received_from_server = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
+                    actual_client_error = last_exception_received_from_server->code();
+                    if (!ignore_error && (!actual_client_error || actual_client_error != expected_client_error))
+                        std::cerr << "Error on processing query: " << full_query << std::endl << last_exception_received_from_server->message();
+                    received_exception_from_server = true;
+                }
+
+                if (!test_hint.checkActual(
+                    actual_server_error, actual_client_error, received_exception_from_server, last_exception_received_from_server))
+                {
+                    connection->forceConnected(connection_parameters.timeouts);
+                }
+
+                if (received_exception_from_server && !ignore_error)
+                {
+                    if (is_interactive)
+                        break;
+                    else
+                        return false;
+                }
+            }

            this_query_begin = this_query_end;
        }
@ -1120,165 +1036,148 @@ private:
    }


-    // Returns the last position we could parse.
-    const char * processWithFuzzing(const String & text)
+    /// Returns false when server is not available.
+    bool processWithFuzzing(const String & text)
    {
-        /// Several queries separated by ';'.
-        /// INSERT data is ended by the end of line, not ';'.
+        ASTPtr orig_ast;

-        const char * begin = text.data();
-        const char * end = begin + text.size();
-
-        while (begin < end)
+        try
        {
-            // Skip whitespace before the query
-            while (isWhitespaceASCII(*begin) || *begin == ';')
+            const char * begin = text.data();
+            orig_ast = parseQuery(begin, begin + text.size(), true);
+        }
+        catch (const Exception & e)
+        {
+            if (e.code() != ErrorCodes::SYNTAX_ERROR)
+                throw;
+        }
+
+        if (!orig_ast)
+        {
+            // Can't continue after a parsing error
+            return true;
+        }
+
+        // Don't repeat inserts, the tables grow too big. Also don't repeat
+        // creates because first we run the unmodified query, it will succeed,
+        // and the subsequent queries will fail. When we run out of fuzzer
+        // errors, it may be interesting to add fuzzing of create queries that
+        // wraps columns into LowCardinality or Nullable. Also there are other
+        // kinds of create queries such as CREATE DICTIONARY, we could fuzz
+        // them as well. Also there is no point fuzzing DROP queries.
+        size_t this_query_runs = query_fuzzer_runs;
+        if (orig_ast->as<ASTInsertQuery>() || orig_ast->as<ASTCreateQuery>() || orig_ast->as<ASTDropQuery>())
+        {
+            this_query_runs = 1;
+        }
+
+        ASTPtr fuzz_base = orig_ast;
+        for (size_t fuzz_step = 0; fuzz_step < this_query_runs; ++fuzz_step)
+        {
+            fmt::print(stderr, "Fuzzing step {} out of {}\n",
+                fuzz_step, this_query_runs);
+
+            ASTPtr ast_to_process;
+            try
            {
-                ++begin;
+                WriteBufferFromOwnString dump_before_fuzz;
+                fuzz_base->dumpTree(dump_before_fuzz);
+                auto base_before_fuzz = fuzz_base->formatForErrorMessage();
+
+                ast_to_process = fuzz_base->clone();
+
+                WriteBufferFromOwnString dump_of_cloned_ast;
+                ast_to_process->dumpTree(dump_of_cloned_ast);
+
+                // Run the original query as well.
+                if (fuzz_step > 0)
+                {
+                    fuzzer.fuzzMain(ast_to_process);
+                }
+
+                auto base_after_fuzz = fuzz_base->formatForErrorMessage();
+
+                // Debug AST cloning errors.
+                if (base_before_fuzz != base_after_fuzz)
+                {
+                    fmt::print(stderr,
+                        "Base before fuzz: {}\n"
+                        "Base after fuzz: {}\n",
+                        base_before_fuzz, base_after_fuzz);
+                    fmt::print(stderr, "Dump before fuzz:\n{}\n", dump_before_fuzz.str());
+                    fmt::print(stderr, "Dump of cloned AST:\n{}\n", dump_of_cloned_ast.str());
+                    fmt::print(stderr, "Dump after fuzz:\n");
+
+                    WriteBufferFromOStream cerr_buf(std::cerr, 4096);
+                    fuzz_base->dumpTree(cerr_buf);
+                    cerr_buf.next();
+
+                    fmt::print(stderr, "IAST::clone() is broken for some AST node. This is a bug. The original AST ('dump before fuzz') and its cloned copy ('dump of cloned AST') refer to the same nodes, which must never happen. This means that their parent node doesn't implement clone() correctly.");
+
+                    assert(false);
+                }
+
+                auto fuzzed_text = ast_to_process->formatForErrorMessage();
+                if (fuzz_step > 0 && fuzzed_text == base_before_fuzz)
+                {
+                    fmt::print(stderr, "Got boring AST\n");
+                    continue;
+                }
+
+                parsed_query = ast_to_process;
+                query_to_send = parsed_query->formatForErrorMessage();
+
+                processParsedSingleQuery();
+            }
+            catch (...)
+            {
+                // Some functions (e.g. protocol parsers) don't throw, but
+                // set last_exception instead, so we'll also do it here for
+                // uniformity.
+                last_exception_received_from_server = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
+                received_exception_from_server = true;
            }

-            const auto * this_query_begin = begin;
-            ASTPtr orig_ast = parseQuery(begin, end, true);
-
-            if (!orig_ast)
+            if (received_exception_from_server)
            {
-                // Can't continue after a parsing error
-                return begin;
+                fmt::print(stderr, "Error on processing query '{}': {}\n",
+                    ast_to_process->formatForErrorMessage(),
+                    last_exception_received_from_server->message());
            }

-            auto * as_insert = orig_ast->as<ASTInsertQuery>();
-            if (as_insert && as_insert->data)
+            if (!connection->isConnected())
            {
-                // INSERT data is ended by newline
-                as_insert->end = find_first_symbols<'\n'>(as_insert->data, end);
-                begin = as_insert->end;
+                // Probably the server is dead because we found an assertion
+                // failure. Fail fast.
+                fmt::print(stderr, "Lost connection to the server\n");
+                return false;
            }

-            full_query = text.substr(this_query_begin - text.data(),
-                begin - text.data());
-
-            // Don't repeat inserts, the tables grow too big. Also don't repeat
-            // creates because first we run the unmodified query, it will succeed,
-            // and the subsequent queries will fail. When we run out of fuzzer
-            // errors, it may be interesting to add fuzzing of create queries that
-            // wraps columns into LowCardinality or Nullable. Also there are other
-            // kinds of create queries such as CREATE DICTIONARY, we could fuzz
-            // them as well.
-            int this_query_runs = query_fuzzer_runs;
-            if (as_insert
-                || orig_ast->as<ASTCreateQuery>())
+            // The server is still alive so we're going to continue fuzzing.
+            // Determine what we're going to use as the starting AST.
+            if (received_exception_from_server)
            {
-                this_query_runs = 1;
+                // Query completed with error, keep the previous starting AST.
+                // Also discard the exception that we now know to be non-fatal,
+                // so that it doesn't influence the exit code.
+                last_exception_received_from_server.reset(nullptr);
+                received_exception_from_server = false;
            }
-
-            ASTPtr fuzz_base = orig_ast;
-            for (int fuzz_step = 0; fuzz_step < this_query_runs; fuzz_step++)
+            else if (ast_to_process->formatForErrorMessage().size() > 500)
            {
-                fprintf(stderr, "fuzzing step %d out of %d for query at pos %zd\n",
-                    fuzz_step, this_query_runs, this_query_begin - text.data());
-
-                ASTPtr ast_to_process;
-                try
-                {
-                    WriteBufferFromOwnString dump_before_fuzz;
-                    fuzz_base->dumpTree(dump_before_fuzz);
-                    auto base_before_fuzz = fuzz_base->formatForErrorMessage();
-
-                    ast_to_process = fuzz_base->clone();
-
-                    WriteBufferFromOwnString dump_of_cloned_ast;
-                    ast_to_process->dumpTree(dump_of_cloned_ast);
-
-                    // Run the original query as well.
-                    if (fuzz_step > 0)
-                    {
-                        fuzzer.fuzzMain(ast_to_process);
-                    }
-
-                    auto base_after_fuzz = fuzz_base->formatForErrorMessage();
-
-                    // Debug AST cloning errors.
-                    if (base_before_fuzz != base_after_fuzz)
-                    {
-                        fprintf(stderr, "base before fuzz: %s\n"
-                            "base after fuzz: %s\n", base_before_fuzz.c_str(),
-                            base_after_fuzz.c_str());
-                        fprintf(stderr, "dump before fuzz:\n%s\n",
-                            dump_before_fuzz.str().c_str());
-                        fprintf(stderr, "dump of cloned ast:\n%s\n",
-                            dump_of_cloned_ast.str().c_str());
-                        fprintf(stderr, "dump after fuzz:\n");
-                        WriteBufferFromOStream cerr_buf(std::cerr, 4096);
-                        fuzz_base->dumpTree(cerr_buf);
-                        cerr_buf.next();
-
-                        fmt::print(stderr, "IAST::clone() is broken for some AST node. This is a bug. The original AST ('dump before fuzz') and its cloned copy ('dump of cloned AST') refer to the same nodes, which must never happen. This means that their parent node doesn't implement clone() correctly.");
-
-                        assert(false);
-                    }
-
-                    auto fuzzed_text = ast_to_process->formatForErrorMessage();
-                    if (fuzz_step > 0 && fuzzed_text == base_before_fuzz)
-                    {
-                        fprintf(stderr, "got boring ast\n");
-                        continue;
-                    }
-
-                    parsed_query = ast_to_process;
-                    query_to_send = parsed_query->formatForErrorMessage();
-
-                    processParsedSingleQuery();
-                }
-                catch (...)
-                {
-                    // Some functions (e.g. protocol parsers) don't throw, but
-                    // set last_exception instead, so we'll also do it here for
-                    // uniformity.
-                    last_exception_received_from_server = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
-                    received_exception_from_server = true;
-                }
-
-                if (received_exception_from_server)
-                {
-                    fmt::print(stderr, "Error on processing query '{}': {}\n",
-                        ast_to_process->formatForErrorMessage(),
-                        last_exception_received_from_server->message());
-                }
-
-                if (!connection->isConnected())
-                {
-                    // Probably the server is dead because we found an assertion
-                    // failure. Fail fast.
-                    fmt::print(stderr, "Lost connection to the server\n");
-                    return begin;
-                }
-
-                // The server is still alive so we're going to continue fuzzing.
-                // Determine what we're going to use as the starting AST.
-                if (received_exception_from_server)
-                {
-                    // Query completed with error, keep the previous starting AST.
-                    // Also discard the exception that we now know to be non-fatal,
-                    // so that it doesn't influence the exit code.
-                    last_exception_received_from_server.reset(nullptr);
-                    received_exception_from_server = false;
-                }
-                else if (ast_to_process->formatForErrorMessage().size() > 500)
-                {
-                    // ast too long, start from original ast
-                    fprintf(stderr, "Current AST is too long, discarding it and using the original AST as a start\n");
-                    fuzz_base = orig_ast;
-                }
-                else
-                {
-                    // fuzz starting from this successful query
-                    fprintf(stderr, "Query succeeded, using this AST as a start\n");
-                    fuzz_base = ast_to_process;
-                }
+                // ast too long, start from original ast
+                fmt::print(stderr, "Current AST is too long, discarding it and using the original AST as a start\n");
+                fuzz_base = orig_ast;
+            }
+            else
+            {
+                // fuzz starting from this successful query
+                fmt::print(stderr, "Query succeeded, using this AST as a start\n");
+                fuzz_base = ast_to_process;
            }
        }

-        return begin;
+        return true;
    }

    void processTextAsSingleQuery(const String & text_)
@ -1891,6 +1790,13 @@ private:
    {
        if (!block_out_stream)
        {
+            /// Ignore all results when fuzzing as they can be huge.
+            if (query_fuzzer_runs)
+            {
+                block_out_stream = std::make_shared<NullBlockOutputStream>(block);
+                return;
+            }
+
            WriteBuffer * out_buf = nullptr;
            String pager = config().getString("pager", "");
            if (!pager.empty())
@ -1934,7 +1840,12 @@ private:
            if (has_vertical_output_suffix)
                current_format = "Vertical";

-            block_out_stream = context.getOutputFormat(current_format, *out_buf, block);
+            /// It is not clear how to write progress with parallel formatting. It may increase code complexity significantly.
+            if (!need_render_progress)
+                block_out_stream = context.getOutputStreamParallelIfPossible(current_format, *out_buf, block);
+            else
+                block_out_stream = context.getOutputStream(current_format, *out_buf, block);
+
            block_out_stream->writePrefix();
        }
    }
@ -1991,15 +1902,18 @@ private:
            written_first_block = true;
        }

-        bool clear_progess = std_out.offset() > 0;
-        if (clear_progess)
+        bool clear_progress = false;
+        if (need_render_progress)
+            clear_progress = std_out.offset() > 0;
+
+        if (clear_progress)
            clearProgress();

        /// Received data block is immediately displayed to the user.
        block_out_stream->flush();

        /// Restore progress bar after data block.
-        if (clear_progess)
+        if (clear_progress)
            writeProgress();
    }

@ -2340,7 +2254,8 @@ public:
                "Suggestion limit for how many databases, tables and columns to fetch.")
            ("multiline,m", "multiline")
            ("multiquery,n", "multiquery")
-            ("queries-file", po::value<std::string>(), "file path with queries to execute")
+            ("queries-file", po::value<std::vector<std::string>>()->multitoken(),
+                "file path with queries to execute; multiple files can be specified (--queries-file file1 file2...)")
            ("format,f", po::value<std::string>(), "default output format")
            ("testmode,T", "enable test hints in comments")
            ("ignore-error", "do not stop processing in multiquery mode")
@ -2470,12 +2385,11 @@ public:
        if (options.count("query"))
            config().setString("query", options["query"].as<std::string>());
        if (options.count("queries-file"))
-            config().setString("queries-file", options["queries-file"].as<std::string>());
+            queries_files = options["queries-file"].as<std::vector<std::string>>();
        if (options.count("database"))
            config().setString("database", options["database"].as<std::string>());
        if (options.count("pager"))
            config().setString("pager", options["pager"].as<std::string>());
-
        if (options.count("port") && !options["port"].defaulted())
            config().setInt("port", options["port"].as<int>());
        if (options.count("secure"))
@ -2529,7 +2443,6 @@ public:
            config().setBool("multiquery", true);

            // Ignore errors in parsing queries.
-            // TODO stop using parseQuery.
            config().setBool("ignore-error", true);
            ignore_error = true;
        }
--- a/programs/compressor/Compressor.cpp
+++ b/programs/compressor/Compressor.cpp
@ -169,11 +169,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)

            if (offset_in_compressed_file || offset_in_decompressed_block)
            {
-                if (!options.count("input"))
-                {
-                    throw DB::Exception("--offset-in-compressed-file/--offset-in-decompressed-block requires --input", DB::ErrorCodes::BAD_ARGUMENTS);
-                }
-                CompressedReadBufferFromFile compressed_file(options["input"].as<std::string>(), 0, 0, 0);
+                CompressedReadBufferFromFile compressed_file(std::move(rb));
                compressed_file.seek(offset_in_compressed_file, offset_in_decompressed_block);
                copyData(compressed_file, *wb);
            }
--- a/programs/install/Install.cpp
+++ b/programs/install/Install.cpp
@ -21,6 +21,7 @@
 #include <Common/OpenSSLHelpers.h>
 #include <Common/hex.h>
 #include <common/getResource.h>
+#include <common/sleep.h>
 #include <IO/ReadBufferFromFileDescriptor.h>
 #include <IO/WriteBufferFromFileDescriptor.h>
 #include <IO/ReadBufferFromFile.h>
@ -763,7 +764,7 @@ namespace
                fmt::print("Server started\n");
                break;
            }
-            ::sleep(1);
+            sleepForSeconds(1);
        }

        if (try_num == num_tries)
@ -875,7 +876,7 @@ namespace
                fmt::print("Server stopped\n");
                break;
            }
-            ::sleep(1);
+            sleepForSeconds(1);
        }

        if (try_num == num_tries)
--- a/programs/main.cpp
+++ b/programs/main.cpp
@ -18,6 +18,7 @@
 #endif

 #include <Common/StringUtils/StringUtils.h>
+#include <Common/getHashOfLoadedBinary.h>

 #include <common/phdr_cache.h>
 #include <ext/scope_guard.h>
@ -62,6 +63,14 @@ int mainEntryClickHouseStatus(int argc, char ** argv);
 int mainEntryClickHouseRestart(int argc, char ** argv);
 #endif

+int mainEntryClickHouseHashBinary(int, char **)
+{
+    /// Intentionally without newline. So you can run:
+    /// objcopy --add-section .note.ClickHouse.hash=<(./clickhouse hash-binary) clickhouse
+    std::cout << getHashOfLoadedBinaryHex();
+    return 0;
+}
+
 #define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0]))

 namespace
@ -110,6 +119,7 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
    {"status", mainEntryClickHouseStatus},
    {"restart", mainEntryClickHouseRestart},
 #endif
+    {"hash-binary", mainEntryClickHouseHashBinary},
 };


--- a/programs/obfuscator/Obfuscator.cpp
+++ b/programs/obfuscator/Obfuscator.cpp
@ -1180,7 +1180,7 @@ try
        file_in.seek(0, SEEK_SET);

        BlockInputStreamPtr input = context.getInputFormat(input_format, file_in, header, max_block_size);
-        BlockOutputStreamPtr output = context.getOutputFormat(output_format, file_out, header);
+        BlockOutputStreamPtr output = context.getOutputStream(output_format, file_out, header);

        if (processed_rows + source_rows > limit)
            input = std::make_shared<LimitBlockInputStream>(input, limit - processed_rows, 0);
--- a/programs/odbc-bridge/MainHandler.cpp
+++ b/programs/odbc-bridge/MainHandler.cpp
@ -1,24 +1,26 @@
 #include "MainHandler.h"

 #include "validateODBCConnectionString.h"
-#include <memory>
-#include <DataStreams/copyData.h>
-#include <DataTypes/DataTypeFactory.h>
 #include "ODBCBlockInputStream.h"
 #include "ODBCBlockOutputStream.h"
+#include "getIdentifierQuote.h"
+#include <DataStreams/copyData.h>
+#include <DataTypes/DataTypeFactory.h>
 #include <Formats/FormatFactory.h>
 #include <IO/WriteBufferFromHTTPServerResponse.h>
 #include <IO/WriteHelpers.h>
 #include <IO/ReadHelpers.h>
+#include <IO/ReadBufferFromIStream.h>
 #include <Poco/Net/HTTPServerRequest.h>
 #include <Poco/Net/HTTPServerResponse.h>
 #include <Poco/Net/HTMLForm.h>
-#include <common/logger_useful.h>
-#include <mutex>
 #include <Poco/ThreadPool.h>
-#include <IO/ReadBufferFromIStream.h>
-#include <Columns/ColumnsNumber.h>
-#include "getIdentifierQuote.h"
+#include <Processors/Formats/InputStreamFromInputFormat.h>
+#include <common/logger_useful.h>
+
+#include <mutex>
+#include <memory>
+

 #if USE_ODBC
 #include <Poco/Data/ODBC/SessionImpl.h>
@ -162,8 +164,9 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne

            auto pool = getPool(connection_string);
            ReadBufferFromIStream read_buf(request.stream());
-            BlockInputStreamPtr input_stream = FormatFactory::instance().getInput(format, read_buf, *sample_block,
-                                                                                  context, max_block_size);
+            auto input_format = FormatFactory::instance().getInput(format, read_buf, *sample_block,
+                                                                   context, max_block_size);
+            auto input_stream = std::make_shared<InputStreamFromInputFormat>(input_format);
            ODBCBlockOutputStream output_stream(pool->get(), db_name, table_name, *sample_block, quoting_style);
            copyData(*input_stream, output_stream);
            writeStringBinary("Ok.", out);
@ -173,7 +176,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne
            std::string query = params.get("query");
            LOG_TRACE(log, "Query: {}", query);

-            BlockOutputStreamPtr writer = FormatFactory::instance().getOutput(format, out, *sample_block, context);
+            BlockOutputStreamPtr writer = FormatFactory::instance().getOutputStream(format, out, *sample_block, context);
            auto pool = getPool(connection_string);
            ODBCBlockInputStream inp(pool->get(), query, *sample_block, max_block_size);
            copyData(inp, *writer);
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@ -65,6 +65,8 @@
 #include <Server/TCPHandlerFactory.h>
 #include <Common/SensitiveDataMasker.h>
 #include <Common/ThreadFuzzer.h>
+#include <Common/getHashOfLoadedBinary.h>
+#include <Common/Elf.h>
 #include <Server/MySQLHandlerFactory.h>
 #include <Server/PostgreSQLHandlerFactory.h>
 #include <Server/ProtocolServerAdapter.h>
@ -184,6 +186,7 @@ namespace ErrorCodes
    extern const int FAILED_TO_GETPWUID;
    extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA;
    extern const int NETWORK_ERROR;
+    extern const int CORRUPTED_DATA;
 }


@ -436,7 +439,44 @@ int Server::main(const std::vector<std::string> & /*args*/)

 #if defined(OS_LINUX)
    std::string executable_path = getExecutablePath();
-    if (executable_path.empty())
+
+    if (!executable_path.empty())
+    {
+        /// Integrity check based on checksum of the executable code.
+        /// Note: it is not intended to protect from malicious party,
+        /// because the reference checksum can be easily modified as well.
+        /// And we don't involve asymmetric encryption with PKI yet.
+        /// It's only intended to protect from faulty hardware.
+        /// Note: it is only based on machine code.
+        /// But there are other sections of the binary (e.g. exception handling tables)
+        /// that are interpreted (not executed) but can alter the behaviour of the program as well.
+
+        String calculated_binary_hash = getHashOfLoadedBinaryHex();
+
+        if (stored_binary_hash.empty())
+        {
+            LOG_WARNING(log, "Calculated checksum of the binary: {}."
+                " There is no information about the reference checksum.", calculated_binary_hash);
+        }
+        else if (calculated_binary_hash == stored_binary_hash)
+        {
+            LOG_INFO(log, "Calculated checksum of the binary: {}, integrity check passed.", calculated_binary_hash);
+        }
+        else
+        {
+            throw Exception(ErrorCodes::CORRUPTED_DATA,
+                "Calculated checksum of the ClickHouse binary ({0}) does not correspond"
+                " to the reference checksum stored in the binary ({1})."
+                " It may indicate one of the following:"
+                " - the file {2} was changed just after startup;"
+                " - the file {2} is damaged on disk due to faulty hardware;"
+                " - the loaded executable is damaged in memory due to faulty hardware;"
+                " - the file {2} was intentionally modified;"
+                " - logical error in code."
+                , calculated_binary_hash, stored_binary_hash, executable_path);
+        }
+    }
+    else
        executable_path = "/usr/bin/clickhouse";    /// It is used for information messages.

    /// After full config loaded
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@ -204,7 +204,7 @@
            <privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile>
            <!-- dhparams are optional. You can delete the <dhParamsFile> element.
                 To generate dhparams, use the following command:
-                  openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096 
+                  openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096
                 Only file format with BEGIN DH PARAMETERS is supported.
              -->
            <dhParamsFile>/etc/clickhouse-server/dhparam.pem</dhParamsFile>
@ -432,7 +432,7 @@
    <!-- Configuration of clusters that could be used in Distributed tables.
         https://clickhouse.tech/docs/en/operations/table_engines/distributed/
      -->
-    <remote_servers incl="clickhouse_remote_servers" >
+    <remote_servers>
        <!-- Test only shard config for testing distributed storage -->
        <test_shard_localhost>
            <!-- Inter-server per-cluster secret for Distributed queries
@ -566,17 +566,37 @@
    <!-- ZooKeeper is used to store metadata about replicas, when using Replicated tables.
         Optional. If you don't use replicated tables, you could omit that.

-         See https://clickhouse.yandex/docs/en/table_engines/replication/
+         See https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/
      -->

-    <zookeeper incl="zookeeper-servers" optional="true" />
+    <!--
+    <zookeeper>
+        <node>
+            <host>example1</host>
+            <port>2181</port>
+        </node>
+        <node>
+            <host>example2</host>
+            <port>2181</port>
+        </node>
+        <node>
+            <host>example3</host>
+            <port>2181</port>
+        </node>
+    </zookeeper>
+    -->

    <!-- Substitutions for parameters of replicated tables.
          Optional. If you don't use replicated tables, you could omit that.

-         See https://clickhouse.yandex/docs/en/table_engines/replication/#creating-replicated-tables
+         See https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/#creating-replicated-tables
      -->
-    <macros incl="macros" optional="true" />
+    <!--
+    <macros>
+        <shard>01</shard>
+        <replica>example01-01-1</replica>
+    </macros>
+    -->


    <!-- Reloading interval for embedded dictionaries, in seconds. Default: 3600. -->
@ -656,7 +676,7 @@
        <database>system</database>
        <table>query_log</table>
        <!--
-            PARTITION BY expr: https://clickhouse.yandex/docs/en/table_engines/custom_partitioning_key/
+            PARTITION BY expr: https://clickhouse.yandex/docs/en/table_engines/mergetree-family/custom_partitioning_key/
            Example:
                event_date
                toMonday(event_date)
@ -810,8 +830,8 @@
    <!-- Uncomment if you want data to be compressed 30-100% better.
         Don't do that if you just started using ClickHouse.
      -->
-    <compression incl="clickhouse_compression">
    <!--
+    <compression>
        <!- - Set of variants. Checked in order. Last matching case wins. If nothing matches, lz4 will be used. - ->
        <case>

@ -822,8 +842,8 @@
            <!- - What compression method to use. - ->
            <method>zstd</method>
        </case>
-    -->
    </compression>
+    -->

    <!-- Allow to execute distributed DDL queries (CREATE, DROP, ALTER, RENAME) on cluster.
         Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. -->
--- a/programs/server/embedded.xml
+++ b/programs/server/embedded.xml
@ -20,7 +20,7 @@
        <default>
            <password></password>

-            <networks incl="networks" replace="replace">
+            <networks>
                <ip>::/0</ip>
            </networks>

--- a/programs/server/users.xml
+++ b/programs/server/users.xml
@ -77,7 +77,7 @@
                     Strongly recommended that regexp is ends with $
                 All results of DNS requests are cached till server restart.
            -->
-            <networks incl="networks" replace="replace">
+            <networks>
                <ip>::/0</ip>
            </networks>

--- a/src/AggregateFunctions/AggregateFunctionGroupArray.h
+++ b/src/AggregateFunctions/AggregateFunctionGroupArray.h
@ -188,13 +188,13 @@ public:
            if (!limit_num_elems)
            {
                if (rhs_elems.value.size())
-                    cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.end(), arena);
+                    cur_elems.value.insertByOffsets(rhs_elems.value, 0, rhs_elems.value.size(), arena);
            }
            else
            {
                UInt64 elems_to_insert = std::min(static_cast<size_t>(max_elems) - cur_elems.value.size(), rhs_elems.value.size());
                if (elems_to_insert)
-                    cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.begin() + elems_to_insert, arena);
+                    cur_elems.value.insertByOffsets(rhs_elems.value, 0, elems_to_insert, arena);
            }
        }

--- a/src/AggregateFunctions/AggregateFunctionIf.cpp
+++ b/src/AggregateFunctions/AggregateFunctionIf.cpp
@ -53,17 +53,35 @@ class AggregateFunctionIfNullUnary final
 private:
    size_t num_arguments;

+    /// The name of the nested function, including combinators (i.e. *If)
+    ///
+    /// getName() from the nested_function cannot be used because in case of *If combinator
+    /// with Nullable argument nested_function will point to the function w/o combinator.
+    /// (I.e. sumIf(Nullable, 1) -> sum()), and distributed query processing will fail.
+    ///
+    /// And nested_function cannot point to the function with *If since
+    /// due to optimization in the add() which pass only one column with the result,
+    /// and so AggregateFunctionIf::add() cannot be called this way
+    /// (it write to the last argument -- num_arguments-1).
+    ///
+    /// And to avoid extra level of indirection, the name of function is cached:
+    ///
+    ///     AggregateFunctionIfNullUnary::add -> [ AggregateFunctionIf::add -> ] AggregateFunctionSum::add
+    String name;
+
    using Base = AggregateFunctionNullBase<result_is_nullable, serialize_flag,
        AggregateFunctionIfNullUnary<result_is_nullable, serialize_flag>>;
 public:

    String getName() const override
    {
-        return Base::getName();
+        return name;
    }

-    AggregateFunctionIfNullUnary(AggregateFunctionPtr nested_function_, const DataTypes & arguments, const Array & params)
-        : Base(std::move(nested_function_), arguments, params), num_arguments(arguments.size())
+    AggregateFunctionIfNullUnary(const String & name_, AggregateFunctionPtr nested_function_, const DataTypes & arguments, const Array & params)
+        : Base(std::move(nested_function_), arguments, params)
+        , num_arguments(arguments.size())
+        , name(name_)
    {
        if (num_arguments == 0)
            throw Exception("Aggregate function " + getName() + " require at least one argument",
@ -174,14 +192,14 @@ AggregateFunctionPtr AggregateFunctionIf::getOwnNullAdapter(
    {
        if (return_type_is_nullable)
        {
-            return std::make_shared<AggregateFunctionIfNullUnary<true, true>>(nested_func, arguments, params);
+            return std::make_shared<AggregateFunctionIfNullUnary<true, true>>(nested_function->getName(), nested_func, arguments, params);
        }
        else
        {
            if (serialize_flag)
-                return std::make_shared<AggregateFunctionIfNullUnary<false, true>>(nested_func, arguments, params);
+                return std::make_shared<AggregateFunctionIfNullUnary<false, true>>(nested_function->getName(), nested_func, arguments, params);
            else
-                return std::make_shared<AggregateFunctionIfNullUnary<false, false>>(nested_func, arguments, params);
+                return std::make_shared<AggregateFunctionIfNullUnary<false, false>>(nested_function->getName(), nested_func, arguments, params);
        }
    }
    else
--- a/src/AggregateFunctions/AggregateFunctionSumMap.cpp
+++ b/src/AggregateFunctions/AggregateFunctionSumMap.cpp
@ -66,8 +66,7 @@ auto parseArguments(const std::string & name, const DataTypes & arguments)
        values_types.push_back(array_type->getNestedType());
    }

-    return  std::tuple{std::move(keys_type), std::move(values_types),
-                tuple_argument};
+    return std::tuple{std::move(keys_type), std::move(values_types), tuple_argument};
 }

 // This function instantiates a particular overload of the sumMap family of
--- a/src/AggregateFunctions/AggregateFunctionSumMap.h
+++ b/src/AggregateFunctions/AggregateFunctionSumMap.h
@ -54,6 +54,8 @@ struct AggregateFunctionMapData
  *  ([1,2,3,4,5,6,7,8,9,10],[10,10,45,20,35,20,15,30,20,20])
  *
  * minMap and maxMap share the same idea, but calculate min and max correspondingly.
+  *
+  * NOTE: The implementation of these functions are "amateur grade" - not efficient and low quality.
  */

 template <typename T, typename Derived, typename Visitor, bool overflow, bool tuple_argument, bool compact>
@ -72,7 +74,8 @@ public:
            const DataTypes & values_types_, const DataTypes & argument_types_)
        : Base(argument_types_, {} /* parameters */), keys_type(keys_type_),
          values_types(values_types_)
-    {}
+    {
+    }

    DataTypePtr getReturnType() const override
    {
@ -81,13 +84,26 @@ public:

        for (const auto & value_type : values_types)
        {
+            if constexpr (std::is_same_v<Visitor, FieldVisitorSum>)
+            {
+                if (!value_type->isSummable())
+                    throw Exception{ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                        "Values for {} cannot be summed, passed type {}",
+                        getName(), value_type->getName()};
+            }
+
            DataTypePtr result_type;

            if constexpr (overflow)
            {
+                if (value_type->onlyNull())
+                    throw Exception{ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                        "Cannot calculate {} of type {}",
+                        getName(), value_type->getName()};
+
                // Overflow, meaning that the returned type is the same as
-                // the input type.
-                result_type = value_type;
+                // the input type. Nulls are skipped.
+                result_type = removeNullable(value_type);
            }
            else
            {
@ -120,9 +136,9 @@ public:
        }
    }

-    void add(AggregateDataPtr place, const IColumn** _columns, const size_t row_num, Arena *) const override
+    void add(AggregateDataPtr place, const IColumn ** columns_, const size_t row_num, Arena *) const override
    {
-        const auto & columns = getArgumentColumns(_columns);
+        const auto & columns = getArgumentColumns(columns_);

        // Column 0 contains array of keys of known type
        const ColumnArray & array_column0 = assert_cast<const ColumnArray &>(*columns[0]);
@ -148,16 +164,13 @@ public:
            // Insert column values for all keys
            for (size_t i = 0; i < keys_vec_size; ++i)
            {
-                auto value = value_column.operator[](values_vec_offset + i);
-                auto key = key_column.operator[](keys_vec_offset + i).get<T>();
+                auto value = value_column[values_vec_offset + i];
+                auto key = key_column[keys_vec_offset + i].get<T>();

                if (!keepKey(key))
                    continue;

-                if (value.isNull())
-                    continue;
-
-                typename std::decay_t<decltype(merged_maps)>::iterator it;
+                decltype(merged_maps.begin()) it;
                if constexpr (IsDecimalNumber<T>)
                {
                    // FIXME why is storing NearestFieldType not enough, and we
@ -170,17 +183,20 @@ public:

                if (it != merged_maps.end())
                {
-                    applyVisitor(Visitor(value), it->second[col]);
+                    if (!value.isNull())
+                    {
+                        if (it->second[col].isNull())
+                            it->second[col] = value;
+                        else
+                            applyVisitor(Visitor(value), it->second[col]);
+                    }
                }
                else
                {
                    // Create a value array for this key
                    Array new_values;
-                    new_values.resize(values_types.size());
-                    for (size_t k = 0; k < new_values.size(); ++k)
-                    {
-                        new_values[k] = (k == col) ? value : values_types[k]->getDefault();
-                    }
+                    new_values.resize(size);
+                    new_values[col] = value;

                    if constexpr (IsDecimalNumber<T>)
                    {
@ -207,7 +223,8 @@ public:
            if (it != merged_maps.end())
            {
                for (size_t col = 0; col < values_types.size(); ++col)
-                    applyVisitor(Visitor(elem.second[col]), it->second[col]);
+                    if (!elem.second[col].isNull())
+                        applyVisitor(Visitor(elem.second[col]), it->second[col]);
            }
            else
                merged_maps[elem.first] = elem.second;
@ -253,6 +270,8 @@ public:

    void insertResultInto(AggregateDataPtr place, IColumn & to, Arena *) const override
    {
+        size_t num_columns = values_types.size();
+
        // Final step does compaction of keys that have zero values, this mutates the state
        auto & merged_maps = this->data(place).merged_maps;

@ -263,9 +282,9 @@ public:
            {
                // Key is not compacted if it has at least one non-zero value
                bool erase = true;
-                for (size_t col = 0; col < values_types.size(); ++col)
+                for (size_t col = 0; col < num_columns; ++col)
                {
-                    if (it->second[col] != values_types[col]->getDefault())
+                    if (!it->second[col].isNull() && it->second[col] != values_types[col]->getDefault())
                    {
                        erase = false;
                        break;
@ -290,7 +309,7 @@ public:
        to_keys_offsets.push_back(to_keys_offsets.back() + size);
        to_keys_col.reserve(size);

-        for (size_t col = 0; col < values_types.size(); ++col)
+        for (size_t col = 0; col < num_columns; ++col)
        {
            auto & to_values_arr = assert_cast<ColumnArray &>(to_tuple.getColumn(col + 1));
            auto & to_values_offsets = to_values_arr.getOffsets();
@ -305,10 +324,13 @@ public:
            to_keys_col.insert(elem.first);

            // Write 0..n arrays of values
-            for (size_t col = 0; col < values_types.size(); ++col)
+            for (size_t col = 0; col < num_columns; ++col)
            {
                auto & to_values_col = assert_cast<ColumnArray &>(to_tuple.getColumn(col + 1)).getData();
-                to_values_col.insert(elem.second[col]);
+                if (elem.second[col].isNull())
+                    to_values_col.insertDefault();
+                else
+                    to_values_col.insert(elem.second[col]);
            }
        }
    }
--- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.h
+++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.h
@ -33,7 +33,7 @@ template <typename T>
 struct AggregateFunctionWindowFunnelData
 {
    using TimestampEvent = std::pair<T, UInt8>;
-    using TimestampEvents = PODArray<TimestampEvent, 64>;
+    using TimestampEvents = PODArrayWithStackMemory<TimestampEvent, 64>;
    using Comparator = ComparePairFirst;

    bool sorted = true;
--- a/src/Columns/ColumnAggregateFunction.cpp
+++ b/src/Columns/ColumnAggregateFunction.cpp
@ -393,6 +393,12 @@ size_t ColumnAggregateFunction::byteSize() const
            + (my_arena ? my_arena->size() : 0);
 }

+size_t ColumnAggregateFunction::byteSizeAt(size_t) const
+{
+    /// Lower estimate as aggregate function can allocate more data in Arena.
+    return sizeof(data[0]) + func->sizeOfData();
+}
+
 /// Like in byteSize(), the size is underestimated.
 size_t ColumnAggregateFunction::allocatedBytes() const
 {
--- a/src/Columns/ColumnAggregateFunction.h
+++ b/src/Columns/ColumnAggregateFunction.h
@ -163,6 +163,8 @@ public:

    size_t byteSize() const override;

+    size_t byteSizeAt(size_t n) const override;
+
    size_t allocatedBytes() const override;

    void protect() override;
--- a/src/Columns/ColumnArray.cpp
+++ b/src/Columns/ColumnArray.cpp
@ -403,6 +403,21 @@ size_t ColumnArray::byteSize() const
 }


+size_t ColumnArray::byteSizeAt(size_t n) const
+{
+    const auto & offsets_data = getOffsets();
+
+    size_t pos = offsets_data[n - 1];
+    size_t end = offsets_data[n];
+
+    size_t res = sizeof(offsets_data[0]);
+    for (; pos < end; ++pos)
+        res += getData().byteSizeAt(pos);
+
+    return res;
+}
+
+
 size_t ColumnArray::allocatedBytes() const
 {
    return getData().allocatedBytes() + getOffsets().allocated_bytes();
--- a/src/Columns/ColumnArray.h
+++ b/src/Columns/ColumnArray.h
@ -84,6 +84,7 @@ public:
    void updatePermutationWithCollation(const Collator & collator, bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges& equal_range) const override;
    void reserve(size_t n) override;
    size_t byteSize() const override;
+    size_t byteSizeAt(size_t n) const override;
    size_t allocatedBytes() const override;
    void protect() override;
    ColumnPtr replicate(const Offsets & replicate_offsets) const override;
--- a/src/Columns/ColumnConst.h
+++ b/src/Columns/ColumnConst.h
@ -187,6 +187,11 @@ public:
        return data->byteSize() + sizeof(s);
    }

+    size_t byteSizeAt(size_t) const override
+    {
+        return data->byteSizeAt(0);
+    }
+
    size_t allocatedBytes() const override
    {
        return data->allocatedBytes() + sizeof(s);
--- a/src/Columns/ColumnDecimal.h
+++ b/src/Columns/ColumnDecimal.h
@ -87,6 +87,7 @@ public:

    size_t size() const override { return data.size(); }
    size_t byteSize() const override { return data.size() * sizeof(data[0]); }
+    size_t byteSizeAt(size_t) const override { return sizeof(data[0]); }
    size_t allocatedBytes() const override { return data.allocated_bytes(); }
    void protect() override { data.protect(); }
    void reserve(size_t n) override { data.reserve(n); }
--- a/src/Columns/ColumnFixedString.h
+++ b/src/Columns/ColumnFixedString.h
@ -57,6 +57,11 @@ public:
        return chars.size() + sizeof(n);
    }

+    size_t byteSizeAt(size_t) const override
+    {
+        return n;
+    }
+
    size_t allocatedBytes() const override
    {
        return chars.allocated_bytes() + sizeof(n);
--- a/src/Columns/ColumnFunction.cpp
+++ b/src/Columns/ColumnFunction.cpp
@ -140,6 +140,15 @@ size_t ColumnFunction::byteSize() const
    return total_size;
 }

+size_t ColumnFunction::byteSizeAt(size_t n) const
+{
+    size_t total_size = 0;
+    for (const auto & column : captured_columns)
+        total_size += column.column->byteSizeAt(n);
+
+    return total_size;
+}
+
 size_t ColumnFunction::allocatedBytes() const
 {
    size_t total_size = 0;
--- a/src/Columns/ColumnFunction.h
+++ b/src/Columns/ColumnFunction.h
@ -47,6 +47,7 @@ public:
    void getExtremes(Field &, Field &) const override {}

    size_t byteSize() const override;
+    size_t byteSizeAt(size_t n) const override;
    size_t allocatedBytes() const override;

    void appendArguments(const ColumnsWithTypeAndName & columns);
--- a/src/Columns/ColumnLowCardinality.h
+++ b/src/Columns/ColumnLowCardinality.h
@ -151,6 +151,7 @@ public:
    void reserve(size_t n) override { idx.reserve(n); }

    size_t byteSize() const override { return idx.getPositions()->byteSize() + getDictionary().byteSize(); }
+    size_t byteSizeAt(size_t n) const override { return getDictionary().byteSizeAt(getIndexes().getUInt(n)); }
    size_t allocatedBytes() const override { return idx.getPositions()->allocatedBytes() + getDictionary().allocatedBytes(); }

    void forEachSubcolumn(ColumnCallback callback) override
--- a/src/Columns/ColumnMap.cpp
+++ b/src/Columns/ColumnMap.cpp
@ -211,6 +211,11 @@ size_t ColumnMap::byteSize() const
    return nested->byteSize();
 }

+size_t ColumnMap::byteSizeAt(size_t n) const
+{
+    return nested->byteSizeAt(n);
+}
+
 size_t ColumnMap::allocatedBytes() const
 {
    return nested->allocatedBytes();
--- a/src/Columns/ColumnMap.h
+++ b/src/Columns/ColumnMap.h
@ -77,6 +77,7 @@ public:
    void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const override;
    void reserve(size_t n) override;
    size_t byteSize() const override;
+    size_t byteSizeAt(size_t n) const override;
    size_t allocatedBytes() const override;
    void protect() override;
    void forEachSubcolumn(ColumnCallback callback) override;
--- a/src/Columns/ColumnNullable.cpp
+++ b/src/Columns/ColumnNullable.cpp
@ -495,6 +495,11 @@ size_t ColumnNullable::byteSize() const
    return getNestedColumn().byteSize() + getNullMapColumn().byteSize();
 }

+size_t ColumnNullable::byteSizeAt(size_t n) const
+{
+    return sizeof(getNullMapData()[0]) + getNestedColumn().byteSizeAt(n);
+}
+
 size_t ColumnNullable::allocatedBytes() const
 {
    return getNestedColumn().allocatedBytes() + getNullMapColumn().allocatedBytes();
--- a/src/Columns/ColumnNullable.h
+++ b/src/Columns/ColumnNullable.h
@ -101,6 +101,7 @@ public:
        const Collator & collator, bool reverse, size_t limit, int null_direction_hint, Permutation & res, EqualRanges& equal_range) const override;
    void reserve(size_t n) override;
    size_t byteSize() const override;
+    size_t byteSizeAt(size_t n) const override;
    size_t allocatedBytes() const override;
    void protect() override;
    ColumnPtr replicate(const Offsets & replicate_offsets) const override;
--- a/src/Columns/ColumnString.h
+++ b/src/Columns/ColumnString.h
@ -71,6 +71,12 @@ public:
        return chars.size() + offsets.size() * sizeof(offsets[0]);
    }

+    size_t byteSizeAt(size_t n) const override
+    {
+        assert(n < size());
+        return sizeAt(n) + sizeof(offsets[0]);
+    }
+
    size_t allocatedBytes() const override
    {
        return chars.allocated_bytes() + offsets.allocated_bytes();
--- a/src/Columns/ColumnTuple.cpp
+++ b/src/Columns/ColumnTuple.cpp
@ -424,6 +424,14 @@ size_t ColumnTuple::byteSize() const
    return res;
 }

+size_t ColumnTuple::byteSizeAt(size_t n) const
+{
+    size_t res = 0;
+    for (const auto & column : columns)
+        res += column->byteSizeAt(n);
+    return res;
+}
+
 size_t ColumnTuple::allocatedBytes() const
 {
    size_t res = 0;
--- a/src/Columns/ColumnTuple.h
+++ b/src/Columns/ColumnTuple.h
@ -83,6 +83,7 @@ public:
    void updatePermutationWithCollation(const Collator & collator, bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges& equal_ranges) const override;
    void reserve(size_t n) override;
    size_t byteSize() const override;
+    size_t byteSizeAt(size_t n) const override;
    size_t allocatedBytes() const override;
    void protect() override;
    void forEachSubcolumn(ColumnCallback callback) override;
--- a/src/Columns/ColumnUnique.h
+++ b/src/Columns/ColumnUnique.h
@ -88,6 +88,10 @@ public:
    bool isNumeric() const override { return column_holder->isNumeric(); }

    size_t byteSize() const override { return column_holder->byteSize(); }
+    size_t byteSizeAt(size_t n) const override
+    {
+        return getNestedColumn()->byteSizeAt(n);
+    }
    void protect() override { column_holder->protect(); }
    size_t allocatedBytes() const override
    {
--- a/src/Columns/ColumnVector.h
+++ b/src/Columns/ColumnVector.h
@ -178,6 +178,11 @@ public:
        return data.size() * sizeof(data[0]);
    }

+    size_t byteSizeAt(size_t) const override
+    {
+        return sizeof(data[0]);
+    }
+
    size_t allocatedBytes() const override
    {
        if constexpr (is_POD)
--- a/src/Columns/IColumn.h
+++ b/src/Columns/IColumn.h
@ -333,6 +333,9 @@ public:
    /// Size of column data in memory (may be approximate) - for profiling. Zero, if could not be determined.
    virtual size_t byteSize() const = 0;

+    /// Size of single value in memory (for accounting purposes)
+    virtual size_t byteSizeAt(size_t /*n*/) const = 0;
+
    /// Size of memory, allocated for column.
    /// This is greater or equals to byteSize due to memory reservation in containers.
    /// Zero, if could not be determined.
--- a/src/Columns/IColumnDummy.h
+++ b/src/Columns/IColumnDummy.h
@ -33,6 +33,7 @@ public:
    void insertDefault() override { ++s; }
    void popBack(size_t n) override { s -= n; }
    size_t byteSize() const override { return 0; }
+    size_t byteSizeAt(size_t) const override { return 0; }
    size_t allocatedBytes() const override { return 0; }
    int compareAt(size_t, size_t, const IColumn &, int) const override { return 0; }
    void compareColumn(const IColumn &, size_t, PaddedPODArray<UInt64> *, PaddedPODArray<Int8> &, int, int) const override
--- a/src/Common/Allocator.h
+++ b/src/Common/Allocator.h
@ -28,7 +28,7 @@
 #include <common/mremap.h>
 #include <common/getPageSize.h>

-#include <Common/MemoryTracker.h>
+#include <Common/CurrentMemoryTracker.h>
 #include <Common/Exception.h>
 #include <Common/formatReadable.h>

--- a/src/Common/Arena.h
+++ b/src/Common/Arena.h
@ -28,7 +28,7 @@ namespace DB
  * - put lot of strings inside pool, keep their addresses;
  * - addresses remain valid during lifetime of pool;
  * - at destruction of pool, all memory is freed;
-  * - memory is allocated and freed by large chunks;
+  * - memory is allocated and freed by large MemoryChunks;
  * - freeing parts of data is not possible (but look at ArenaWithFreeLists if you need);
  */
 class Arena : private boost::noncopyable
@ -37,16 +37,16 @@ private:
    /// Padding allows to use 'memcpySmallAllowReadWriteOverflow15' instead of 'memcpy'.
    static constexpr size_t pad_right = 15;

-    /// Contiguous chunk of memory and pointer to free space inside it. Member of single-linked list.
-    struct alignas(16) Chunk : private Allocator<false>    /// empty base optimization
+    /// Contiguous MemoryChunk of memory and pointer to free space inside it. Member of single-linked list.
+    struct alignas(16) MemoryChunk : private Allocator<false>    /// empty base optimization
    {
        char * begin;
        char * pos;
        char * end; /// does not include padding.

-        Chunk * prev;
+        MemoryChunk * prev;

-        Chunk(size_t size_, Chunk * prev_)
+        MemoryChunk(size_t size_, MemoryChunk * prev_)
        {
            ProfileEvents::increment(ProfileEvents::ArenaAllocChunks);
            ProfileEvents::increment(ProfileEvents::ArenaAllocBytes, size_);
@ -59,7 +59,7 @@ private:
            ASAN_POISON_MEMORY_REGION(begin, size_);
        }

-        ~Chunk()
+        ~MemoryChunk()
        {
            /// We must unpoison the memory before returning to the allocator,
            /// because the allocator might not have asan integration, and the
@ -80,8 +80,8 @@ private:
    size_t growth_factor;
    size_t linear_growth_threshold;

-    /// Last contiguous chunk of memory.
-    Chunk * head;
+    /// Last contiguous MemoryChunk of memory.
+    MemoryChunk * head;
    size_t size_in_bytes;
    size_t page_size;

@ -90,7 +90,7 @@ private:
        return (s + page_size - 1) / page_size * page_size;
    }

-    /// If chunks size is less than 'linear_growth_threshold', then use exponential growth, otherwise - linear growth
+    /// If MemoryChunks size is less than 'linear_growth_threshold', then use exponential growth, otherwise - linear growth
    ///  (to not allocate too much excessive memory).
    size_t nextSize(size_t min_next_size) const
    {
@ -104,7 +104,7 @@ private:
        {
            // allocContinue() combined with linear growth results in quadratic
            // behavior: we append the data by small amounts, and when it
-            // doesn't fit, we create a new chunk and copy all the previous data
+            // doesn't fit, we create a new MemoryChunk and copy all the previous data
            // into it. The number of times we do this is directly proportional
            // to the total size of data that is going to be serialized. To make
            // the copying happen less often, round the next size up to the
@ -117,10 +117,10 @@ private:
        return roundUpToPageSize(size_after_grow, page_size);
    }

-    /// Add next contiguous chunk of memory with size not less than specified.
-    void NO_INLINE addChunk(size_t min_size)
+    /// Add next contiguous MemoryChunk of memory with size not less than specified.
+    void NO_INLINE addMemoryChunk(size_t min_size)
    {
-        head = new Chunk(nextSize(min_size + pad_right), head);
+        head = new MemoryChunk(nextSize(min_size + pad_right), head);
        size_in_bytes += head->size();
    }

@ -130,7 +130,7 @@ private:
 public:
    Arena(size_t initial_size_ = 4096, size_t growth_factor_ = 2, size_t linear_growth_threshold_ = 128 * 1024 * 1024)
        : growth_factor(growth_factor_), linear_growth_threshold(linear_growth_threshold_),
-        head(new Chunk(initial_size_, nullptr)), size_in_bytes(head->size()),
+        head(new MemoryChunk(initial_size_, nullptr)), size_in_bytes(head->size()),
        page_size(static_cast<size_t>(::getPageSize()))
    {
    }
@ -144,7 +144,7 @@ public:
    char * alloc(size_t size)
    {
        if (unlikely(head->pos + size > head->end))
-            addChunk(size);
+            addMemoryChunk(size);

        char * res = head->pos;
        head->pos += size;
@ -169,7 +169,7 @@ public:
                return res;
            }

-            addChunk(size + alignment);
+            addMemoryChunk(size + alignment);
        } while (true);
    }

@ -194,8 +194,8 @@ public:
    /** Begin or expand a contiguous range of memory.
      * 'range_start' is the start of range. If nullptr, a new range is
      * allocated.
-      * If there is no space in the current chunk to expand the range,
-      * the entire range is copied to a new, bigger memory chunk, and the value
+      * If there is no space in the current MemoryChunk to expand the range,
+      * the entire range is copied to a new, bigger memory MemoryChunk, and the value
      * of 'range_start' is updated.
      * If the optional 'start_alignment' is specified, the start of range is
      * kept aligned to this value.
@ -209,7 +209,7 @@ public:
        /*
         * Allocating zero bytes doesn't make much sense. Also, a zero-sized
         * range might break the invariant that the range begins at least before
-         * the current chunk end.
+         * the current MemoryChunk end.
         */
        assert(additional_bytes > 0);

@ -228,19 +228,19 @@ public:

        // This method only works for extending the last allocation. For lack of
        // original size, check a weaker condition: that 'begin' is at least in
-        // the current Chunk.
+        // the current MemoryChunk.
        assert(range_start >= head->begin);
        assert(range_start < head->end);

        if (head->pos + additional_bytes <= head->end)
        {
-            // The new size fits into the last chunk, so just alloc the
+            // The new size fits into the last MemoryChunk, so just alloc the
            // additional size. We can alloc without alignment here, because it
            // only applies to the start of the range, and we don't change it.
            return alloc(additional_bytes);
        }

-        // New range doesn't fit into this chunk, will copy to a new one.
+        // New range doesn't fit into this MemoryChunk, will copy to a new one.
        //
        // Note: among other things, this method is used to provide a hack-ish
        // implementation of realloc over Arenas in ArenaAllocators. It wastes a
@ -301,16 +301,16 @@ public:
        return res;
    }

-    /// Size of chunks in bytes.
+    /// Size of MemoryChunks in bytes.
    size_t size() const
    {
        return size_in_bytes;
    }

-    /// Bad method, don't use it -- the chunks are not your business, the entire
+    /// Bad method, don't use it -- the MemoryChunks are not your business, the entire
    /// purpose of the arena code is to manage them for you, so if you find
    /// yourself having to use this method, probably you're doing something wrong.
-    size_t remainingSpaceInCurrentChunk() const
+    size_t remainingSpaceInCurrentMemoryChunk() const
    {
        return head->remaining();
    }
--- a/src/Common/CurrentMemoryTracker.cpp
+++ b/src/Common/CurrentMemoryTracker.cpp
@ -0,0 +1,81 @@
+#include <Common/MemoryTracker.h>
+#include <Common/CurrentThread.h>
+
+#include <Common/CurrentMemoryTracker.h>
+
+namespace
+{
+
+MemoryTracker * getMemoryTracker()
+{
+    if (auto * thread_memory_tracker = DB::CurrentThread::getMemoryTracker())
+        return thread_memory_tracker;
+
+    /// Once the main thread is initialized,
+    /// total_memory_tracker is initialized too.
+    /// And can be used, since MainThreadStatus is required for profiling.
+    if (DB::MainThreadStatus::get())
+        return &total_memory_tracker;
+
+    return nullptr;
+}
+
+}
+
+namespace CurrentMemoryTracker
+{
+
+using DB::current_thread;
+
+void alloc(Int64 size)
+{
+    if (auto * memory_tracker = getMemoryTracker())
+    {
+        if (current_thread)
+        {
+            current_thread->untracked_memory += size;
+            if (current_thread->untracked_memory > current_thread->untracked_memory_limit)
+            {
+                /// Zero untracked before track. If tracker throws out-of-limit we would be able to alloc up to untracked_memory_limit bytes
+                /// more. It could be useful to enlarge Exception message in rethrow logic.
+                Int64 tmp = current_thread->untracked_memory;
+                current_thread->untracked_memory = 0;
+                memory_tracker->alloc(tmp);
+            }
+        }
+        /// total_memory_tracker only, ignore untracked_memory
+        else
+        {
+            memory_tracker->alloc(size);
+        }
+    }
+}
+
+void realloc(Int64 old_size, Int64 new_size)
+{
+    Int64 addition = new_size - old_size;
+    addition > 0 ? alloc(addition) : free(-addition);
+}
+
+void free(Int64 size)
+{
+    if (auto * memory_tracker = getMemoryTracker())
+    {
+        if (current_thread)
+        {
+            current_thread->untracked_memory -= size;
+            if (current_thread->untracked_memory < -current_thread->untracked_memory_limit)
+            {
+                memory_tracker->free(-current_thread->untracked_memory);
+                current_thread->untracked_memory = 0;
+            }
+        }
+        /// total_memory_tracker only, ignore untracked_memory
+        else
+        {
+            memory_tracker->free(size);
+        }
+    }
+}
+
+}
--- a/src/Common/CurrentMemoryTracker.h
+++ b/src/Common/CurrentMemoryTracker.h
@ -0,0 +1,11 @@
+#pragma once
+
+#include <common/types.h>
+
+/// Convenience methods, that use current thread's memory_tracker if it is available.
+namespace CurrentMemoryTracker
+{
+    void alloc(Int64 size);
+    void realloc(Int64 old_size, Int64 new_size);
+    void free(Int64 size);
+}
--- a/src/Common/DirectorySyncGuard.cpp
+++ b/src/Common/DirectorySyncGuard.cpp
@ -0,0 +1,41 @@
+#include <Common/DirectorySyncGuard.h>
+#include <Common/Exception.h>
+#include <Disks/IDisk.h>
+#include <fcntl.h> // O_RDWR
+
+/// OSX does not have O_DIRECTORY
+#ifndef O_DIRECTORY
+#define O_DIRECTORY O_RDWR
+#endif
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int CANNOT_FSYNC;
+}
+
+DirectorySyncGuard::DirectorySyncGuard(const DiskPtr & disk_, const String & path)
+    : disk(disk_)
+    , fd(disk_->open(path, O_DIRECTORY))
+{}
+
+DirectorySyncGuard::~DirectorySyncGuard()
+{
+    try
+    {
+#if defined(OS_DARWIN)
+        if (fcntl(fd, F_FULLFSYNC, 0))
+            throwFromErrno("Cannot fcntl(F_FULLFSYNC)", ErrorCodes::CANNOT_FSYNC);
+#endif
+        disk->sync(fd);
+        disk->close(fd);
+    }
+    catch (...)
+    {
+        tryLogCurrentException(__PRETTY_FUNCTION__);
+    }
+}
+
+}
--- a/src/Common/DirectorySyncGuard.h
+++ b/src/Common/DirectorySyncGuard.h
@ -1,36 +1,26 @@
 #pragma once

-#include <Disks/IDisk.h>
+#include <string>
+#include <memory>

 namespace DB
 {

+class IDisk;
+using DiskPtr = std::shared_ptr<IDisk>;
+
 /// Helper class, that receives file descriptor and does fsync for it in destructor.
 /// It's used to keep descriptor open, while doing some operations with it, and do fsync at the end.
 /// Guaranties of sequence 'close-reopen-fsync' may depend on kernel version.
 /// Source: linux-fsdevel mailing-list https://marc.info/?l=linux-fsdevel&m=152535409207496
-class FileSyncGuard
+class DirectorySyncGuard
 {
 public:
    /// NOTE: If you have already opened descriptor, it's preferred to use
    /// this constructor instead of constructor with path.
-    FileSyncGuard(const DiskPtr & disk_, int fd_) : disk(disk_), fd(fd_) {}
-
-    FileSyncGuard(const DiskPtr & disk_, const String & path)
-        : disk(disk_), fd(disk_->open(path, O_RDWR)) {}
-
-    ~FileSyncGuard()
-    {
-        try
-        {
-            disk->sync(fd);
-            disk->close(fd);
-        }
-        catch (...)
-        {
-            tryLogCurrentException(__PRETTY_FUNCTION__);
-        }
-    }
+    DirectorySyncGuard(const DiskPtr & disk_, int fd_) : disk(disk_), fd(fd_) {}
+    DirectorySyncGuard(const DiskPtr & disk_, const std::string & path);
+    ~DirectorySyncGuard();

 private:
    DiskPtr disk;
--- a/src/Common/Elf.cpp
+++ b/src/Common/Elf.cpp
@ -151,6 +151,15 @@ String Elf::getBuildID(const char * nhdr_pos, size_t size)
 }


+String Elf::getBinaryHash() const
+{
+    if (auto section = findSectionByName(".note.ClickHouse.hash"))
+        return {section->begin(), section->end()};
+    else
+        return {};
+}
+
+
 const char * Elf::Section::name() const
 {
    if (!elf.section_names)
--- a/src/Common/Elf.h
+++ b/src/Common/Elf.h
@ -59,6 +59,9 @@ public:
    String getBuildID() const;
    static String getBuildID(const char * nhdr_pos, size_t size);

+    /// Hash of the binary for integrity checks.
+    String getBinaryHash() const;
+
 private:
    MMapReadBufferFromFile in;
    size_t elf_size;
--- a/src/Common/ErrorCodes.cpp
+++ b/src/Common/ErrorCodes.cpp
@ -531,6 +531,7 @@
    M(562, TLD_LIST_NOT_FOUND) \
    M(563, CANNOT_READ_MAP_FROM_TEXT) \
    M(564, INTERSERVER_SCHEME_DOESNT_MATCH) \
+    M(565, TOO_MANY_PARTITIONS) \
    \
    M(999, KEEPER_EXCEPTION) \
    M(1000, POCO_EXCEPTION) \
--- a/src/Common/FiberStack.h
+++ b/src/Common/FiberStack.h
@ -2,7 +2,7 @@
 #include <common/defines.h>
 #include <boost/context/stack_context.hpp>
 #include <Common/formatReadable.h>
-#include <Common/MemoryTracker.h>
+#include <Common/CurrentMemoryTracker.h>

 #include <sys/time.h>
 #include <sys/resource.h>
--- a/src/Common/FileChecker.cpp
+++ b/src/Common/FileChecker.cpp
@ -21,18 +21,16 @@ namespace ErrorCodes
 FileChecker::FileChecker(DiskPtr disk_, const String & file_info_path_) : disk(std::move(disk_))
 {
    setPath(file_info_path_);
+    load();
 }

 void FileChecker::setPath(const String & file_info_path_)
 {
    files_info_path = file_info_path_;
-
-    tmp_files_info_path = parentPath(files_info_path) + "tmp_" + fileName(files_info_path);
 }

 void FileChecker::update(const String & full_file_path)
 {
-    initialize();
    map[fileName(full_file_path)] = disk->getFileSize(full_file_path);
 }

@ -41,19 +39,19 @@ void FileChecker::setEmpty(const String & full_file_path)
    map[fileName(full_file_path)] = 0;
 }

+FileChecker::Map FileChecker::getFileSizes() const
+{
+    return map;
+}
+
 CheckResults FileChecker::check() const
 {
-    // Read the files again every time you call `check` - so as not to violate the constancy.
-    // `check` method is rarely called.
-
-    CheckResults results;
-    Map local_map;
-    load(local_map, files_info_path);
-
-    if (local_map.empty())
+    if (map.empty())
        return {};

-    for (const auto & name_size : local_map)
+    CheckResults results;
+
+    for (const auto & name_size : map)
    {
        const String & name = name_size.first;
        String path = parentPath(files_info_path) + name;
@ -97,17 +95,10 @@ void FileChecker::repair()
    }
 }

-void FileChecker::initialize()
-{
-    if (initialized)
-        return;
-
-    load(map, files_info_path);
-    initialized = true;
-}
-
 void FileChecker::save() const
 {
+    std::string tmp_files_info_path = parentPath(files_info_path) + "tmp_" + fileName(files_info_path);
+
    {
        std::unique_ptr<WriteBuffer> out = disk->writeFile(tmp_files_info_path);

@ -134,14 +125,14 @@ void FileChecker::save() const
    disk->replaceFile(tmp_files_info_path, files_info_path);
 }

-void FileChecker::load(Map & local_map, const String & path) const
+void FileChecker::load()
 {
-    local_map.clear();
+    map.clear();

-    if (!disk->exists(path))
+    if (!disk->exists(files_info_path))
        return;

-    std::unique_ptr<ReadBuffer> in = disk->readFile(path);
+    std::unique_ptr<ReadBuffer> in = disk->readFile(files_info_path);
    WriteBufferFromOwnString out;

    /// The JSON library does not support whitespace. We delete them. Inefficient.
@ -156,7 +147,7 @@ void FileChecker::load(Map & local_map, const String & path) const

    JSON files = json["yandex"];
    for (const JSON file : files) // NOLINT
-        local_map[unescapeForFileName(file.getName())] = file.getValue()["size"].toUInt();
+        map[unescapeForFileName(file.getName())] = file.getValue()["size"].toUInt();
 }

 }
--- a/src/Common/FileChecker.h
+++ b/src/Common/FileChecker.h
@ -18,6 +18,7 @@ public:
    void update(const String & full_file_path);
    void setEmpty(const String & full_file_path);
    void save() const;
+    bool empty() const { return map.empty(); }

    /// Check the files whose parameters are specified in sizes.json
    CheckResults check() const;
@ -27,21 +28,18 @@ public:
    /// The purpose of this function is to rollback a group of unfinished writes.
    void repair();

-private:
    /// File name -> size.
    using Map = std::map<String, UInt64>;

-    void initialize();
-    void updateImpl(const String & file_path);
-    void load(Map & local_map, const String & path) const;
+    Map getFileSizes() const;
+
+private:
+    void load();

    DiskPtr disk;
    String files_info_path;
-    String tmp_files_info_path;

-    /// The data from the file is read lazily.
    Map map;
-    bool initialized = false;

    Poco::Logger * log = &Poco::Logger::get("FileChecker");
 };
--- a/Show More
+++ b/Show More
				`@ -0,0 +1 @@`
				`A library that traps whenever harmful functions from libc are called.`