Merge branch 'master' into merging_external_source_cassandra

CMakeLists.txt

@@ -327,20 +327,16 @@ message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE
 
 include (GNUInstallDirs)
 include (cmake/contrib_finder.cmake)
-include (cmake/lib_name.cmake)
 
 find_contrib_lib(double-conversion) # Must be before parquet
 include (cmake/find/ssl.cmake)
 include (cmake/find/ldap.cmake) # after ssl
 include (cmake/find/icu.cmake)
-include (cmake/find/boost.cmake)
 include (cmake/find/zlib.cmake)
 include (cmake/find/zstd.cmake)
 include (cmake/find/ltdl.cmake) # for odbc
 include (cmake/find/termcap.cmake)
 # openssl, zlib before poco
-include (cmake/find/lz4.cmake)
-include (cmake/find/xxhash.cmake)
 include (cmake/find/sparsehash.cmake)
 include (cmake/find/re2.cmake)
 include (cmake/find/libgsasl.cmake)
@@ -358,7 +354,6 @@ include (cmake/find/hdfs3.cmake) # uses protobuf
 include (cmake/find/s3.cmake)
 include (cmake/find/base64.cmake)
 include (cmake/find/parquet.cmake)
-include (cmake/find/hyperscan.cmake)
 include (cmake/find/simdjson.cmake)
 include (cmake/find/rapidjson.cmake)
 include (cmake/find/fastops.cmake)
@@ -369,7 +364,6 @@ include (cmake/find/cassandra.cmake)
 
 find_contrib_lib(cityhash)
 find_contrib_lib(farmhash)
-find_contrib_lib(metrohash)
 find_contrib_lib(btrie)
 
 if (ENABLE_TESTS)

base/common/CMakeLists.txt

@@ -16,6 +16,7 @@ set (SRCS
     shift10.cpp
     sleep.cpp
     terminalColors.cpp
+    errnoToString.cpp
 )
 
 if (ENABLE_REPLXX)
@@ -43,10 +44,6 @@ endif()
 
 target_include_directories(common PUBLIC .. ${CMAKE_CURRENT_BINARY_DIR}/..)
 
-if (NOT USE_INTERNAL_BOOST_LIBRARY)
-    target_include_directories (common SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
-endif ()
-
 # Allow explicit fallback to readline
 if (NOT ENABLE_REPLXX AND ENABLE_READLINE)
     message (STATUS "Attempt to fallback to readline explicitly")
@@ -72,7 +69,8 @@ endif ()
 target_link_libraries (common
     PUBLIC
         ${CITYHASH_LIBRARIES}
-        ${Boost_SYSTEM_LIBRARY}
+        boost::headers_only
+        boost::system
         FastMemcpy
         Poco::Net
         Poco::Net::SSL

base/common/ReplxxLineReader.cpp

@@ -1,9 +1,11 @@
 #include <common/ReplxxLineReader.h>
+#include <common/errnoToString.h>
 
 #include <errno.h>
 #include <string.h>
 #include <unistd.h>
 #include <functional>
+#include <sys/file.h>
 
 namespace
 {
@@ -17,15 +19,42 @@ void trim(String & s)
 }
 
 ReplxxLineReader::ReplxxLineReader(
-    const Suggest & suggest, const String & history_file_path_, bool multiline_, Patterns extenders_, Patterns delimiters_)
+    const Suggest & suggest,
-    : LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_))
+    const String & history_file_path_,
+    bool multiline_,
+    Patterns extenders_,
+    Patterns delimiters_,
+    replxx::Replxx::highlighter_callback_t highlighter_)
+    : LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_)), highlighter(std::move(highlighter_))
 {
     using namespace std::placeholders;
     using Replxx = replxx::Replxx;
 
     if (!history_file_path.empty())
+    {
+        history_file_fd = open(history_file_path.c_str(), O_RDWR);
+        if (history_file_fd < 0)
+        {
+            rx.print("Open of history file failed: %s\n", errnoToString(errno).c_str());
+        }
+        else
+        {
+            if (flock(history_file_fd, LOCK_SH))
+            {
+                rx.print("Shared lock of history file failed: %s\n", errnoToString(errno).c_str());
+            }
+            else
+            {
                 rx.history_load(history_file_path);
 
+                if (flock(history_file_fd, LOCK_UN))
+                {
+                    rx.print("Unlock of history file failed: %s\n", errnoToString(errno).c_str());
+                }
+            }
+        }
+    }
+
     auto callback = [&suggest] (const String & context, size_t context_size)
     {
         auto range = suggest.getCompletions(context, context_size);
@@ -36,6 +65,9 @@ ReplxxLineReader::ReplxxLineReader(
     rx.set_complete_on_empty(false);
     rx.set_word_break_characters(word_break_characters);
 
+    if (highlighter)
+        rx.set_highlighter_callback(highlighter);
+
     /// By default C-p/C-n binded to COMPLETE_NEXT/COMPLETE_PREV,
     /// bind C-p/C-n to history-previous/history-next like readline.
     rx.bind_key(Replxx::KEY::control('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_NEXT, code); });
@@ -49,8 +81,8 @@ ReplxxLineReader::ReplxxLineReader(
 
 ReplxxLineReader::~ReplxxLineReader()
 {
-    if (!history_file_path.empty())
+    if (close(history_file_fd))
-        rx.history_save(history_file_path);
+        rx.print("Close of history file failed: %s\n", strerror(errno));
 }
 
 LineReader::InputStatus ReplxxLineReader::readOneLine(const String & prompt)
@@ -68,7 +100,20 @@ LineReader::InputStatus ReplxxLineReader::readOneLine(const String & prompt)
 
 void ReplxxLineReader::addToHistory(const String & line)
 {
+    // locking history file to prevent from inconsistent concurrent changes
+    bool locked = false;
+    if (flock(history_file_fd, LOCK_EX))
+        rx.print("Lock of history file failed: %s\n", strerror(errno));
+    else
+        locked = true;
+
     rx.history_add(line);
+
+    // flush changes to the disk
+    rx.history_save(history_file_path);
+
+    if (locked && 0 != flock(history_file_fd, LOCK_UN))
+        rx.print("Unlock of history file failed: %s\n", strerror(errno));
 }
 
 void ReplxxLineReader::enableBracketedPaste()

base/common/ReplxxLineReader.h

@@ -4,10 +4,17 @@
 
 #include <replxx.hxx>
 
+
 class ReplxxLineReader : public LineReader
 {
 public:
-    ReplxxLineReader(const Suggest & suggest, const String & history_file_path, bool multiline, Patterns extenders_, Patterns delimiters_);
+    ReplxxLineReader(
+        const Suggest & suggest,
+        const String & history_file_path,
+        bool multiline,
+        Patterns extenders_,
+        Patterns delimiters_,
+        replxx::Replxx::highlighter_callback_t highlighter_);
     ~ReplxxLineReader() override;
 
     void enableBracketedPaste() override;
@@ -17,4 +24,8 @@ private:
     void addToHistory(const String & line) override;
 
     replxx::Replxx rx;
+    replxx::Replxx::highlighter_callback_t highlighter;
+
+    // used to call flock() to synchronize multiple clients using same history file
+    int history_file_fd = -1;
 };

base/common/errnoToString.cpp

@@ -0,0 +1,29 @@
+#include "errnoToString.h"
+
+#include <fmt/format.h>
+
+
+std::string errnoToString(int code, int the_errno)
+{
+    const size_t buf_size = 128;
+    char buf[buf_size];
+#ifndef _GNU_SOURCE
+    int rc = strerror_r(the_errno, buf, buf_size);
+#ifdef __APPLE__
+    if (rc != 0 && rc != EINVAL)
+#else
+    if (rc != 0)
+#endif
+    {
+        std::string tmp = std::to_string(code);
+        const char * code_str = tmp.c_str();
+        const char * unknown_message = "Unknown error ";
+        strcpy(buf, unknown_message);
+        strcpy(buf + strlen(unknown_message), code_str);
+    }
+    return fmt::format("errno: {}, strerror: {}", the_errno, buf);
+#else
+    (void)code;
+    return fmt::format("errno: {}, strerror: {}", the_errno, strerror_r(the_errno, buf, sizeof(buf)));
+#endif
+}

base/common/errnoToString.h

@@ -0,0 +1,6 @@
+#pragma once
+
+#include <cerrno>
+#include <string>
+
+std::string errnoToString(int code, int the_errno = errno);

base/common/strong_typedef.h

@@ -1,6 +1,8 @@
 #pragma once
 
+#include <functional>
 #include <type_traits>
+#include <utility>
 
 template <class T, class Tag>
 struct StrongTypedef

base/common/ya.make

@@ -47,6 +47,7 @@ SRCS(
     shift10.cpp
     sleep.cpp
     terminalColors.cpp
+    errnoToString.cpp
 )
 
 END()

base/mysqlxx/CMakeLists.txt

@@ -32,10 +32,18 @@ else ()
     endif ()
 endif ()
 
-target_link_libraries(mysqlxx PUBLIC common PRIVATE ${MYSQLCLIENT_LIBRARIES} PUBLIC ${Boost_SYSTEM_LIBRARY} PRIVATE ${ZLIB_LIBRARIES})
+target_link_libraries (mysqlxx
+    PUBLIC
+        common
+    PRIVATE
+        ${MYSQLCLIENT_LIBRARIES}
+        ${ZLIB_LIBRARIES}
+)
+
 if(OPENSSL_LIBRARIES)
     target_link_libraries(mysqlxx PRIVATE ${OPENSSL_LIBRARIES})
 endif()
+
 target_link_libraries(mysqlxx PRIVATE ${PLATFORM_LIBRARIES})
 
 if (NOT USE_INTERNAL_MYSQL_LIBRARY AND OPENSSL_INCLUDE_DIR)

cmake/Modules/Findmetrohash.cmake

@@ -1,44 +0,0 @@
-# - Try to find metrohash headers and libraries.
-#
-# Usage of this module as follows:
-#
-#     find_package(metrohash)
-#
-# Variables used by this module, they can change the default behaviour and need
-# to be set before calling find_package:
-#
-#  METROHASH_ROOT_DIR Set this variable to the root installation of
-#                    metrohash if the module has problems finding
-#                    the proper installation path.
-#
-# Variables defined by this module:
-#
-#  METROHASH_FOUND             System has metrohash libs/headers
-#  METROHASH_LIBRARIES         The metrohash library/libraries
-#  METROHASH_INCLUDE_DIR       The location of metrohash headers
-
-find_path(METROHASH_ROOT_DIR
-    NAMES include/metrohash.h
-)
-
-find_library(METROHASH_LIBRARIES
-    NAMES metrohash
-    PATHS ${METROHASH_ROOT_DIR}/lib ${METROHASH_LIBRARIES_PATHS}
-)
-
-find_path(METROHASH_INCLUDE_DIR
-    NAMES metrohash.h
-    PATHS ${METROHASH_ROOT_DIR}/include PATH_SUFFIXES metrohash ${METROHASH_INCLUDE_PATHS}
-)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(metrohash DEFAULT_MSG
-    METROHASH_LIBRARIES
-    METROHASH_INCLUDE_DIR
-)
-
-mark_as_advanced(
-    METROHASH_ROOT_DIR
-    METROHASH_LIBRARIES
-    METROHASH_INCLUDE_DIR
-)

cmake/find/boost.cmake

@@ -1,52 +0,0 @@
-option (USE_INTERNAL_BOOST_LIBRARY "Set to FALSE to use system boost library instead of bundled" ${NOT_UNBUNDLED})
-
-# Test random file existing in all package variants
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boost/libs/system/src/error_code.cpp")
-    if(USE_INTERNAL_BOOST_LIBRARY)
-        message(WARNING "submodules in contrib/boost is missing. to fix try run: \n git submodule update --init --recursive")
-    endif()
-    set (USE_INTERNAL_BOOST_LIBRARY 0)
-    set (MISSING_INTERNAL_BOOST_LIBRARY 1)
-endif ()
-
-if (NOT USE_INTERNAL_BOOST_LIBRARY)
-    set (Boost_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES})
-    set (BOOST_ROOT "/usr/local")
-    find_package (Boost 1.60 COMPONENTS program_options system filesystem thread regex)
-    # incomplete, no include search, who use it?
-    if (NOT Boost_FOUND)
-        #    # Try to find manually.
-        #    set (BOOST_PATHS "")
-        #    find_library (Boost_PROGRAM_OPTIONS_LIBRARY boost_program_options PATHS ${BOOST_PATHS})
-        #    find_library (Boost_SYSTEM_LIBRARY boost_system PATHS ${BOOST_PATHS})
-        #    find_library (Boost_FILESYSTEM_LIBRARY boost_filesystem PATHS ${BOOST_PATHS})
-        # maybe found but incorrect version.
-        set (Boost_INCLUDE_DIRS "")
-        set (Boost_SYSTEM_LIBRARY "")
-    endif ()
-endif ()
-
-if (NOT Boost_SYSTEM_LIBRARY AND NOT MISSING_INTERNAL_BOOST_LIBRARY)
-    set (USE_INTERNAL_BOOST_LIBRARY 1)
-    set (Boost_SYSTEM_LIBRARY boost_system_internal)
-    set (Boost_PROGRAM_OPTIONS_LIBRARY boost_program_options_internal)
-    set (Boost_FILESYSTEM_LIBRARY boost_filesystem_internal ${Boost_SYSTEM_LIBRARY})
-    set (Boost_IOSTREAMS_LIBRARY boost_iostreams_internal)
-    set (Boost_REGEX_LIBRARY boost_regex_internal)
-
-    set (Boost_INCLUDE_DIRS)
-
-    set (BOOST_ROOT "${ClickHouse_SOURCE_DIR}/contrib/boost")
-
-    # For boost from github:
-    file (GLOB Boost_INCLUDE_DIRS_ "${ClickHouse_SOURCE_DIR}/contrib/boost/libs/*/include")
-    list (APPEND Boost_INCLUDE_DIRS ${Boost_INCLUDE_DIRS_})
-    # numeric has additional level
-    file (GLOB Boost_INCLUDE_DIRS_ "${ClickHouse_SOURCE_DIR}/contrib/boost/libs/numeric/*/include")
-    list (APPEND Boost_INCLUDE_DIRS ${Boost_INCLUDE_DIRS_})
-
-    # For packaged version:
-    list (APPEND Boost_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/boost")
-endif ()
-
-message (STATUS "Using Boost: ${Boost_INCLUDE_DIRS} : ${Boost_PROGRAM_OPTIONS_LIBRARY},${Boost_SYSTEM_LIBRARY},${Boost_FILESYSTEM_LIBRARY},${Boost_IOSTREAMS_LIBRARY},${Boost_REGEX_LIBRARY}")

cmake/find/hyperscan.cmake

@@ -1,33 +0,0 @@
-if (HAVE_SSSE3)
-    option (ENABLE_HYPERSCAN "Enable hyperscan" ${ENABLE_LIBRARIES})
-endif ()
-
-if (ENABLE_HYPERSCAN)
-
-option (USE_INTERNAL_HYPERSCAN_LIBRARY "Set to FALSE to use system hyperscan instead of the bundled" ${NOT_UNBUNDLED})
-
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/hyperscan/CMakeLists.txt")
-    if (USE_INTERNAL_HYPERSCAN_LIBRARY)
-        message (WARNING "submodule contrib/hyperscan is missing. to fix try run: \n git submodule update --init --recursive")
-    endif ()
-   set (MISSING_INTERNAL_HYPERSCAN_LIBRARY 1)
-   set (USE_INTERNAL_HYPERSCAN_LIBRARY 0)
-endif ()
-
-if (NOT USE_INTERNAL_HYPERSCAN_LIBRARY)
-    find_library (HYPERSCAN_LIBRARY hs)
-    find_path (HYPERSCAN_INCLUDE_DIR NAMES hs/hs.h hs.h PATHS ${HYPERSCAN_INCLUDE_PATHS})
-endif ()
-
-if (HYPERSCAN_LIBRARY AND HYPERSCAN_INCLUDE_DIR)
-    set (USE_HYPERSCAN 1)
-elseif (NOT MISSING_INTERNAL_HYPERSCAN_LIBRARY)
-    set (HYPERSCAN_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/hyperscan/src)
-    set (HYPERSCAN_LIBRARY hs)
-    set (USE_HYPERSCAN 1)
-    set (USE_INTERNAL_HYPERSCAN_LIBRARY 1)
-endif()
-
-message (STATUS "Using hyperscan=${USE_HYPERSCAN}: ${HYPERSCAN_INCLUDE_DIR} : ${HYPERSCAN_LIBRARY}")
-
-endif ()

cmake/find/lz4.cmake

@@ -1,23 +0,0 @@
-option (USE_INTERNAL_LZ4_LIBRARY "Set to FALSE to use system lz4 library instead of bundled" ${NOT_UNBUNDLED})
-
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/lz4/lib/lz4.h")
-    if (USE_INTERNAL_LZ4_LIBRARY)
-       message (WARNING "submodule contrib/lz4 is missing. to fix try run: \n git submodule update --init --recursive")
-       set (USE_INTERNAL_LZ4_LIBRARY 0)
-    endif ()
-    set (MISSING_INTERNAL_LZ4_LIBRARY 1)
-endif ()
-
-if (NOT USE_INTERNAL_LZ4_LIBRARY)
-    find_library (LZ4_LIBRARY lz4)
-    find_path (LZ4_INCLUDE_DIR NAMES lz4.h PATHS ${LZ4_INCLUDE_PATHS})
-endif ()
-
-if (LZ4_LIBRARY AND LZ4_INCLUDE_DIR)
-elseif (NOT MISSING_INTERNAL_LZ4_LIBRARY)
-    set (LZ4_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/lz4/lib)
-    set (USE_INTERNAL_LZ4_LIBRARY 1)
-    set (LZ4_LIBRARY lz4)
-endif ()
-
-message (STATUS "Using lz4: ${LZ4_INCLUDE_DIR} : ${LZ4_LIBRARY}")

cmake/find/parquet.cmake

@@ -63,7 +63,7 @@ elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD)
         set(ARROW_LIBRARY arrow_shared)
         set(PARQUET_LIBRARY parquet_shared)
         if(USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE)
-            list(APPEND PARQUET_LIBRARY ${Boost_REGEX_LIBRARY})
+            list(APPEND PARQUET_LIBRARY boost::regex)
         endif()
         set(THRIFT_LIBRARY thrift)
     endif()

cmake/find/xxhash.cmake

@@ -1,22 +0,0 @@
-option (USE_INTERNAL_XXHASH_LIBRARY "Set to FALSE to use system xxHash library instead of bundled" ${NOT_UNBUNDLED})
-
-if (USE_INTERNAL_XXHASH_LIBRARY AND NOT USE_INTERNAL_LZ4_LIBRARY)
-    message (WARNING "can not use internal xxhash without internal lz4")
-    set (USE_INTERNAL_XXHASH_LIBRARY 0)
-endif ()
-
-if (USE_INTERNAL_XXHASH_LIBRARY)
-    set (XXHASH_LIBRARY lz4)
-    set (XXHASH_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/lz4/lib)
-else ()
-    find_library (XXHASH_LIBRARY xxhash)
-    find_path (XXHASH_INCLUDE_DIR NAMES xxhash.h PATHS ${XXHASH_INCLUDE_PATHS})
-endif ()
-
-if (XXHASH_LIBRARY AND XXHASH_INCLUDE_DIR)
-    set (USE_XXHASH 1)
-else ()
-    set (USE_XXHASH 0)
-endif ()
-
-message (STATUS "Using xxhash=${USE_XXHASH}: ${XXHASH_INCLUDE_DIR} : ${XXHASH_LIBRARY}")

cmake/lib_name.cmake

@@ -1,4 +0,0 @@
-set(DIVIDE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libdivide)
-set(DBMS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/src ${ClickHouse_BINARY_DIR}/src)
-set(DOUBLE_CONVERSION_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/double-conversion)
-set(METROHASH_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)

cmake/print_include_directories.cmake

@@ -21,11 +21,6 @@ if (TARGET double-conversion)
     list(APPEND dirs ${dirs1})
 endif ()
 
-if (TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY})
-    get_property (dirs1 TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY} PROPERTY INCLUDE_DIRECTORIES)
-    list(APPEND dirs ${dirs1})
-endif ()
-
 list(REMOVE_DUPLICATES dirs)
 file (WRITE ${CMAKE_CURRENT_BINARY_DIR}/include_directories.txt "")
 foreach (dir ${dirs})

contrib/CMakeLists.txt

@@ -16,13 +16,18 @@ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
 
 set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1)
 
+add_subdirectory (boost-cmake)
 add_subdirectory (cctz-cmake)
 add_subdirectory (consistent-hashing-sumbur)
 add_subdirectory (consistent-hashing)
 add_subdirectory (croaring)
 add_subdirectory (FastMemcpy)
+add_subdirectory (hyperscan-cmake)
 add_subdirectory (jemalloc-cmake)
 add_subdirectory (libcpuid-cmake)
+add_subdirectory (libdivide)
+add_subdirectory (libmetrohash)
+add_subdirectory (lz4-cmake)
 add_subdirectory (murmurhash)
 add_subdirectory (replxx-cmake)
 add_subdirectory (ryu-cmake)
@@ -33,14 +38,6 @@ add_subdirectory (poco-cmake)
 
 # TODO: refactor the contrib libraries below this comment.
 
-if (USE_INTERNAL_BOOST_LIBRARY)
-    add_subdirectory (boost-cmake)
-endif ()
-
-if (USE_INTERNAL_LZ4_LIBRARY)
-    add_subdirectory (lz4-cmake)
-endif ()
-
 if (USE_INTERNAL_ZSTD_LIBRARY)
     add_subdirectory (zstd-cmake)
 endif ()
@@ -63,10 +60,6 @@ if (USE_INTERNAL_FARMHASH_LIBRARY)
     add_subdirectory (libfarmhash)
 endif ()
 
-if (USE_INTERNAL_METROHASH_LIBRARY)
-    add_subdirectory (libmetrohash)
-endif ()
-
 if (USE_INTERNAL_BTRIE_LIBRARY)
     add_subdirectory (libbtrie)
 endif ()
@@ -294,18 +287,6 @@ if (USE_BASE64)
     add_subdirectory (base64-cmake)
 endif()
 
-if (USE_INTERNAL_HYPERSCAN_LIBRARY)
-    # The library is large - avoid bloat.
-    if (USE_STATIC_LIBRARIES)
-        add_subdirectory (hyperscan)
-        target_compile_options (hs PRIVATE -g0)
-    else ()
-        set(BUILD_SHARED_LIBS 1 CACHE INTERNAL "")
-        add_subdirectory (hyperscan)
-        target_compile_options (hs_shared PRIVATE -g0)
-    endif ()
-endif()
-
 if (USE_SIMDJSON)
     add_subdirectory (simdjson-cmake)
 endif()

contrib/arrow-cmake/CMakeLists.txt

@@ -47,7 +47,8 @@ set(thriftcpp_threads_SOURCES
         )
 add_library(${THRIFT_LIBRARY} ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES})
 set_target_properties(${THRIFT_LIBRARY} PROPERTIES CXX_STANDARD 14) # REMOVE after https://github.com/apache/thrift/pull/1641
-target_include_directories(${THRIFT_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src PRIVATE ${Boost_INCLUDE_DIRS})
+target_include_directories(${THRIFT_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src)
+target_link_libraries (${THRIFT_LIBRARY} PRIVATE boost::headers_only)
 
 
 # === orc
@@ -286,10 +287,6 @@ set(ARROW_SRCS ${ARROW_SRCS}
         ${LIBRARY_DIR}/compute/kernels/util_internal.cc
         )
 
-if (LZ4_INCLUDE_DIR AND LZ4_LIBRARY)
-    set(ARROW_WITH_LZ4 1)
-endif ()
-
 if (SNAPPY_INCLUDE_DIR AND SNAPPY_LIBRARY)
     set(ARROW_WITH_SNAPPY 1)
 endif ()
@@ -302,10 +299,8 @@ if (ZSTD_INCLUDE_DIR AND ZSTD_LIBRARY)
     set(ARROW_WITH_ZSTD 1)
 endif ()
 
-if (ARROW_WITH_LZ4)
 add_definitions(-DARROW_WITH_LZ4)
 SET(ARROW_SRCS ${LIBRARY_DIR}/util/compression_lz4.cc ${ARROW_SRCS})
-endif ()
 
 if (ARROW_WITH_SNAPPY)
     add_definitions(-DARROW_WITH_SNAPPY)
@@ -328,18 +323,15 @@ add_library(${ARROW_LIBRARY} ${ARROW_SRCS})
 # Arrow dependencies
 add_dependencies(${ARROW_LIBRARY} ${FLATBUFFERS_LIBRARY} metadata_fbs)
 
-target_link_libraries(${ARROW_LIBRARY} PRIVATE boost_system_internal boost_filesystem_internal boost_regex_internal)
+target_link_libraries(${ARROW_LIBRARY} PRIVATE ${FLATBUFFERS_LIBRARY} boost::filesystem)
-target_link_libraries(${ARROW_LIBRARY} PRIVATE ${FLATBUFFERS_LIBRARY})
 
 if (USE_INTERNAL_PROTOBUF_LIBRARY)
     add_dependencies(${ARROW_LIBRARY} protoc)
 endif ()
 
-target_include_directories(${ARROW_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src ${Boost_INCLUDE_DIRS})
+target_include_directories(${ARROW_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src)
 target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY})
-if (ARROW_WITH_LZ4)
+target_link_libraries(${ARROW_LIBRARY} PRIVATE lz4)
-    target_link_libraries(${ARROW_LIBRARY} PRIVATE ${LZ4_LIBRARY})
-endif ()
 if (ARROW_WITH_SNAPPY)
     target_link_libraries(${ARROW_LIBRARY} PRIVATE ${SNAPPY_LIBRARY})
 endif ()
@@ -396,8 +388,7 @@ list(APPEND PARQUET_SRCS
 add_library(${PARQUET_LIBRARY} ${PARQUET_SRCS})
 target_include_directories(${PARQUET_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src)
 include(${ClickHouse_SOURCE_DIR}/contrib/thrift/build/cmake/ConfigureChecks.cmake) # makes config.h
-target_link_libraries(${PARQUET_LIBRARY} PUBLIC ${ARROW_LIBRARY} PRIVATE ${THRIFT_LIBRARY} ${Boost_REGEX_LIBRARY})
+target_link_libraries(${PARQUET_LIBRARY} PUBLIC ${ARROW_LIBRARY} PRIVATE ${THRIFT_LIBRARY} boost::headers_only boost::regex)
-target_include_directories(${PARQUET_LIBRARY} PRIVATE ${Boost_INCLUDE_DIRS})
 
 if (SANITIZE STREQUAL "undefined")
     target_compile_options(${PARQUET_LIBRARY} PRIVATE -fno-sanitize=undefined)

contrib/avro-cmake/CMakeLists.txt

@@ -45,13 +45,12 @@ set_target_properties (avrocpp PROPERTIES VERSION ${AVRO_VERSION_MAJOR}.${AVRO_V
 
 target_include_directories(avrocpp SYSTEM PUBLIC ${AVROCPP_INCLUDE_DIR})
 
-target_include_directories(avrocpp SYSTEM PUBLIC ${Boost_INCLUDE_DIRS})
+target_link_libraries (avrocpp PRIVATE boost::headers_only boost::iostreams)
-target_link_libraries (avrocpp ${Boost_IOSTREAMS_LIBRARY})
 
 if (SNAPPY_INCLUDE_DIR AND SNAPPY_LIBRARY)
     target_compile_definitions (avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE)
     target_include_directories (avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR})
-    target_link_libraries (avrocpp ${SNAPPY_LIBRARY})
+    target_link_libraries (avrocpp PRIVATE ${SNAPPY_LIBRARY})
 endif ()
 
 if (COMPILER_GCC)

contrib/aws

@@ -1 +1 @@
-Subproject commit fb5c604525f5151d75a856462653e7e38b559b79
+Subproject commit f7d9ce39f41323300044567be007c233338bb94a

contrib/boost-cmake/CMakeLists.txt

@@ -1,45 +1,101 @@
-# Supported contrib/boost source variants:
+option (USE_INTERNAL_BOOST_LIBRARY "Use internal Boost library" ${NOT_UNBUNDLED})
-# 1. Default - Minimized vrsion from release archive : https://github.com/ClickHouse-Extras/boost
-# 2. Release archive unpacked to contrib/boost
-# 3. Full boost https://github.com/boostorg/boost
-
-# if boostorg/boost connected as submodule: Update all boost internal submodules to tag:
-# git submodule foreach "git fetch --all && git checkout boost-1.66.0 || true"
-
-#
-# Important boost patch: 094c18b
-#
-
-include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 
+if (USE_INTERNAL_BOOST_LIBRARY)
     set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/boost)
 
-if(NOT MSVC)
+    # filesystem
-    add_definitions(-Wno-unused-variable -Wno-deprecated-declarations)
+
-endif()
+    set (SRCS_FILESYSTEM
-
+        ${LIBRARY_DIR}/libs/filesystem/src/codecvt_error_category.cpp
-macro(add_boost_lib lib_name)
+        ${LIBRARY_DIR}/libs/filesystem/src/operations.cpp
-    add_headers_and_sources(boost_${lib_name} ${LIBRARY_DIR}/libs/${lib_name}/src)
+        ${LIBRARY_DIR}/libs/filesystem/src/path_traits.cpp
-    add_library(boost_${lib_name}_internal ${boost_${lib_name}_sources})
+        ${LIBRARY_DIR}/libs/filesystem/src/path.cpp
-    target_include_directories(boost_${lib_name}_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
+        ${LIBRARY_DIR}/libs/filesystem/src/portability.cpp
-    target_compile_definitions(boost_${lib_name}_internal PUBLIC BOOST_SYSTEM_NO_DEPRECATED)
+        ${LIBRARY_DIR}/libs/filesystem/src/unique_path.cpp
-endmacro()
+        ${LIBRARY_DIR}/libs/filesystem/src/utf8_codecvt_facet.cpp
-
+        ${LIBRARY_DIR}/libs/filesystem/src/windows_file_codecvt.cpp
-add_boost_lib(system)
+    )
-
+
-add_boost_lib(program_options)
+    add_library (_boost_filesystem ${SRCS_FILESYSTEM})
-
+    add_library (boost::filesystem ALIAS _boost_filesystem)
-add_boost_lib(filesystem)
+    target_include_directories (_boost_filesystem SYSTEM BEFORE PUBLIC ${LIBRARY_DIR})
-target_link_libraries(boost_filesystem_internal PRIVATE boost_system_internal)
+
-
+    # headers-only
-#add_boost_lib(random)
+
-
+    add_library (_boost_headers_only INTERFACE)
-if (USE_INTERNAL_PARQUET_LIBRARY)
+    add_library (boost::headers_only ALIAS _boost_headers_only)
-    add_boost_lib(regex)
+    target_include_directories (_boost_headers_only SYSTEM BEFORE INTERFACE ${LIBRARY_DIR})
-endif()
+
-
+    # iostreams
-if (USE_INTERNAL_AVRO_LIBRARY)
+
-    add_boost_lib(iostreams)
+    set (SRCS_IOSTREAMS
-    target_link_libraries(boost_iostreams_internal PUBLIC ${ZLIB_LIBRARIES})
+        ${LIBRARY_DIR}/libs/iostreams/src/file_descriptor.cpp
-    target_include_directories(boost_iostreams_internal SYSTEM BEFORE PRIVATE ${ZLIB_INCLUDE_DIR})
+        ${LIBRARY_DIR}/libs/iostreams/src/gzip.cpp
+        ${LIBRARY_DIR}/libs/iostreams/src/mapped_file.cpp
+        ${LIBRARY_DIR}/libs/iostreams/src/zlib.cpp
+    )
+
+    add_library (_boost_iostreams ${SRCS_IOSTREAMS})
+    add_library (boost::iostreams ALIAS _boost_iostreams)
+    target_include_directories (_boost_iostreams PRIVATE ${LIBRARY_DIR})
+    target_link_libraries (_boost_iostreams PRIVATE zlib)
+
+    # program_options
+
+    set (SRCS_PROGRAM_OPTIONS
+        ${LIBRARY_DIR}/libs/program_options/src/cmdline.cpp
+        ${LIBRARY_DIR}/libs/program_options/src/config_file.cpp
+        ${LIBRARY_DIR}/libs/program_options/src/convert.cpp
+        ${LIBRARY_DIR}/libs/program_options/src/options_description.cpp
+        ${LIBRARY_DIR}/libs/program_options/src/parsers.cpp
+        ${LIBRARY_DIR}/libs/program_options/src/positional_options.cpp
+        ${LIBRARY_DIR}/libs/program_options/src/split.cpp
+        ${LIBRARY_DIR}/libs/program_options/src/utf8_codecvt_facet.cpp
+        ${LIBRARY_DIR}/libs/program_options/src/value_semantic.cpp
+        ${LIBRARY_DIR}/libs/program_options/src/variables_map.cpp
+        ${LIBRARY_DIR}/libs/program_options/src/winmain.cpp
+    )
+
+    add_library (_boost_program_options ${SRCS_PROGRAM_OPTIONS})
+    add_library (boost::program_options ALIAS _boost_program_options)
+    target_include_directories (_boost_program_options SYSTEM BEFORE PUBLIC ${LIBRARY_DIR})
+
+    # regex
+
+    set (SRCS_REGEX
+        ${LIBRARY_DIR}/libs/regex/src/c_regex_traits.cpp
+        ${LIBRARY_DIR}/libs/regex/src/cpp_regex_traits.cpp
+        ${LIBRARY_DIR}/libs/regex/src/cregex.cpp
+        ${LIBRARY_DIR}/libs/regex/src/fileiter.cpp
+        ${LIBRARY_DIR}/libs/regex/src/icu.cpp
+        ${LIBRARY_DIR}/libs/regex/src/instances.cpp
+        ${LIBRARY_DIR}/libs/regex/src/internals.hpp
+        ${LIBRARY_DIR}/libs/regex/src/posix_api.cpp
+        ${LIBRARY_DIR}/libs/regex/src/regex_debug.cpp
+        ${LIBRARY_DIR}/libs/regex/src/regex_raw_buffer.cpp
+        ${LIBRARY_DIR}/libs/regex/src/regex_traits_defaults.cpp
+        ${LIBRARY_DIR}/libs/regex/src/regex.cpp
+        ${LIBRARY_DIR}/libs/regex/src/static_mutex.cpp
+        ${LIBRARY_DIR}/libs/regex/src/usinstances.cpp
+        ${LIBRARY_DIR}/libs/regex/src/w32_regex_traits.cpp
+        ${LIBRARY_DIR}/libs/regex/src/wc_regex_traits.cpp
+        ${LIBRARY_DIR}/libs/regex/src/wide_posix_api.cpp
+        ${LIBRARY_DIR}/libs/regex/src/winstances.cpp
+    )
+
+    add_library (_boost_regex ${SRCS_REGEX})
+    add_library (boost::regex ALIAS _boost_regex)
+    target_include_directories (_boost_regex PRIVATE ${LIBRARY_DIR})
+
+    # system
+
+    set (SRCS_SYSTEM
+        ${LIBRARY_DIR}/libs/system/src/error_code.cpp
+    )
+
+    add_library (_boost_system ${SRCS_SYSTEM})
+    add_library (boost::system ALIAS _boost_system)
+    target_include_directories (_boost_system PRIVATE ${LIBRARY_DIR})
+else ()
+    message (FATAL_ERROR "TODO: external Boost library is not supported!")
 endif ()

contrib/cppkafka-cmake/CMakeLists.txt

@@ -1,31 +1,33 @@
-set(CPPKAFKA_DIR ${ClickHouse_SOURCE_DIR}/contrib/cppkafka)
+set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/cppkafka)
 
 set(SRCS
-  ${CPPKAFKA_DIR}/src/configuration.cpp
+    ${LIBRARY_DIR}/src/buffer.cpp
-  ${CPPKAFKA_DIR}/src/topic_configuration.cpp
+    ${LIBRARY_DIR}/src/configuration_option.cpp
-  ${CPPKAFKA_DIR}/src/configuration_option.cpp
+    ${LIBRARY_DIR}/src/configuration.cpp
-  ${CPPKAFKA_DIR}/src/exceptions.cpp
+    ${LIBRARY_DIR}/src/consumer.cpp
-  ${CPPKAFKA_DIR}/src/topic.cpp
+    ${LIBRARY_DIR}/src/error.cpp
-  ${CPPKAFKA_DIR}/src/buffer.cpp
+    ${LIBRARY_DIR}/src/event.cpp
-  ${CPPKAFKA_DIR}/src/queue.cpp
+    ${LIBRARY_DIR}/src/exceptions.cpp
-  ${CPPKAFKA_DIR}/src/message.cpp
+    ${LIBRARY_DIR}/src/group_information.cpp
-  ${CPPKAFKA_DIR}/src/message_timestamp.cpp
+    ${LIBRARY_DIR}/src/kafka_handle_base.cpp
-  ${CPPKAFKA_DIR}/src/message_internal.cpp
+    ${LIBRARY_DIR}/src/message_internal.cpp
-  ${CPPKAFKA_DIR}/src/topic_partition.cpp
+    ${LIBRARY_DIR}/src/message_timestamp.cpp
-  ${CPPKAFKA_DIR}/src/topic_partition_list.cpp
+    ${LIBRARY_DIR}/src/message.cpp
-  ${CPPKAFKA_DIR}/src/metadata.cpp
+    ${LIBRARY_DIR}/src/metadata.cpp
-  ${CPPKAFKA_DIR}/src/group_information.cpp
+    ${LIBRARY_DIR}/src/producer.cpp
-  ${CPPKAFKA_DIR}/src/error.cpp
+    ${LIBRARY_DIR}/src/queue.cpp
-  ${CPPKAFKA_DIR}/src/event.cpp
+    ${LIBRARY_DIR}/src/topic_configuration.cpp
-
+    ${LIBRARY_DIR}/src/topic_partition_list.cpp
-  ${CPPKAFKA_DIR}/src/kafka_handle_base.cpp
+    ${LIBRARY_DIR}/src/topic_partition.cpp
-  ${CPPKAFKA_DIR}/src/producer.cpp
+    ${LIBRARY_DIR}/src/topic.cpp
-  ${CPPKAFKA_DIR}/src/consumer.cpp
 )
 
 add_library(cppkafka ${SRCS})
 
-target_link_libraries(cppkafka PRIVATE ${RDKAFKA_LIBRARY})
+target_link_libraries(cppkafka
-target_include_directories(cppkafka PRIVATE ${CPPKAFKA_DIR}/include/cppkafka)
+    PRIVATE
-target_include_directories(cppkafka PRIVATE ${Boost_INCLUDE_DIRS})
+        ${RDKAFKA_LIBRARY}
-target_include_directories(cppkafka SYSTEM PUBLIC ${CPPKAFKA_DIR}/include)
+        boost::headers_only
+)
+target_include_directories(cppkafka PRIVATE ${LIBRARY_DIR}/include/cppkafka)
+target_include_directories(cppkafka SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}/include)

contrib/hyperscan-cmake/CMakeLists.txt

@@ -0,0 +1,252 @@
+option (ENABLE_HYPERSCAN "Enable hyperscan library" ${ENABLE_LIBRARIES})
+
+if (NOT HAVE_SSSE3)
+    set (ENABLE_HYPERSCAN OFF)
+endif ()
+
+if (ENABLE_HYPERSCAN)
+    option (USE_INTERNAL_HYPERSCAN_LIBRARY "Use internal hyperscan library" ${NOT_UNBUNDLED})
+
+    if (USE_INTERNAL_HYPERSCAN_LIBRARY)
+        set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/hyperscan)
+
+        set (SRCS
+            ${LIBRARY_DIR}/src/alloc.c
+            ${LIBRARY_DIR}/src/compiler/asserts.cpp
+            ${LIBRARY_DIR}/src/compiler/compiler.cpp
+            ${LIBRARY_DIR}/src/compiler/error.cpp
+            ${LIBRARY_DIR}/src/crc32.c
+            ${LIBRARY_DIR}/src/database.c
+            ${LIBRARY_DIR}/src/fdr/engine_description.cpp
+            ${LIBRARY_DIR}/src/fdr/fdr_compile_util.cpp
+            ${LIBRARY_DIR}/src/fdr/fdr_compile.cpp
+            ${LIBRARY_DIR}/src/fdr/fdr_confirm_compile.cpp
+            ${LIBRARY_DIR}/src/fdr/fdr_engine_description.cpp
+            ${LIBRARY_DIR}/src/fdr/fdr.c
+            ${LIBRARY_DIR}/src/fdr/flood_compile.cpp
+            ${LIBRARY_DIR}/src/fdr/teddy_compile.cpp
+            ${LIBRARY_DIR}/src/fdr/teddy_engine_description.cpp
+            ${LIBRARY_DIR}/src/fdr/teddy.c
+            ${LIBRARY_DIR}/src/grey.cpp
+            ${LIBRARY_DIR}/src/hs_valid_platform.c
+            ${LIBRARY_DIR}/src/hs_version.c
+            ${LIBRARY_DIR}/src/hs.cpp
+            ${LIBRARY_DIR}/src/hwlm/hwlm_build.cpp
+            ${LIBRARY_DIR}/src/hwlm/hwlm_literal.cpp
+            ${LIBRARY_DIR}/src/hwlm/hwlm.c
+            ${LIBRARY_DIR}/src/hwlm/noodle_build.cpp
+            ${LIBRARY_DIR}/src/hwlm/noodle_engine.c
+            ${LIBRARY_DIR}/src/nfa/accel_dfa_build_strat.cpp
+            ${LIBRARY_DIR}/src/nfa/accel.c
+            ${LIBRARY_DIR}/src/nfa/accelcompile.cpp
+            ${LIBRARY_DIR}/src/nfa/castle.c
+            ${LIBRARY_DIR}/src/nfa/castlecompile.cpp
+            ${LIBRARY_DIR}/src/nfa/dfa_build_strat.cpp
+            ${LIBRARY_DIR}/src/nfa/dfa_min.cpp
+            ${LIBRARY_DIR}/src/nfa/gough.c
+            ${LIBRARY_DIR}/src/nfa/goughcompile_accel.cpp
+            ${LIBRARY_DIR}/src/nfa/goughcompile_reg.cpp
+            ${LIBRARY_DIR}/src/nfa/goughcompile.cpp
+            ${LIBRARY_DIR}/src/nfa/lbr.c
+            ${LIBRARY_DIR}/src/nfa/limex_64.c
+            ${LIBRARY_DIR}/src/nfa/limex_accel.c
+            ${LIBRARY_DIR}/src/nfa/limex_compile.cpp
+            ${LIBRARY_DIR}/src/nfa/limex_native.c
+            ${LIBRARY_DIR}/src/nfa/limex_simd128.c
+            ${LIBRARY_DIR}/src/nfa/limex_simd256.c
+            ${LIBRARY_DIR}/src/nfa/limex_simd384.c
+            ${LIBRARY_DIR}/src/nfa/limex_simd512.c
+            ${LIBRARY_DIR}/src/nfa/mcclellan.c
+            ${LIBRARY_DIR}/src/nfa/mcclellancompile_util.cpp
+            ${LIBRARY_DIR}/src/nfa/mcclellancompile.cpp
+            ${LIBRARY_DIR}/src/nfa/mcsheng_compile.cpp
+            ${LIBRARY_DIR}/src/nfa/mcsheng_data.c
+            ${LIBRARY_DIR}/src/nfa/mcsheng.c
+            ${LIBRARY_DIR}/src/nfa/mpv.c
+            ${LIBRARY_DIR}/src/nfa/mpvcompile.cpp
+            ${LIBRARY_DIR}/src/nfa/nfa_api_dispatch.c
+            ${LIBRARY_DIR}/src/nfa/nfa_build_util.cpp
+            ${LIBRARY_DIR}/src/nfa/rdfa_graph.cpp
+            ${LIBRARY_DIR}/src/nfa/rdfa_merge.cpp
+            ${LIBRARY_DIR}/src/nfa/rdfa.cpp
+            ${LIBRARY_DIR}/src/nfa/repeat.c
+            ${LIBRARY_DIR}/src/nfa/repeatcompile.cpp
+            ${LIBRARY_DIR}/src/nfa/sheng.c
+            ${LIBRARY_DIR}/src/nfa/shengcompile.cpp
+            ${LIBRARY_DIR}/src/nfa/shufti.c
+            ${LIBRARY_DIR}/src/nfa/shufticompile.cpp
+            ${LIBRARY_DIR}/src/nfa/tamarama.c
+            ${LIBRARY_DIR}/src/nfa/tamaramacompile.cpp
+            ${LIBRARY_DIR}/src/nfa/truffle.c
+            ${LIBRARY_DIR}/src/nfa/trufflecompile.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_anchored_acyclic.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_anchored_dots.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_asserts.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_builder.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_calc_components.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_cyclic_redundancy.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_depth.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_dominators.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_edge_redundancy.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_equivalence.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_execute.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_expr_info.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_extparam.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_fixed_width.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_fuzzy.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_haig.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_holder.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_is_equal.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_lbr.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_limex_accel.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_limex.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_literal_analysis.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_literal_component.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_literal_decorated.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_mcclellan.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_misc_opt.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_netflow.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_prefilter.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_prune.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_puff.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_redundancy.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_region_redundancy.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_region.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_repeat.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_reports.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_restructuring.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_revacc.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_sep.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_small_literal_set.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_som_add_redundancy.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_som_util.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_som.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_split.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_squash.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_stop.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_uncalc_components.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_utf8.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_util.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_vacuous.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_violet.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng_width.cpp
+            ${LIBRARY_DIR}/src/nfagraph/ng.cpp
+            ${LIBRARY_DIR}/src/parser/AsciiComponentClass.cpp
+            ${LIBRARY_DIR}/src/parser/buildstate.cpp
+            ${LIBRARY_DIR}/src/parser/check_refs.cpp
+            ${LIBRARY_DIR}/src/parser/Component.cpp
+            ${LIBRARY_DIR}/src/parser/ComponentAlternation.cpp
+            ${LIBRARY_DIR}/src/parser/ComponentAssertion.cpp
+            ${LIBRARY_DIR}/src/parser/ComponentAtomicGroup.cpp
+            ${LIBRARY_DIR}/src/parser/ComponentBackReference.cpp
+            ${LIBRARY_DIR}/src/parser/ComponentBoundary.cpp
+            ${LIBRARY_DIR}/src/parser/ComponentByte.cpp
+            ${LIBRARY_DIR}/src/parser/ComponentClass.cpp
+            ${LIBRARY_DIR}/src/parser/ComponentCondReference.cpp
+            ${LIBRARY_DIR}/src/parser/ComponentEmpty.cpp
+            ${LIBRARY_DIR}/src/parser/ComponentEUS.cpp
+            ${LIBRARY_DIR}/src/parser/ComponentRepeat.cpp
+            ${LIBRARY_DIR}/src/parser/ComponentSequence.cpp
+            ${LIBRARY_DIR}/src/parser/ComponentVisitor.cpp
+            ${LIBRARY_DIR}/src/parser/ComponentWordBoundary.cpp
+            ${LIBRARY_DIR}/src/parser/ConstComponentVisitor.cpp
+            ${LIBRARY_DIR}/src/parser/control_verbs.cpp
+            ${LIBRARY_DIR}/src/parser/logical_combination.cpp
+            ${LIBRARY_DIR}/src/parser/parse_error.cpp
+            ${LIBRARY_DIR}/src/parser/parser_util.cpp
+            ${LIBRARY_DIR}/src/parser/Parser.cpp
+            ${LIBRARY_DIR}/src/parser/prefilter.cpp
+            ${LIBRARY_DIR}/src/parser/shortcut_literal.cpp
+            ${LIBRARY_DIR}/src/parser/ucp_table.cpp
+            ${LIBRARY_DIR}/src/parser/unsupported.cpp
+            ${LIBRARY_DIR}/src/parser/utf8_validate.cpp
+            ${LIBRARY_DIR}/src/parser/Utf8ComponentClass.cpp
+            ${LIBRARY_DIR}/src/rose/block.c
+            ${LIBRARY_DIR}/src/rose/catchup.c
+            ${LIBRARY_DIR}/src/rose/init.c
+            ${LIBRARY_DIR}/src/rose/match.c
+            ${LIBRARY_DIR}/src/rose/program_runtime.c
+            ${LIBRARY_DIR}/src/rose/rose_build_add_mask.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_add.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_anchored.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_bytecode.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_castle.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_compile.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_convert.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_dedupe.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_engine_blob.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_exclusive.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_groups.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_infix.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_instructions.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_lit_accel.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_long_lit.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_lookaround.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_matchers.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_merge.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_misc.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_program.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_role_aliasing.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_scatter.cpp
+            ${LIBRARY_DIR}/src/rose/rose_build_width.cpp
+            ${LIBRARY_DIR}/src/rose/rose_in_util.cpp
+            ${LIBRARY_DIR}/src/rose/stream.c
+            ${LIBRARY_DIR}/src/runtime.c
+            ${LIBRARY_DIR}/src/scratch.c
+            ${LIBRARY_DIR}/src/smallwrite/smallwrite_build.cpp
+            ${LIBRARY_DIR}/src/som/slot_manager.cpp
+            ${LIBRARY_DIR}/src/som/som_runtime.c
+            ${LIBRARY_DIR}/src/som/som_stream.c
+            ${LIBRARY_DIR}/src/stream_compress.c
+            ${LIBRARY_DIR}/src/util/alloc.cpp
+            ${LIBRARY_DIR}/src/util/charreach.cpp
+            ${LIBRARY_DIR}/src/util/clique.cpp
+            ${LIBRARY_DIR}/src/util/compile_context.cpp
+            ${LIBRARY_DIR}/src/util/compile_error.cpp
+            ${LIBRARY_DIR}/src/util/cpuid_flags.c
+            ${LIBRARY_DIR}/src/util/depth.cpp
+            ${LIBRARY_DIR}/src/util/fatbit_build.cpp
+            ${LIBRARY_DIR}/src/util/multibit_build.cpp
+            ${LIBRARY_DIR}/src/util/multibit.c
+            ${LIBRARY_DIR}/src/util/report_manager.cpp
+            ${LIBRARY_DIR}/src/util/simd_utils.c
+            ${LIBRARY_DIR}/src/util/state_compress.c
+            ${LIBRARY_DIR}/src/util/target_info.cpp
+            ${LIBRARY_DIR}/src/util/ue2string.cpp
+        )
+
+        add_library (hyperscan ${SRCS})
+
+        target_compile_definitions (hyperscan PUBLIC USE_HYPERSCAN=1)
+        target_compile_options (hyperscan
+            PRIVATE -g0 -march=corei7 # library has too much debug information
+            PUBLIC -Wno-documentation
+        )
+        target_include_directories (hyperscan
+            PRIVATE
+                common
+                ${LIBRARY_DIR}/include
+            PUBLIC
+                ${LIBRARY_DIR}/src
+        )
+        if (ARCH_AMD64)
+            target_include_directories (hyperscan PRIVATE x86_64)
+        endif ()
+        target_link_libraries (hyperscan PRIVATE boost::headers_only)
+    else ()
+        find_library (LIBRARY_HYPERSCAN hs)
+        find_path (INCLUDE_HYPERSCAN NAMES hs.h HINTS /usr/include/hs) # Ubuntu puts headers in this folder
+
+        add_library (hyperscan UNKNOWN IMPORTED GLOBAL)
+        set_target_properties (hyperscan PROPERTIES IMPORTED_LOCATION ${LIBRARY_HYPERSCAN})
+        set_target_properties (hyperscan PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_HYPERSCAN})
+        set_property(TARGET hyperscan APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_HYPERSCAN=1)
+    endif ()
+
+    message (STATUS "Using hyperscan")
+else ()
+    add_library (hyperscan INTERFACE)
+    target_compile_definitions (hyperscan INTERFACE USE_HYPERSCAN=0)
+
+    message (STATUS "Not using hyperscan")
+endif ()

contrib/hyperscan-cmake/common/hs_version.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2015, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef HS_VERSION_H_C6428FAF8E3713
+#define HS_VERSION_H_C6428FAF8E3713
+
+/**
+ * A version string to identify this release of Hyperscan.
+ */
+#define HS_VERSION_STRING "5.1.1 2000-01-01"
+
+#define HS_VERSION_32BIT ((5 << 24) | (1 << 16) | (1 << 8) | 0)
+
+#endif /* HS_VERSION_H_C6428FAF8E3713 */
+

contrib/hyperscan-cmake/x86_64/config.h

@@ -0,0 +1,106 @@
+/* used by cmake */
+
+#ifndef CONFIG_H_
+#define CONFIG_H_
+
+/* "Define if the build is 32 bit" */
+/* #undef ARCH_32_BIT */
+
+/* "Define if the build is 64 bit" */
+#define ARCH_64_BIT
+
+/* "Define if building for IA32" */
+/* #undef ARCH_IA32 */
+
+/* "Define if building for EM64T" */
+#define ARCH_X86_64
+
+/* internal build, switch on dump support. */
+/* #undef DUMP_SUPPORT */
+
+/* Define if building "fat" runtime. */
+/* #undef FAT_RUNTIME */
+
+/* Define if building AVX-512 in the fat runtime. */
+/* #undef BUILD_AVX512 */
+
+/* Define to 1 if `backtrace' works. */
+#define HAVE_BACKTRACE
+
+/* C compiler has __builtin_assume_aligned */
+#define HAVE_CC_BUILTIN_ASSUME_ALIGNED
+
+/* C++ compiler has __builtin_assume_aligned */
+#define HAVE_CXX_BUILTIN_ASSUME_ALIGNED
+
+/* C++ compiler has x86intrin.h */
+#define HAVE_CXX_X86INTRIN_H
+
+/* C compiler has x86intrin.h */
+#define HAVE_C_X86INTRIN_H
+
+/* C++ compiler has intrin.h */
+/* #undef HAVE_CXX_INTRIN_H */
+
+/* C compiler has intrin.h */
+/* #undef HAVE_C_INTRIN_H */
+
+/* Define to 1 if you have the declaration of `pthread_setaffinity_np', and to
+   0 if you don't. */
+/* #undef HAVE_DECL_PTHREAD_SETAFFINITY_NP */
+
+/* #undef HAVE_PTHREAD_NP_H */
+
+/* Define to 1 if you have the `malloc_info' function. */
+/* #undef HAVE_MALLOC_INFO */
+
+/* Define to 1 if you have the `memmem' function. */
+/* #undef HAVE_MEMMEM */
+
+/* Define to 1 if you have a working `mmap' system call. */
+#define HAVE_MMAP
+
+/* Define to 1 if `posix_memalign' works. */
+#define HAVE_POSIX_MEMALIGN
+
+/* Define to 1 if you have the `setrlimit' function. */
+#define HAVE_SETRLIMIT
+
+/* Define to 1 if you have the `shmget' function. */
+/* #undef HAVE_SHMGET */
+
+/* Define to 1 if you have the `sigaction' function. */
+#define HAVE_SIGACTION
+
+/* Define to 1 if you have the `sigaltstack' function. */
+#define HAVE_SIGALTSTACK
+
+/* Define if the sqlite3_open_v2 call is available */
+/* #undef HAVE_SQLITE3_OPEN_V2 */
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#define HAVE_UNISTD_H
+
+/* Define to 1 if you have the `_aligned_malloc' function. */
+/* #undef HAVE__ALIGNED_MALLOC */
+
+/* Define if compiler has __builtin_constant_p */
+#define HAVE__BUILTIN_CONSTANT_P
+
+/* Optimize, inline critical functions */
+#define HS_OPTIMIZE
+
+#define HS_VERSION
+#define HS_MAJOR_VERSION
+#define HS_MINOR_VERSION
+#define HS_PATCH_VERSION
+
+#define BUILD_DATE
+
+/* define if this is a release build. */
+#define RELEASE_BUILD
+
+/* define if reverse_graph requires patch for boost 1.62.0 */
+/* #undef BOOST_REVGRAPH_PATCH */
+
+#endif /* CONFIG_H_ */

contrib/libdivide/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_library (libdivide INTERFACE)
+target_include_directories (libdivide SYSTEM BEFORE INTERFACE .)

contrib/libhdfs3-cmake/CMakeLists.txt

@@ -209,9 +209,8 @@ endif()
 target_link_libraries(hdfs3 PRIVATE ${LIBXML2_LIBRARY})
 
 # inherit from parent cmake
-target_include_directories(hdfs3 PRIVATE ${Boost_INCLUDE_DIRS})
 target_include_directories(hdfs3 PRIVATE ${Protobuf_INCLUDE_DIR})
-target_link_libraries(hdfs3 PRIVATE ${Protobuf_LIBRARY})
+target_link_libraries(hdfs3 PRIVATE ${Protobuf_LIBRARY} boost::headers_only)
 if(OPENSSL_INCLUDE_DIR AND OPENSSL_LIBRARIES)
     target_include_directories(hdfs3 PRIVATE ${OPENSSL_INCLUDE_DIR})
     target_link_libraries(hdfs3 PRIVATE ${OPENSSL_LIBRARIES})

contrib/libmetrohash/CMakeLists.txt

@@ -1,13 +1,10 @@
-if (HAVE_SSE42) # Not used. Pretty easy to port.
+set (SRCS
-    set (SOURCES_SSE42_ONLY src/metrohash128crc.cpp src/metrohash128crc.h)
-endif ()
-
-add_library(metrohash
-    src/metrohash.h
-    src/testvector.h
-
     src/metrohash64.cpp
     src/metrohash128.cpp
-    ${SOURCES_SSE42_ONLY})
+)
+if (HAVE_SSE42) # Not used. Pretty easy to port.
+    list (APPEND SRCS src/metrohash128crc.cpp)
+endif ()
 
-target_include_directories(metrohash PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src)
+add_library(metrohash ${SRCS})
+target_include_directories(metrohash PUBLIC src)

contrib/librdkafka-cmake/CMakeLists.txt

@@ -82,7 +82,7 @@ target_compile_options(rdkafka PRIVATE -fno-sanitize=undefined)
 target_include_directories(rdkafka SYSTEM PUBLIC include)
 target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR})         # Because weird logic with "include_next" is used.
 target_include_directories(rdkafka SYSTEM PRIVATE ${ZSTD_INCLUDE_DIR}/common)   # Because wrong path to "zstd_errors.h" is used.
-target_link_libraries(rdkafka PRIVATE ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${LIBGSASL_LIBRARY})
+target_link_libraries(rdkafka PRIVATE lz4 ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LIBGSASL_LIBRARY})
 if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY)
     target_link_libraries(rdkafka PRIVATE ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
 endif()

contrib/lz4-cmake/CMakeLists.txt

@@ -1,17 +1,28 @@
-SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/lz4/lib)
+option (USE_INTERNAL_LZ4_LIBRARY "Use internal lz4 library" ${NOT_UNBUNDLED})
 
-add_library (lz4
+if (USE_INTERNAL_LZ4_LIBRARY)
-    ${LIBRARY_DIR}/lz4.c
+    set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/lz4)
-    ${LIBRARY_DIR}/lz4hc.c
-    ${LIBRARY_DIR}/lz4frame.c
-    ${LIBRARY_DIR}/lz4frame.h
-    ${LIBRARY_DIR}/xxhash.c
-    ${LIBRARY_DIR}/xxhash.h
 
-    ${LIBRARY_DIR}/lz4.h
+    set (SRCS
-    ${LIBRARY_DIR}/lz4hc.h)
+        ${LIBRARY_DIR}/lib/lz4.c
+        ${LIBRARY_DIR}/lib/lz4hc.c
+        ${LIBRARY_DIR}/lib/lz4frame.c
+        ${LIBRARY_DIR}/lib/xxhash.c
+    )
 
-target_compile_definitions(lz4 PUBLIC LZ4_DISABLE_DEPRECATE_WARNINGS=1)
+    add_library (lz4 ${SRCS})
+
+    target_compile_definitions (lz4 PUBLIC LZ4_DISABLE_DEPRECATE_WARNINGS=1 USE_XXHASH=1)
+    if (SANITIZE STREQUAL "undefined")
         target_compile_options (lz4 PRIVATE -fno-sanitize=undefined)
+    endif ()
+    target_include_directories(lz4 PUBLIC ${LIBRARY_DIR}/lib)
+else ()
+    find_library (LIBRARY_LZ4 lz4)
+    find_path (INCLUDE_LZ4 lz4.h)
 
-target_include_directories(lz4 PUBLIC ${LIBRARY_DIR})
+    add_library (lz4 UNKNOWN IMPORTED)
+    set_property (TARGET lz4 PROPERTY IMPORTED_LOCATION ${LIBRARY_LZ4})
+    set_property (TARGET lz4 PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_LZ4})
+    set_property (TARGET lz4 APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_XXHASH=0)
+endif ()

contrib/replxx

@@ -1 +1 @@
-Subproject commit f1332626639d6492eaf170758642da14fbbda7bf
+Subproject commit 2d37daaad24be71e76514a36b0a47120be2f9086

docker/packager/binary/build.sh

@@ -18,8 +18,7 @@ ccache --zero-stats ||:
 ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||:
 rm -f CMakeCache.txt
 cmake .. -LA -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER $CMAKE_FLAGS
-ninja
+ninja clickhouse-bundle
-ccache --show-stats ||:
 mv ./programs/clickhouse* /output
 mv ./src/unit_tests_dbms /output
 find . -name '*.so' -print -exec mv '{}' /output \;
@@ -47,3 +46,4 @@ then
     rm -r /output/*
     mv "$COMBINED_OUTPUT.tgz" /output
 fi
+ccache --show-stats ||:

docker/packager/deb/Dockerfile

@@ -82,8 +82,8 @@ RUN apt-get --allow-unauthenticated update -y \
             libcctz-dev \
             libldap2-dev \
             libsasl2-dev \
-            heimdal-multidev
+            heimdal-multidev \
-
+            libhyperscan-dev
 
 
 # This symlink required by gcc to find lld compiler

docker/packager/packager

@@ -120,6 +120,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
         result.append("CCACHE_BASEDIR=/build")
         result.append("CCACHE_NOHASHDIR=true")
         result.append("CCACHE_COMPILERCHECK=content")
+        result.append("CCACHE_MAXSIZE=15G")
         # result.append("CCACHE_UMASK=777")
 
     if distcc_hosts:
@@ -141,7 +142,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
 
     if unbundled:
         # TODO: fix build with ENABLE_RDKAFKA
-        cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0')
+        cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0 -DUSE_INTERNAL_BOOST_LIBRARY=1')
 
     if split_binary:
         cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1')

docker/test/integration/compose/docker_compose_minio.yml

@@ -43,7 +43,10 @@ services:
 
 # Empty container to run proxy resolver.
   resolver:
-    image: python:3
+    build:
+      context: ../../../docker/test/integration/
+      dockerfile: resolver/Dockerfile
+      network: host
     ports:
       - "4083:8080"
     tty: true

docker/test/integration/resolver/Dockerfile

@@ -0,0 +1,4 @@
+# Helper docker container to run python bottle apps
+
+FROM python:3
+RUN python -m pip install bottle

docker/test/performance-comparison/compare.sh

@@ -104,13 +104,12 @@ function run_tests
         # allows the tests to pass even when we add new functions and tests for
         # them, that are not supported in the old revision.
         test_prefix=left/performance
-    elif [ "$PR_TO_TEST" != "" ] && [ "$PR_TO_TEST" != "0" ]
+    else
-    then
         # For PRs, use newer test files so we can test these changes.
         test_prefix=right/performance
 
-        # If some tests were changed in the PR, we may want to run only these
+        # If only the perf tests were changed in the PR, we will run only these
-        # ones. The list of changed tests in changed-test.txt is prepared in
+        # tests. The list of changed tests in changed-test.txt is prepared in
         # entrypoint.sh from git diffs, because it has the cloned repo.  Used
         # to use rsync for that but it was really ugly and not always correct
         # (e.g. when the reference SHA is really old and has some other

docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml

@@ -19,6 +19,5 @@
         <collect_interval_milliseconds>1000</collect_interval_milliseconds>
     </metric_log>
 
-    <use_uncompressed_cache>0</use_uncompressed_cache>
     <uncompressed_cache_size>1000000000</uncompressed_cache_size>
 </yandex>

docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml

@@ -5,6 +5,7 @@
             <query_profiler_cpu_time_period_ns>0</query_profiler_cpu_time_period_ns>
             <allow_introspection_functions>1</allow_introspection_functions>
             <log_queries>1</log_queries>
+            <metrics_perf_events_enabled>1</metrics_perf_events_enabled>
         </default>
     </profiles>
 </yandex>

docker/test/performance-comparison/entrypoint.sh

@@ -83,10 +83,17 @@ if [ "$REF_PR" == "" ]; then echo Reference PR is not specified ; exit 1 ; fi
 
 if [ "$PR_TO_TEST" != "0" ]
 then
-    # Prepare the list of tests changed in the PR for use by compare.sh. Compare to
+    # If the PR only changes the tests and nothing else, prepare a list of these
-    # merge base, because master might be far in the future and have unrelated test
+    # tests for use by compare.sh. Compare to merge base, because master might be
-    # changes.
+    # far in the future and have unrelated test changes.
-    git -C ch diff --name-only "$SHA_TO_TEST" "$(git -C ch merge-base "$SHA_TO_TEST" master)" -- tests/performance | tee changed-tests.txt
+    base=$(git -C ch merge-base "$SHA_TO_TEST" master)
+    git -C ch diff --name-only "$SHA_TO_TEST" "$base" | tee changed-tests.txt
+    if grep -vq '^tests/performance' changed-tests.txt
+    then
+        # Have some other changes besides the tests, so truncate the test list,
+        # meaning, run all tests.
+        : > changed-tests.txt
+    fi
 fi
 
 # Set python output encoding so that we can print queries with Russian letters.
@@ -124,5 +131,5 @@ done
 
 dmesg -T > dmesg.log
 
-7z a /output/output.7z ./*.{log,tsv,html,txt,rep,svg} {right,left}/{performance,db/preprocessed_configs,scripts} report analyze
+7z a '-x!*/tmp' /output/output.7z ./*.{log,tsv,html,txt,rep,svg,columns} {right,left}/{performance,db/preprocessed_configs,scripts} report analyze
 cp compare.log /output

docker/test/performance-comparison/perf.py

@@ -100,11 +100,20 @@ for c in connections:
 
 report_stage_end('drop1')
 
-# Apply settings
+# Apply settings.
+# If there are errors, report them and continue -- maybe a new test uses a setting
+# that is not in master, but the queries can still run. If we have multiple
+# settings and one of them throws an exception, all previous settings for this
+# connection will be reset, because the driver reconnects on error (not
+# configurable). So the end result is uncertain, but hopefully we'll be able to
+# run at least some queries.
 settings = root.findall('settings/*')
 for c in connections:
     for s in settings:
+        try:
             c.execute("set {} = '{}'".format(s.tag, s.text))
+        except:
+            print(traceback.format_exc(), file=sys.stderr)
 
 report_stage_end('settings')
 

docs/en/development/build.md

@@ -5,9 +5,13 @@ toc_title: How to Build ClickHouse on Linux
 
 # How to Build ClickHouse for Development {#how-to-build-clickhouse-for-development}
 
-The following tutorial is based on the Ubuntu Linux system.
+The following tutorial is based on the Ubuntu Linux system. With appropriate changes, it should also work on any other Linux distribution.
-With appropriate changes, it should also work on any other Linux distribution.
+
-Supported platforms: x86\_64 and AArch64. Support for Power9 is experimental.
+Supported platforms:
+
+-   x86\_64
+-   AArch64
+-   Power9 (experimental)
 
 ## Install Git, CMake, Python and Ninja {#install-git-cmake-python-and-ninja}
 
@@ -21,8 +25,18 @@ Or cmake3 instead of cmake on older systems.
 
 There are several ways to do this.
 
+### Install from Repository {#install-from-repository}
+
+On Ubuntu 19.10 or newer:
+```
+$ sudo apt-get update
+$ sudo apt-get install gcc-9 g++-9
+```
+
 ### Install from a PPA Package {#install-from-a-ppa-package}
 
+On older Ubuntu:
+
 ``` bash
 $ sudo apt-get install software-properties-common
 $ sudo apt-add-repository ppa:ubuntu-toolchain-r/test
@@ -32,7 +46,7 @@ $ sudo apt-get install gcc-9 g++-9
 
 ### Install from Sources {#install-from-sources}
 
-Look at [utils/ci/build-gcc-from-sources.sh](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh)
+See [utils/ci/build-gcc-from-sources.sh](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh)
 
 ## Use GCC 9 for Builds {#use-gcc-9-for-builds}
 
@@ -61,7 +75,6 @@ $ mkdir build
 $ cd build
 $ cmake ..
 $ ninja
-$ cd ..
 ```
 
 To create an executable, run `ninja clickhouse`.

docs/en/development/developer-instruction.md

@@ -137,7 +137,7 @@ Official Yandex builds currently use GCC because it generates machine code of sl
 
 To install GCC on Ubuntu run: `sudo apt install gcc g++`
 
-Check the version of gcc: `gcc --version`. If it is below 9, then follow the instruction here: https://clickhouse.tech/docs/en/development/build/\#install-gcc-9.
+Check the version of gcc: `gcc --version`. If it is below 9, then follow the instruction here: https://clickhouse.tech/docs/en/development/build/#install-gcc-9.
 
 Mac OS X build is supported only for Clang. Just run `brew install llvm`
 
@@ -245,7 +245,7 @@ The Code Style Guide: https://clickhouse.tech/docs/en/development/style/
 
 Writing tests: https://clickhouse.tech/docs/en/development/tests/
 
-List of tasks: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md
+List of tasks: https://github.com/ClickHouse/ClickHouse/contribute
 
 ## Test Data {#test-data}
 

docs/en/engines/table-engines/index.md

@@ -60,7 +60,7 @@ Engines in the family:
 -   [Distributed](special/distributed.md#distributed)
 -   [MaterializedView](special/materializedview.md#materializedview)
 -   [Dictionary](special/dictionary.md#dictionary)
--   [Merge](special/merge.md#merge
+-   [Merge](special/merge.md#merge)
 -   [File](special/file.md#file)
 -   [Null](special/null.md#null)
 -   [Set](special/set.md#set)

docs/en/engines/table-engines/mergetree-family/mergetree.md

@@ -41,8 +41,8 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
     INDEX index_name1 expr1 TYPE type1(...) GRANULARITY value1,
     INDEX index_name2 expr2 TYPE type2(...) GRANULARITY value2
 ) ENGINE = MergeTree()
+ORDER BY expr
 [PARTITION BY expr]
-[ORDER BY expr]
 [PRIMARY KEY expr]
 [SAMPLE BY expr]
 [TTL expr [DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'], ...]
@@ -58,23 +58,27 @@ For a description of parameters, see the [CREATE query description](../../../sql
 
 -   `ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. The `MergeTree` engine does not have parameters.
 
--   `PARTITION BY` — The [partitioning key](custom-partitioning-key.md).
+-   `ORDER BY` — The sorting key.
+
+    A tuple of column names or arbitrary expressions. Example: `ORDER BY (CounterID, EventDate)`.
+
+    ClickHouse uses the sorting key as a primary key if the primary key is not defined obviously by the `PRIMARY KEY` clause. 
+    
+    Use the `ORDER BY tuple()` syntax, if you don't need sorting. See [Selecting the Primary Key](#selecting-the-primary-key).
+
+-   `PARTITION BY` — The [partitioning key](custom-partitioning-key.md). Optional.
 
     For partitioning by month, use the `toYYYYMM(date_column)` expression, where `date_column` is a column with a date of the type [Date](../../../sql-reference/data-types/date.md). The partition names here have the `"YYYYMM"` format.
 
--   `ORDER BY` — The sorting key.
+-   `PRIMARY KEY` — The primary key if it [differs from the sorting key](#choosing-a-primary-key-that-differs-from-the-sorting-key). Optional.
-
-    A tuple of columns or arbitrary expressions. Example: `ORDER BY (CounterID, EventDate)`.
-
--   `PRIMARY KEY` — The primary key if it [differs from the sorting key](#choosing-a-primary-key-that-differs-from-the-sorting-key).
 
     By default the primary key is the same as the sorting key (which is specified by the `ORDER BY` clause). Thus in most cases it is unnecessary to specify a separate `PRIMARY KEY` clause.
 
--   `SAMPLE BY` — An expression for sampling.
+-   `SAMPLE BY` — An expression for sampling. Optional.
 
     If a sampling expression is used, the primary key must contain it. Example: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`.
 
--   `TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [between disks and volumes](#table_engine-mergetree-multiple-volumes).
+-   `TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [between disks and volumes](#table_engine-mergetree-multiple-volumes). Optional.
 
     Expression must have one `Date` or `DateTime` column as a result. Example:
     `TTL date + INTERVAL 1 DAY`
@@ -83,7 +87,7 @@ For a description of parameters, see the [CREATE query description](../../../sql
 
     For more details, see [TTL for columns and tables](#table_engine-mergetree-ttl)
 
--   `SETTINGS` — Additional parameters that control the behavior of the `MergeTree`:
+-   `SETTINGS` — Additional parameters that control the behavior of the `MergeTree` (optional):
 
     -   `index_granularity` — Maximum number of data rows between the marks of an index. Default value: 8192. See [Data Storage](#mergetree-data-storage).
     -   `index_granularity_bytes` — Maximum size of data granules in bytes. Default value: 10Mb. To restrict the granule size only by number of rows, set to 0 (not recommended). See [Data Storage](#mergetree-data-storage).
@@ -198,6 +202,10 @@ The number of columns in the primary key is not explicitly limited. Depending on
 
 A long primary key will negatively affect the insert performance and memory consumption, but extra columns in the primary key do not affect ClickHouse performance during `SELECT` queries.
 
+You can create a table without a primary key using the `ORDER BY tuple()` syntax. In this case, ClickHouse stores data in the order of inserting. If you want to save data order when inserting data by `INSERT ... SELECT` queries, set [max_insert_threads = 1](../../../operations/settings/settings.md#settings-max-insert-threads).
+    
+To select data in the initial order, use [single-threaded](../../../operations/settings/settings.md#settings-max_threads) `SELECT` queries.
+
 ### Choosing a Primary Key that Differs from the Sorting Key {#choosing-a-primary-key-that-differs-from-the-sorting-key}
 
 It is possible to specify a primary key (an expression with values that are written in the index file for each mark) that is different from the sorting key (an expression for sorting the rows in data parts). In this case the primary key expression tuple must be a prefix of the sorting key expression tuple.
@@ -332,8 +340,8 @@ The `set` index can be used with all functions. Function subsets for other index
 |------------------------------------------------------------------------------------------------------------|-------------|--------|-------------|-------------|---------------|
 | [equals (=, ==)](../../../sql-reference/functions/comparison-functions.md#function-equals)                 | ✔           | ✔      | ✔           | ✔           | ✔             |
 | [notEquals(!=, \<\>)](../../../sql-reference/functions/comparison-functions.md#function-notequals)         | ✔           | ✔      | ✔           | ✔           | ✔             |
-| [like](../../../sql-reference/functions/string-search-functions.md#function-like)                          | ✔           | ✔      | ✔           | ✗           | ✗             |
+| [like](../../../sql-reference/functions/string-search-functions.md#function-like)                          | ✔           | ✔      | ✔           | ✔           | ✔             |
-| [notLike](../../../sql-reference/functions/string-search-functions.md#function-notlike)                    | ✔           | ✔      | ✔           | ✗           | ✗             |
+| [notLike](../../../sql-reference/functions/string-search-functions.md#function-notlike)                    | ✔           | ✔      | ✗           | ✗           | ✗             |
 | [startsWith](../../../sql-reference/functions/string-functions.md#startswith)                              | ✔           | ✔      | ✔           | ✔           | ✗             |
 | [endsWith](../../../sql-reference/functions/string-functions.md#endswith)                                  | ✗           | ✗      | ✔           | ✔           | ✗             |
 | [multiSearchAny](../../../sql-reference/functions/string-search-functions.md#function-multisearchany)      | ✗           | ✗      | ✔           | ✗           | ✗             |
@@ -349,6 +357,7 @@ The `set` index can be used with all functions. Function subsets for other index
 
 Functions with a constant argument that is less than ngram size can’t be used by `ngrambf_v1` for query optimization.
 
+!!! note "Note"
     Bloom filters can have false positive matches, so the `ngrambf_v1`, `tokenbf_v1`, and `bloom_filter` indexes can’t be used for optimizing queries where the result of a function is expected to be false, for example:
 
 -   Can be optimized:
@@ -652,4 +661,3 @@ After the completion of background merges and mutations, old parts are removed o
 During this time, they are not moved to other volumes or disks. Therefore, until the parts are finally removed, they are still taken into account for evaluation of the occupied disk space.
 
 [Original article](https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/) <!--hide-->
-

docs/en/operations/backup.md

@@ -31,6 +31,7 @@ For smaller volumes of data, a simple `INSERT INTO ... SELECT ...` to remote tab
 ## Manipulations with Parts {#manipulations-with-parts}
 
 ClickHouse allows using the `ALTER TABLE ... FREEZE PARTITION ...` query to create a local copy of table partitions. This is implemented using hardlinks to the `/var/lib/clickhouse/shadow/` folder, so it usually does not consume extra disk space for old data. The created copies of files are not handled by ClickHouse server, so you can just leave them there: you will have a simple backup that doesn’t require any additional external system, but it will still be prone to hardware issues. For this reason, it’s better to remotely copy them to another location and then remove the local copies. Distributed filesystems and object stores are still a good options for this, but normal attached file servers with a large enough capacity might work as well (in this case the transfer will occur via the network filesystem or maybe [rsync](https://en.wikipedia.org/wiki/Rsync)).
+Data can be restored from backup using the `ALTER TABLE ... ATTACH PARTITION ...`
 
 For more information about queries related to partition manipulations, see the [ALTER documentation](../sql-reference/statements/alter.md#alter_manipulations-with-partitions).
 

docs/en/operations/configuration-files.md

@@ -18,7 +18,7 @@ If `replace` is specified, it replaces the entire element with the specified one
 
 If `remove` is specified, it deletes the element.
 
-The config can also define “substitutions”. If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include\_from](server-configuration-parameters/settings.md#server_configuration_parameters-include_from) element in the server config. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in `incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional="true"` attribute (for example, settings for [macros](server-configuration-parameters/settings.md)).
+The config can also define "substitutions". If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include\_from](server-configuration-parameters/settings.md#server_configuration_parameters-include_from) element in the server config. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in `incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional="true"` attribute (for example, settings for [macros](server-configuration-parameters/settings.md)).
 
 Substitutions can also be performed from ZooKeeper. To do this, specify the attribute `from_zk = "/path/to/node"`. The element value is replaced with the contents of the node at `/path/to/node` in ZooKeeper. You can also put an entire XML subtree on the ZooKeeper node and it will be fully inserted into the source element.
 

docs/en/operations/system-tables.md

@@ -18,9 +18,11 @@ System tables:
 - Available only for reading data.
 - Can't be dropped or altered, but can be detached.
 
-Most of system tables store their data in RAM. ClickHouse server creates such system tables at the start.
+Most of system tables store their data in RAM. A ClickHouse server creates such system tables at the start.
 
-The [metric_log](#system_tables-metric_log), [query_log](#system_tables-query_log), [query_thread_log](#system_tables-query_thread_log), [trace_log](#system_tables-trace_log) system tables store data in a storage filesystem. You can alter them or remove from a disk manually. If you remove one of that tables from a disk, the ClickHouse server creates the table again at the time of the next recording. A storage period for these tables is not limited, and ClickHouse server doesn't delete their data automatically. You need to organize removing of outdated logs by yourself. For example, you can use [TTL](../sql-reference/statements/alter.md#manipulations-with-table-ttl) settings for removing outdated log records. 
+Unlike other system tables, the system tables [metric_log](#system_tables-metric_log), [query_log](#system_tables-query_log), [query_thread_log](#system_tables-query_thread_log), [trace_log](#system_tables-trace_log) are served by [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one.
+
+By default, table growth is unlimited. To control a size of a table, you can use [TTL](../sql-reference/statements/alter.md#manipulations-with-table-ttl) settings for removing outdated log records. Also you can use the partitioning feature of `MergeTree`-engine tables.
 
 
 ### Sources of System Metrics {#system-tables-sources-of-system-metrics}
@@ -636,9 +638,9 @@ You can change settings of queries logging in the [query_log](server-configurati
 
 You can disable queries logging by setting [log_queries = 0](settings/settings.md#settings-log-queries). We don't recommend to turn off logging because information in this table is important for solving issues.
 
-The flushing period of logs is set in `flush_interval_milliseconds` parameter of the [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server settings section. To force flushing logs, use the [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs) query.
+The flushing period of data is set in `flush_interval_milliseconds` parameter of the [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server settings section. To force flushing, use the [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs) query.
 
-ClickHouse doesn't delete logs from the table automatically. See [Introduction](#system-tables-introduction) for more details.
+ClickHouse doesn't delete data from the table automatically. See [Introduction](#system-tables-introduction) for more details.
 
 The `system.query_log` table registers two kinds of queries:
 
@@ -766,68 +768,117 @@ Settings.Values:      ['0','random','1','10000000000']
 
 ## system.query_thread_log {#system_tables-query_thread_log}
 
-The table contains information about each query execution thread.
+Contains information about threads which execute queries, for example, thread name, thread start time, duration of query processing.
 
-ClickHouse creates this table only if the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in.
+To start logging:
 
-To enable query logging, set the [log\_query\_threads](settings/settings.md#settings-log-query-threads) parameter to 1. For details, see the [Settings](settings/settings.md) section.
+1. Configure parameters in the [query_thread_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) section.
+2. Set [log_query_threads](settings/settings.md#settings-log-query-threads) to 1.
+
+The flushing period of data is set in `flush_interval_milliseconds` parameter of the [query_thread_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server settings section. To force flushing, use the [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs) query.
+
+ClickHouse doesn't delete data from the table automatically. See [Introduction](#system-tables-introduction) for more details.
 
 Columns:
 
--   `event_date` (Date) — the date when the thread has finished execution of the query.
+-   `event_date` ([Date](../sql-reference/data-types/date.md)) — The date when the thread has finished execution of the query.
--   `event_time` (DateTime) — the date and time when the thread has finished execution of the query.
+-   `event_time` ([DateTime](../sql-reference/data-types/datetime.md)) — The date and time when the thread has finished execution of the query.
--   `query_start_time` (DateTime) — Start time of query execution.
+-   `query_start_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Start time of query execution.
--   `query_duration_ms` (UInt64) — Duration of query execution.
+-   `query_duration_ms` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Duration of query execution.
--   `read_rows` (UInt64) — Number of read rows.
+-   `read_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of read rows.
--   `read_bytes` (UInt64) — Number of read bytes.
+-   `read_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of read bytes.
--   `written_rows` (UInt64) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0.
+-   `written_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0.
--   `written_bytes` (UInt64) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0.
+-   `written_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0.
--   `memory_usage` (Int64) — The difference between the amount of allocated and freed memory in context of this thread.
+-   `memory_usage` ([Int64](../sql-reference/data-types/int-uint.md)) — The difference between the amount of allocated and freed memory in context of this thread.
--   `peak_memory_usage` (Int64) — The maximum difference between the amount of allocated and freed memory in context of this thread.
+-   `peak_memory_usage` ([Int64](../sql-reference/data-types/int-uint.md)) — The maximum difference between the amount of allocated and freed memory in context of this thread.
--   `thread_name` (String) — Name of the thread.
+-   `thread_name` ([String](../sql-reference/data-types/string.md)) — Name of the thread.
--   `thread_number` (UInt32) — Internal thread ID.
+-   `thread_number` ([UInt32](../sql-reference/data-types/int-uint.md)) — Internal thread ID.
--   `os_thread_id` (Int32) — OS thread ID.
+-   `thread_id` ([Int32](../sql-reference/data-types/int-uint.md)) — thread ID.
--   `master_thread_id` (UInt64) — OS initial ID of initial thread.
+-   `master_thread_id` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — OS initial ID of initial thread.
--   `query` (String) — Query string.
+-   `query` ([String](../sql-reference/data-types/string.md)) — Query string.
--   `is_initial_query` (UInt8) — Query type. Possible values:
+-   `is_initial_query` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Query type. Possible values:
     -   1 — Query was initiated by the client.
     -   0 — Query was initiated by another query for distributed query execution.
--   `user` (String) — Name of the user who initiated the current query.
+-   `user` ([String](../sql-reference/data-types/string.md)) — Name of the user who initiated the current query.
--   `query_id` (String) — ID of the query.
+-   `query_id` ([String](../sql-reference/data-types/string.md)) — ID of the query.
--   `address` (IPv6) — IP address that was used to make the query.
+-   `address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the query.
--   `port` (UInt16) — The client port that was used to make the query.
+-   `port` ([UInt16](../sql-reference/data-types/int-uint.md#uint-ranges)) — The client port that was used to make the query.
--   `initial_user` (String) — Name of the user who ran the initial query (for distributed query execution).
+-   `initial_user` ([String](../sql-reference/data-types/string.md)) — Name of the user who ran the initial query (for distributed query execution).
--   `initial_query_id` (String) — ID of the initial query (for distributed query execution).
+-   `initial_query_id` ([String](../sql-reference/data-types/string.md)) — ID of the initial query (for distributed query execution).
--   `initial_address` (IPv6) — IP address that the parent query was launched from.
+-   `initial_address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP address that the parent query was launched from.
--   `initial_port` (UInt16) — The client port that was used to make the parent query.
+-   `initial_port` ([UInt16](../sql-reference/data-types/int-uint.md#uint-ranges)) — The client port that was used to make the parent query.
--   `interface` (UInt8) — Interface that the query was initiated from. Possible values:
+-   `interface` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Interface that the query was initiated from. Possible values:
     -   1 — TCP.
     -   2 — HTTP.
--   `os_user` (String) — OS’s username who runs [clickhouse-client](../interfaces/cli.md).
+-   `os_user` ([String](../sql-reference/data-types/string.md)) — OS’s username who runs [clickhouse-client](../interfaces/cli.md).
--   `client_hostname` (String) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) or another TCP client is run.
+-   `client_hostname` ([String](../sql-reference/data-types/string.md)) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) or another TCP client is run.
--   `client_name` (String) — The [clickhouse-client](../interfaces/cli.md) or another TCP client name.
+-   `client_name` ([String](../sql-reference/data-types/string.md)) — The [clickhouse-client](../interfaces/cli.md) or another TCP client name.
--   `client_revision` (UInt32) — Revision of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
+-   `client_revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — Revision of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
--   `client_version_major` (UInt32) — Major version of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
+-   `client_version_major` ([UInt32](../sql-reference/data-types/int-uint.md)) — Major version of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
--   `client_version_minor` (UInt32) — Minor version of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
+-   `client_version_minor` ([UInt32](../sql-reference/data-types/int-uint.md)) — Minor version of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
--   `client_version_patch` (UInt32) — Patch component of the [clickhouse-client](../interfaces/cli.md) or another TCP client version.
+-   `client_version_patch` ([UInt32](../sql-reference/data-types/int-uint.md)) — Patch component of the [clickhouse-client](../interfaces/cli.md) or another TCP client version.
--   `http_method` (UInt8) — HTTP method that initiated the query. Possible values:
+-   `http_method` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — HTTP method that initiated the query. Possible values:
     -   0 — The query was launched from the TCP interface.
     -   1 — `GET` method was used.
     -   2 — `POST` method was used.
--   `http_user_agent` (String) — The `UserAgent` header passed in the HTTP request.
+-   `http_user_agent` ([String](../sql-reference/data-types/string.md)) — The `UserAgent` header passed in the HTTP request.
--   `quota_key` (String) — The “quota key” specified in the [quotas](quotas.md) setting (see `keyed`).
+-   `quota_key` ([String](../sql-reference/data-types/string.md)) — The “quota key” specified in the [quotas](quotas.md) setting (see `keyed`).
--   `revision` (UInt32) — ClickHouse revision.
+-   `revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ClickHouse revision.
--   `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics for this thread. The description of them could be found in the table [system.events](#system_tables-events)
+-   `ProfileEvents.Names` ([Array(String)](../sql-reference/data-types/array.md)) — Counters that measure different metrics for this thread. The description of them could be found in the table [system.events](#system_tables-events).
--   `ProfileEvents.Values` (Array(UInt64)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` column.
+-   `ProfileEvents.Values` ([Array(UInt64)](../sql-reference/data-types/array.md)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` column.
 
-By default, logs are added to the table at intervals of 7.5 seconds. You can set this interval in the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server setting (see the `flush_interval_milliseconds` parameter). To flush the logs forcibly from the memory buffer into the table, use the `SYSTEM FLUSH LOGS` query.
+**Example**
 
-When the table is deleted manually, it will be automatically created on the fly. Note that all the previous logs will be deleted.
+``` sql
+ SELECT * FROM system.query_thread_log LIMIT 1 FORMAT Vertical
+```
 
-!!! note "Note"
+``` text
-    The storage period for logs is unlimited. Logs aren’t automatically deleted from the table. You need to organize the removal of outdated logs yourself.
+Row 1:
+──────
+event_date:           2020-05-13
+event_time:           2020-05-13 14:02:28
+query_start_time:     2020-05-13 14:02:28
+query_duration_ms:    0
+read_rows:            1
+read_bytes:           1
+written_rows:         0
+written_bytes:        0
+memory_usage:         0
+peak_memory_usage:    0
+thread_name:          QueryPipelineEx
+thread_id:            28952
+master_thread_id:     28924
+query:                SELECT 1
+is_initial_query:     1
+user:                 default
+query_id:             5e834082-6f6d-4e34-b47b-cd1934f4002a
+address:              ::ffff:127.0.0.1
+port:                 57720
+initial_user:         default
+initial_query_id:     5e834082-6f6d-4e34-b47b-cd1934f4002a
+initial_address:      ::ffff:127.0.0.1
+initial_port:         57720
+interface:            1
+os_user:              bayonet
+client_hostname:      clickhouse.ru-central1.internal
+client_name:          ClickHouse client
+client_revision:      54434
+client_version_major: 20
+client_version_minor: 4
+client_version_patch: 1
+http_method:          0
+http_user_agent:
+quota_key:
+revision:             54434
+ProfileEvents.Names:  ['ContextLock','RealTimeMicroseconds','UserTimeMicroseconds','OSCPUWaitMicroseconds','OSCPUVirtualTimeMicroseconds']
+ProfileEvents.Values: [1,97,81,5,81]
+...
+```
 
-You can specify an arbitrary partitioning key for the `system.query_thread_log` table in the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server setting (see the `partition_by` parameter).
+**See Also**
+
+- [system.query_log](#system_tables-query_log) — Description of the `query_log` system table which contains common information about queries execution.
 
 ## system.trace\_log {#system_tables-trace_log}
 

docs/en/sql-reference/functions/comparison-functions.md

@@ -22,7 +22,7 @@ Strings are compared by bytes. A shorter string is smaller than all strings that
 
 ## equals, a = b and a == b operator {#function-equals}
 
-## notEquals, a ! operator= b and a \<\> b {#function-notequals}
+## notEquals, a != b and a \<\> b operator {#function-notequals}
 
 ## less, \< operator {#function-less}
 

docs/en/sql-reference/statements/system.md

@@ -5,10 +5,13 @@ toc_title: SYSTEM
 
 # SYSTEM Queries {#query-language-system}
 
+-   [RELOAD EMBEDDED DICTIONARIES](#query_language-system-reload-emdedded-dictionaries) 
 -   [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries)
 -   [RELOAD DICTIONARY](#query_language-system-reload-dictionary)
 -   [DROP DNS CACHE](#query_language-system-drop-dns-cache)
 -   [DROP MARK CACHE](#query_language-system-drop-mark-cache)
+-   [DROP UNCOMPRESSED CACHE](#query_language-system-drop-uncompressed-cache) 
+-   [DROP COMPILED EXPRESSION CACHE](#query_language-system-drop-compiled-expression-cache)
 -   [FLUSH LOGS](#query_language-system-flush_logs)
 -   [RELOAD CONFIG](#query_language-system-reload-config)
 -   [SHUTDOWN](#query_language-system-shutdown)
@@ -18,6 +21,24 @@ toc_title: SYSTEM
 -   [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends)
 -   [STOP MERGES](#query_language-system-stop-merges)
 -   [START MERGES](#query_language-system-start-merges)
+-   [STOP TTL MERGES](#query_language-stop-ttl-merges) 
+-   [START TTL MERGES](#query_language-start-ttl-merges) 
+-   [STOP MOVES](#query_language-stop-moves) 
+-   [START MOVES](#query_language-start-moves) 
+-   [STOP FETCHES](#query_language-system-stop-fetches)
+-   [START FETCHES](#query_language-system-start-fetches)
+-   [STOP REPLICATED SENDS](#query_language-system-start-replicated-sends)
+-   [START REPLICATED SENDS](#query_language-system-start-replicated-sends)
+-   [STOP REPLICATION QUEUES](#query_language-system-stop-replication-queues)
+-   [START REPLICATION QUEUES](#query_language-system-start-replication-queues)
+-   [SYNC REPLICA](#query_language-system-sync-replica)
+-   [RESTART REPLICA](#query_language-system-restart-replica)
+-   [RESTART REPLICAS](#query_language-system-restart-replicas) 
+
+## RELOAD EMBEDDED DICTIONARIES] {#query_language-system-reload-emdedded-dictionaries} 
+Reload all [Internal dictionaries](../dictionaries/internal-dicts.md).
+By default, internal dictionaries are disabled. 
+Always returns `Ok.` regardless of the result of the internal dictionary update.
    
 ## RELOAD DICTIONARIES {#query_language-system-reload-dictionaries}
 
@@ -45,6 +66,16 @@ For more convenient (automatic) cache management, see disable\_internal\_dns\_ca
 
 Resets the mark cache. Used in development of ClickHouse and performance tests.
 
+## DROP UNCOMPRESSED CACHE {#query_language-system-drop-uncompressed-cache}
+
+Reset the uncompressed data cache. Used in development of ClickHouse and performance tests.
+For manage uncompressed data cache parameters use following server level settings [uncompressed_cache_size](../../operations/server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) and query/user/profile level settings [use_uncompressed_cache](../../operations/settings/settings.md#setting-use_uncompressed_cache)
+
+
+## DROP COMPILED EXPRESSION CACHE {#query_language-system-drop-compiled-expression-cache}
+Reset the compiled expression cache. Used in development of ClickHouse and performance tests.
+Complied expression cache used when query/user/profile enable option [compile](../../operations/settings/settings.md#compile)
+
 ## FLUSH LOGS {#query_language-system-flush_logs}
 
 Flushes buffers of log messages to system tables (e.g. system.query\_log). Allows you to not wait 7.5 seconds when debugging.
@@ -89,6 +120,10 @@ Enables background data distribution when inserting data into distributed tables
 SYSTEM START DISTRIBUTED SENDS [db.]<distributed_table_name>
 ```
 
+## Managing MergeTree Tables {#query-language-system-mergetree}
+
+ClickHouse can manage background processes in [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) tables.
+
 ### STOP MERGES {#query_language-system-stop-merges}
 
 Provides possibility to stop background merges for tables in the MergeTree family:
@@ -108,4 +143,110 @@ Provides possibility to start background merges for tables in the MergeTree fami
 SYSTEM START MERGES [[db.]merge_tree_family_table_name]
 ```
 
+### STOP TTL MERGES {#query_language-stop-ttl-merges}
+
+Provides possibility to stop background delete old data according to [TTL expression](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl) for tables in the MergeTree family:
+Return `Ok.` even table doesn't exists or table have not MergeTree engine. Return error when database doesn't exists:
+
+``` sql
+SYSTEM STOP TTL MERGES [[db.]merge_tree_family_table_name]
+```
+
+### START TTL MERGES {#query_language-start-ttl-merges} 
+
+Provides possibility to start background delete old data according to [TTL expression](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl) for tables in the MergeTree family:
+Return `Ok.` even table doesn't exists. Return error when database doesn't exists:
+
+``` sql
+SYSTEM START TTL MERGES [[db.]merge_tree_family_table_name]
+```
+
+### STOP MOVES {#query_language-stop-moves} 
+
+Provides possibility to stop background move data according to [TTL table expression with TO VOLUME or TO DISK clause](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) for tables in the MergeTree family:
+Return `Ok.` even table doesn't exists. Return error when database doesn't exists:
+
+``` sql
+SYSTEM STOP MOVES [[db.]merge_tree_family_table_name]
+```
+
+### START MOVES {#query_language-start-moves} 
+
+Provides possibility to start background move data according to [TTL table expression with TO VOLUME and TO DISK clause](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) for tables in the MergeTree family:
+Return `Ok.` even table doesn't exists. Return error when database doesn't exists:
+
+``` sql
+SYSTEM STOP MOVES [[db.]merge_tree_family_table_name]
+```
+
+## Managing ReplicatedMergeTree Tables {#query-language-system-replicated}
+
+ClickHouse can manage background replication related processes in [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md) tables.
+
+### STOP FETCHES {#query_language-system-stop-fetches}
+Provides possibility to stop background fetches for inserted parts for tables in the `ReplicatedMergeTree` family:
+Always returns `Ok.` regardless of the table engine and even table or database doesn't exists.
+
+``` sql
+SYSTEM STOP FETCHES [[db.]replicated_merge_tree_family_table_name]
+```
+
+### START FETCHES {#query_language-system-start-fetches}
+Provides possibility to start background fetches for inserted parts for tables in the `ReplicatedMergeTree` family:
+Always returns `Ok.` regardless of the table engine and even table or database doesn't exists.
+
+``` sql
+SYSTEM START FETCHES [[db.]replicated_merge_tree_family_table_name]
+```
+
+### STOP REPLICATED SENDS {#query_language-system-start-replicated-sends}
+Provides possibility to stop background sends to other replicas in cluster for new inserted parts for tables in the `ReplicatedMergeTree` family:
+
+``` sql
+SYSTEM STOP REPLICATED SENDS [[db.]replicated_merge_tree_family_table_name]
+```
+
+### START REPLICATED SENDS {#query_language-system-start-replicated-sends}
+Provides possibility to start background sends to other replicas in cluster for new inserted parts for tables in the `ReplicatedMergeTree` family:
+
+``` sql
+SYSTEM START REPLICATED SENDS [[db.]replicated_merge_tree_family_table_name]
+```
+
+### STOP REPLICATION QUEUES {#query_language-system-stop-replication-queues}
+Provides possibility to stop background fetch tasks from replication queues which stored in Zookeeper for tables in the `ReplicatedMergeTree` family. Possible background tasks types - merges, fetches, mutation, DDL statements with ON CLUSTER clause: 
+
+``` sql
+SYSTEM STOP REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name]
+```
+
+### START REPLICATION QUEUES {#query_language-system-start-replication-queues}
+Provides possibility to start background fetch tasks from replication queues which stored in Zookeeper for tables in the `ReplicatedMergeTree` family. Possible background tasks types - merges, fetches, mutation, DDL statements with ON CLUSTER clause: 
+
+``` sql
+SYSTEM START REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name]
+```
+
+### SYNC REPLICA {#query_language-system-sync-replica}
+Wait until a `ReplicatedMergeTree` table will be synced with other replicas in a cluster. Will run until `receive_timeout` if fetches currently disabled for the table.
+
+``` sql
+SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name
+```
+
+### RESTART REPLICA {#query_language-system-restart-replica}
+Provides possibility to reinitialize Zookeeper sessions state for `ReplicatedMergeTree` table, will compare current state with Zookeeper as source of true and add tasks to Zookeeper queue if needed  
+Initialization replication quene based on ZooKeeper date happens in the same way as `ATTACH TABLE` statement. For a short time the table will be unavailable for any operations.
+
+``` sql
+SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name
+```
+
+### RESTART REPLICAS {#query_language-system-restart-replicas}
+Provides possibility to reinitialize Zookeeper sessions state for all `ReplicatedMergeTree` tables, will compare current state with Zookeeper as source of true and add tasks to Zookeeper queue if needed  
+
+``` sql
+SYSTEM RESTART QUEUES [db.]replicated_merge_tree_family_table_name
+```
+
 [Original article](https://clickhouse.tech/docs/en/query_language/system/) <!--hide-->

docs/es/development/developer-instruction.md

@@ -141,7 +141,7 @@ Las compilaciones oficiales de Yandex actualmente usan GCC porque genera código
 
 Para instalar GCC en Ubuntu, ejecute: `sudo apt install gcc g++`
 
-Compruebe la versión de gcc: `gcc --version`. Si está por debajo de 9, siga las instrucciones aquí: https://clickhouse .tech/docs/en/development/build/\#install-gcc-9.
+Compruebe la versión de gcc: `gcc --version`. Si está por debajo de 9, siga las instrucciones aquí: https://clickhouse.tech/docs/es/development/build/#install-gcc-9.
 
 La compilación de Mac OS X solo es compatible con Clang. Sólo tiene que ejecutar `brew install llvm`
 
@@ -249,7 +249,7 @@ La Guía de estilo de código: https://clickhouse.tech/docs/en/development/style
 
 Pruebas de escritura: https://clickhouse.tech/docs/en/development/tests/
 
-Lista de tareas: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md
+Lista de tareas: https://github.com/ClickHouse/ClickHouse/contribute
 
 # Datos de prueba {#test-data}
 

docs/fa/development/developer-instruction.md

@@ -143,7 +143,7 @@ toc_title: "\u062F\u0633\u062A\u0648\u0631\u0627\u0644\u0639\u0645\u0644 \u062A\
 
 برای نصب شورای همکاری خلیج فارس در اوبونتو اجرای: `sudo apt install gcc g++`
 
-بررسی نسخه شورای همکاری خلیج فارس: `gcc --version`. اگر زیر است 9, سپس دستورالعمل اینجا را دنبال کنید: https://clickhouse.فناوری / اسناد / ارتباطات / توسعه/ساختن / \#نصب شورای همکاری خلیج فارس-9.
+بررسی نسخه شورای همکاری خلیج فارس: `gcc --version`. اگر زیر است 9, سپس دستورالعمل اینجا را دنبال کنید: https://clickhouse.tech/docs/fa/development/build/#install-gcc-9.
 
 سیستم عامل مک ایکس ساخت فقط برای صدای جرنگ جرنگ پشتیبانی می شود. فقط فرار کن `brew install llvm`
 
@@ -251,7 +251,7 @@ KDevelop و QTCreator دیگر از جایگزین های بسیار خوبی ا
 
 تست نوشتن: https://clickhouse.فناوری / اسناد/توسعه/تست/
 
-فهرست تکلیفها: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md
+فهرست تکلیفها: https://github.com/ClickHouse/ClickHouse/contribute
 
 # داده های تست {#test-data}
 

docs/fr/development/developer-instruction.md

@@ -141,7 +141,7 @@ Les builds officiels de Yandex utilisent actuellement GCC car ils génèrent du
 
 Pour installer GCC sur Ubuntu Exécutez: `sudo apt install gcc g++`
 
-Vérifiez la version de gcc: `gcc --version`. Si elle est inférieure à 9, suivez les instructions ici: https://clickhouse.tech/docs/fr/développement/construction/\#install-gcc-9.
+Vérifiez la version de gcc: `gcc --version`. Si elle est inférieure à 9, suivez les instructions ici: https://clickhouse.tech/docs/fr/development/build/#install-gcc-9.
 
 Mac OS X build est pris en charge uniquement pour Clang. Il suffit d'exécuter `brew install llvm`
 
@@ -249,7 +249,7 @@ Le code Style Guide: https://clickhouse.tech/docs/fr/développement/style/
 
 Rédaction de tests: https://clickhouse.tech/docs/fr/développement/tests/
 
-Liste des tâches: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md
+Liste des tâches: https://github.com/ClickHouse/ClickHouse/contribute
 
 # Des Données De Test {#test-data}
 

docs/ja/development/developer-instruction.md

@@ -141,7 +141,7 @@ ClickHouseのビルドには、バージョン9以降のGCCとClangバージョ
 
 UBUNTUにGCCをインストールするには: `sudo apt install gcc g++`
 
-Gccのバージョンを確認する: `gcc --version`. の場合は下記9その指示に従う。https://clickhouse.tech/docs/en/development/build/\#install-gcc-9.
+Gccのバージョンを確認する: `gcc --version`. の場合は下記9その指示に従う。https://clickhouse.tech/docs/ja/development/build/#install-gcc-9.
 
 Mac OS XのビルドはClangでのみサポートされています。 ちょうど実行 `brew install llvm`
 
@@ -249,7 +249,7 @@ KDevelopとQTCreatorは、ClickHouseを開発するためのIDEの他の優れ
 
 筆記試験：https://clickhouse.tech/docs/en/development/tests/
 
-タスクのリスト：https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md
+タスクのリスト：https://github.com/ClickHouse/ClickHouse/contribute
 
 # テストデータ {#test-data}
 

docs/ru/development/developer-instruction.md

@@ -135,7 +135,7 @@ ClickHouse использует для сборки некоторое коли
 
 Для установки GCC под Ubuntu, выполните: `sudo apt install gcc g++`.
 
-Проверьте версию gcc: `gcc --version`. Если версия меньше 9, то следуйте инструкции: https://clickhouse.tech/docs/en/development/build/\#install-gcc-9
+Проверьте версию gcc: `gcc --version`. Если версия меньше 9, то следуйте инструкции: https://clickhouse.tech/docs/ru/development/build/#install-gcc-9.
 
 Сборка под Mac OS X поддерживается только для компилятора Clang. Чтобы установить его выполните `brew install llvm`
 
@@ -244,7 +244,7 @@ Mac OS X:
 
 Разработка тестов: https://clickhouse.tech/docs/ru/development/tests/
 
-Список задач: https://github.com/ClickHouse/ClickHouse/blob/master/tests/instructions/easy\_tasks\_sorted\_ru.md
+Список задач: https://github.com/ClickHouse/ClickHouse/contribute
 
 # Тестовые данные {#testovye-dannye}
 

docs/ru/operations/system-tables.md

@@ -593,15 +593,9 @@ CurrentMetric_ReplicatedChecks:                             0
 
 Можно отключить логгирование настройкой [log_queries = 0](settings/settings.md#settings-log-queries). По-возможности, не отключайте логгирование, поскольку информация из таблицы важна при решении проблем.
 
-Период сброса логов в таблицу задаётся параметром `flush_interval_milliseconds` в конфигурационной секции [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs).
+Период сброса данных в таблицу задаётся параметром `flush_interval_milliseconds` в конфигурационной секции [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs).
 
-ClickHouse не удаляет логи из таблица автоматически. Смотрите [Введение](#system-tables-introduction).
+ClickHouse не удаляет данные из таблица автоматически. Смотрите [Введение](#system-tables-introduction).
-
-Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) (параметр `partition_by`).
-
-
-
-Если таблицу удалить вручную, она создается заново автоматически «на лету». При этом все логи на момент удаления таблицы будут убраны.
 
 Таблица `system.query_log` содержит информацию о двух видах запросов:
 
@@ -729,71 +723,116 @@ Settings.Values:      ['0','random','1','10000000000']
 
 ## system.query_thread_log {#system_tables-query_thread_log}
 
-Содержит информацию о каждом потоке выполняемых запросов.
+Содержит информацию о потоках, которые выполняют запросы, например, имя потока, время его запуска, продолжительность обработки запроса.
 
-ClickHouse создаёт таблицу только в том случае, когда установлен конфигурационный параметр сервера [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log). Параметр задаёт правила ведения лога, такие как интервал логирования или имя таблицы, в которую будут логгироваться запросы.
+Чтобы начать логирование:
 
-Чтобы включить логирование, задайте значение параметра [log\_query\_threads](settings/settings.md#settings-log-query-threads) равным 1. Подробности смотрите в разделе [Настройки](settings/settings.md#settings).
+1. Настройте параметры [query_thread_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) в конфигурации сервера.
+2. Установите значение [log_query_threads](settings/settings.md#settings-log-query-threads) равным 1.
+
+Интервал сброса данных в таблицу задаётся параметром `flush_interval_milliseconds` в разделе настроек сервера [query_thread_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log). Чтобы принудительно записать логи из буфера памяти в таблицу, используйте запрос [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs).
+
+ClickHouse не удаляет данные из таблицы автоматически. Подробности в разделе [Введение](#system-tables-introduction).
 
 Столбцы:
 
--   `event_date` (Date) — дата завершения выполнения запроса потоком.
+-   `event_date` ([Date](../sql-reference/data-types/date.md)) — дата завершения выполнения запроса потоком.
--   `event_time` (DateTime) — дата и время завершения выполнения запроса потоком.
+-   `event_time` ([DateTime](../sql-reference/data-types/datetime.md)) — дата и время завершения выполнения запроса потоком.
--   `query_start_time` (DateTime) — время начала обработки запроса.
+-   `query_start_time` ([DateTime](../sql-reference/data-types/datetime.md)) — время начала обработки запроса.
--   `query_duration_ms` (UInt64) — длительность обработки запроса в миллисекундах.
+-   `query_duration_ms` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — длительность обработки запроса в миллисекундах.
--   `read_rows` (UInt64) — количество прочитанных строк.
+-   `read_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — количество прочитанных строк.
--   `read_bytes` (UInt64) — количество прочитанных байтов.
+-   `read_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — количество прочитанных байтов.
--   `written_rows` (UInt64) — количество записанных строк для запросов `INSERT`. Для других запросов, значение столбца 0.
+-   `written_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — количество записанных строк для запросов `INSERT`. Для других запросов, значение столбца 0.
--   `written_bytes` (UInt64) — объём записанных данных в байтах для запросов `INSERT`. Для других запросов, значение столбца 0.
+-   `written_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — объём записанных данных в байтах для запросов `INSERT`. Для других запросов, значение столбца 0.
--   `memory_usage` (Int64) — разница между выделенной и освобождённой памятью в контексте потока.
+-   `memory_usage` ([Int64](../sql-reference/data-types/int-uint.md)) — разница между выделенной и освобождённой памятью в контексте потока.
--   `peak_memory_usage` (Int64) — максимальная разница между выделенной и освобождённой памятью в контексте потока.
+-   `peak_memory_usage` ([Int64](../sql-reference/data-types/int-uint.md)) — максимальная разница между выделенной и освобождённой памятью в контексте потока.
--   `thread_name` (String) — Имя потока.
+-   `thread_name` ([String](../sql-reference/data-types/string.md)) — Имя потока.
--   `thread_id` (UInt64) — tid (ID потока операционной системы).
+-   `thread_id` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — tid (ID потока операционной системы).
--   `master_thread_id` (UInt64) — tid (ID потока операционной системы) главного потока.
+-   `master_thread_id` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — tid (ID потока операционной системы) главного потока.
--   `query` (String) — текст запроса.
+-   `query` ([String](../sql-reference/data-types/string.md)) — текст запроса.
--   `is_initial_query` (UInt8) — вид запроса. Возможные значения:
+-   `is_initial_query` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — вид запроса. Возможные значения:
     -   1 — запрос был инициирован клиентом.
     -   0 — запрос был инициирован другим запросом при распределенном запросе.
--   `user` (String) — пользователь, запустивший текущий запрос.
+-   `user` ([String](../sql-reference/data-types/string.md)) — пользователь, запустивший текущий запрос.
--   `query_id` (String) — ID запроса.
+-   `query_id` ([String](../sql-reference/data-types/string.md)) — ID запроса.
--   `address` (IPv6) — IP адрес, с которого пришел запрос.
+-   `address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел запрос.
--   `port` (UInt16) — порт, с которого пришел запрос.
+-   `port` ([UInt16](../sql-reference/data-types/int-uint.md#uint-ranges)) — порт, с которого пришел запрос.
--   `initial_user` (String) — пользователь, запустивший первоначальный запрос (для распределенных запросов).
+-   `initial_user` ([String](../sql-reference/data-types/string.md)) — пользователь, запустивший первоначальный запрос (для распределенных запросов).
--   `initial_query_id` (String) — ID родительского запроса.
+-   `initial_query_id` ([String](../sql-reference/data-types/string.md)) — ID родительского запроса.
--   `initial_address` (IPv6) — IP адрес, с которого пришел родительский запрос.
+-   `initial_address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел родительский запрос.
--   `initial_port` (UInt16) — порт, пришел родительский запрос.
+-   `initial_port` ([UInt16](../sql-reference/data-types/int-uint.md#uint-ranges)) — порт, пришел родительский запрос.
--   `interface` (UInt8) — интерфейс, с которого ушёл запрос. Возможные значения:
+-   `interface` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — интерфейс, с которого ушёл запрос. Возможные значения:
     -   1 — TCP.
     -   2 — HTTP.
--   `os_user` (String) — имя пользователя в OS, который запустил [clickhouse-client](../interfaces/cli.md).
+-   `os_user` ([String](../sql-reference/data-types/string.md)) — имя пользователя в OS, который запустил [clickhouse-client](../interfaces/cli.md).
--   `client_hostname` (String) — hostname клиентской машины, с которой присоединился [clickhouse-client](../interfaces/cli.md) или другой TCP клиент.
+-   `client_hostname` ([String](../sql-reference/data-types/string.md)) — hostname клиентской машины, с которой присоединился [clickhouse-client](../interfaces/cli.md) или другой TCP клиент.
--   `client_name` (String) — [clickhouse-client](../interfaces/cli.md) или другой TCP клиент.
+-   `client_name` ([String](../sql-reference/data-types/string.md)) — [clickhouse-client](../interfaces/cli.md) или другой TCP клиент.
--   `client_revision` (UInt32) — ревизия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
+-   `client_revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ревизия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
--   `client_version_major` (UInt32) — старшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
+-   `client_version_major` ([UInt32](../sql-reference/data-types/int-uint.md)) — старшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
--   `client_version_minor` (UInt32) — младшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
+-   `client_version_minor` ([UInt32](../sql-reference/data-types/int-uint.md)) — младшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
--   `client_version_patch` (UInt32) — патч [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
+-   `client_version_patch` ([UInt32](../sql-reference/data-types/int-uint.md)) — патч [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
--   `http_method` (UInt8) — HTTP метод, инициировавший запрос. Возможные значения:
+-   `http_method` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — HTTP метод, инициировавший запрос. Возможные значения:
     -   0 — запрос запущен с интерфейса TCP.
     -   1 — `GET`.
     -   2 — `POST`.
--   `http_user_agent` (String) — HTTP заголовок `UserAgent`.
+-   `http_user_agent` ([String](../sql-reference/data-types/string.md)) — HTTP заголовок `UserAgent`.
--   `quota_key` (String) — «ключ квоты» из настроек [квот](quotas.md) (см. `keyed`).
+-   `quota_key` ([String](../sql-reference/data-types/string.md)) — «ключ квоты» из настроек [квот](quotas.md) (см. `keyed`).
--   `revision` (UInt32) — ревизия ClickHouse.
+-   `revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ревизия ClickHouse.
--   `ProfileEvents.Names` (Array(String)) — Счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events)(\#system\_tables-events
+-   `ProfileEvents.Names` ([Array(String)](../sql-reference/data-types/array.md)) — Счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events).
--   `ProfileEvents.Values` (Array(UInt64)) — метрики для данного потока, перечисленные в столбце `ProfileEvents.Names`.
+-   `ProfileEvents.Values` ([Array(UInt64)](../sql-reference/data-types/array.md)) — метрики для данного потока, перечисленные в столбце `ProfileEvents.Names`.
 
-По умолчанию, строки добавляются в таблицу логирования с интервалом в 7,5 секунд. Можно задать интервал в конфигурационном параметре сервера [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) (смотрите параметр `flush_interval_milliseconds`). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос `SYSTEM FLUSH LOGS`.
+**Пример**
 
-Если таблицу удалить вручную, она пересоздастся автоматически «на лету». При этом все логи на момент удаления таблицы будут удалены.
+``` sql
+ SELECT * FROM system.query_thread_log LIMIT 1 FORMAT Vertical
+```
 
-!!! note "Примечание"
+``` text
-    Срок хранения логов не ограничен. Логи не удаляются из таблицы автоматически. Вам необходимо самостоятельно организовать удаление устаревших логов.
+Row 1:
+──────
+event_date:           2020-05-13
+event_time:           2020-05-13 14:02:28
+query_start_time:     2020-05-13 14:02:28
+query_duration_ms:    0
+read_rows:            1
+read_bytes:           1
+written_rows:         0
+written_bytes:        0
+memory_usage:         0
+peak_memory_usage:    0
+thread_name:          QueryPipelineEx
+thread_id:            28952
+master_thread_id:     28924
+query:                SELECT 1
+is_initial_query:     1
+user:                 default
+query_id:             5e834082-6f6d-4e34-b47b-cd1934f4002a
+address:              ::ffff:127.0.0.1
+port:                 57720
+initial_user:         default
+initial_query_id:     5e834082-6f6d-4e34-b47b-cd1934f4002a
+initial_address:      ::ffff:127.0.0.1
+initial_port:         57720
+interface:            1
+os_user:              bayonet
+client_hostname:      clickhouse.ru-central1.internal
+client_name:          ClickHouse client
+client_revision:      54434
+client_version_major: 20
+client_version_minor: 4
+client_version_patch: 1
+http_method:          0
+http_user_agent:
+quota_key:
+revision:             54434
+ProfileEvents.Names:  ['ContextLock','RealTimeMicroseconds','UserTimeMicroseconds','OSCPUWaitMicroseconds','OSCPUVirtualTimeMicroseconds']
+ProfileEvents.Values: [1,97,81,5,81]
+...
+```
 
-Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) (параметр `partition_by`).
+**Смотрите также**
 
-## system.query_thread_log {#system_tables-query_thread_log}
+- [system.query_log](#system_tables-query_log) — описание системной таблицы `query_log`, которая содержит общую информацию о выполненных запросах.
-
-Содержит информацию о каждом потоке исполнения запроса.
 
 ## system.trace\_log {#system_tables-trace_log}
 

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md

@@ -32,7 +32,7 @@ ClickHouse поддерживает иерархические словари с
 
 ClickHouse поддерживает свойство [hierarchical](external-dicts-dict-structure.md#hierarchical-dict-attr) для атрибутов [внешнего словаря](index.md). Это свойство позволяет конфигурировать словари, подобные описанному выше.
 
-С помощью функции [dictGetHierarchy](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md#dictgethierarchy) можно получить цепочку предков элемента.
+С помощью функции [dictGetHierarchy](../../../sql-reference/functions/ext-dict-functions.md#dictgethierarchy) можно получить цепочку предков элемента.
 
 Структура словаря для нашего примера может выглядеть следующим образом:
 

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md

@@ -2,7 +2,7 @@
 
 Словари можно размещать в памяти множеством способов.
 
-Рекомендуем [flat](#flat), [hashed](#hashed) и [complex\_key\_hashed](#complex-key-hashed). Скорость обработки словарей при этом максимальна.
+Рекомендуем [flat](#flat), [hashed](#dicts-external_dicts_dict_layout-hashed) и [complex\_key\_hashed](#complex-key-hashed). Скорость обработки словарей при этом максимальна.
 
 Размещение с кэшированием не рекомендуется использовать из-за потенциально низкой производительности и сложностей в подборе оптимальных параметров. Читайте об этом подробнее в разделе «[cache](#cache)».
 
@@ -34,7 +34,7 @@
 </yandex>
 ```
 
-Соответствущий [DDL-запрос](../../../sql-reference/statements/create.md#create-dictionary-query):
+Соответствущий [DDL-запрос](../../statements/create.md#create-dictionary-query):
 
 ``` sql
 CREATE DICTIONARY (...)
@@ -46,7 +46,7 @@ LAYOUT(LAYOUT_TYPE(param value)) -- layout settings
 ## Способы размещения словарей в памяти {#sposoby-razmeshcheniia-slovarei-v-pamiati}
 
 -   [flat](#flat)
--   [hashed](#hashed)
+-   [hashed](#dicts-external_dicts_dict_layout-hashed)
 -   [sparse\_hashed](#dicts-external_dicts_dict_layout-sparse_hashed)
 -   [cache](#cache)
 -   [direct](#direct)
@@ -80,7 +80,7 @@ LAYOUT(LAYOUT_TYPE(param value)) -- layout settings
 LAYOUT(FLAT())
 ```
 
-### hashed {#hashed}
+### hashed {#dicts-external_dicts_dict_layout-hashed}
 
 Словарь полностью хранится в оперативной памяти в виде хэш-таблиц. Словарь может содержать произвольное количество элементов с произвольными идентификаторами. На практике, количество ключей может достигать десятков миллионов элементов.
 

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md

@@ -19,7 +19,7 @@
 </yandex>
 ```
 
-Аналогичный [DDL-запрос](../../../sql-reference/statements/create.md#create-dictionary-query):
+Аналогичный [DDL-запрос](../../statements/create.md#create-dictionary-query):
 
 ``` sql
 CREATE DICTIONARY dict_name (...)
@@ -150,7 +150,7 @@ SOURCE(HTTP(
 ))
 ```
 
-Чтобы ClickHouse смог обратиться к HTTPS-ресурсу, необходимо [настроить openSSL](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) в конфигурации сервера.
+Чтобы ClickHouse смог обратиться к HTTPS-ресурсу, необходимо [настроить openSSL](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-openssl) в конфигурации сервера.
 
 Поля настройки:
 
@@ -531,7 +531,7 @@ SOURCE(CLICKHOUSE(
 
 Поля настройки:
 
--   `host` — хост ClickHouse. Если host локальный, то запрос выполняется без сетевого взаимодействия. Чтобы повысить отказоустойчивость решения, можно создать таблицу типа [Distributed](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) и прописать её в дальнейших настройках.
+-   `host` — хост ClickHouse. Если host локальный, то запрос выполняется без сетевого взаимодействия. Чтобы повысить отказоустойчивость решения, можно создать таблицу типа [Distributed](../../../engines/table-engines/special/distributed.md) и прописать её в дальнейших настройках.
 -   `port` — порт сервера ClickHouse.
 -   `user` — имя пользователя ClickHouse.
 -   `password` — пароль пользователя ClickHouse.

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md

@@ -154,7 +154,7 @@ CREATE DICTIONARY somename (
 | Тег                                                  | Описание                                                                                                                                                                                                                                                                                                                                                      | Обязательный |
 |------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------|
 | `name`                                               | Имя столбца.                                                                                                                                                                                                                                                                                                                                                  | Да           |
-| `type`                                               | Тип данных ClickHouse.<br/>ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`. [Nullable](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md) не поддерживается. | Да           |
+| `type`                                               | Тип данных ClickHouse.<br/>ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`. [Nullable](../../../sql-reference/data-types/nullable.md) не поддерживается. | Да           |
 | `null_value`                                         | Значение по умолчанию для несуществующего элемента.<br/>В примере это пустая строка. Нельзя указать значение `NULL`.                                                                                                                                                                                                                                          | Да           |
 | `expression`                                         | [Выражение](../../syntax.md#syntax-expressions), которое ClickHouse выполняет со значением.<br/>Выражением может быть имя столбца в удаленной SQL базе. Таким образом, вы можете использовать его для создания псевдонима удаленного столбца.<br/><br/>Значение по умолчанию: нет выражения.                                                                  | Нет          |
 | <a name="hierarchical-dict-attr"></a> `hierarchical` | Если `true`, то атрибут содержит ключ предка для текущего элемента. Смотрите [Иерархические словари](external-dicts-dict-hierarchical.md).<br/><br/>Default value: `false`.                                                                                                                                                                                   | No           |
@@ -162,6 +162,6 @@ CREATE DICTIONARY somename (
 
 ## Смотрите также {#smotrite-takzhe}
 
--   [Функции для работы с внешними словарями](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md).
+-   [Функции для работы с внешними словарями](../../../sql-reference/functions/ext-dict-functions.md).
 
 [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_structure/) <!--hide-->

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md

@@ -24,7 +24,7 @@ XML-конфигурация словаря имеет следующую стр
 </dictionary>
 ```
 
-Соответствующий [DDL-запрос](../../../sql-reference/statements/create.md#create-dictionary-query) имеет следующий вид:
+Соответствующий [DDL-запрос](../../statements/create.md#create-dictionary-query) имеет следующий вид:
 
 ``` sql
 CREATE DICTIONARY dict_name

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md

@@ -5,11 +5,11 @@
 ClickHouse:
 - Полностью или частично хранит словари в оперативной памяти.
 - Периодически обновляет их и динамически подгружает отсутствующие значения.
-- Позволяет создавать внешние словари с помощью xml-файлов или [DDL-запросов](../../../sql-reference/statements/create.md#create-dictionary-query).
+- Позволяет создавать внешние словари с помощью xml-файлов или [DDL-запросов](../../statements/create.md#create-dictionary-query).
 
-Конфигурация внешних словарей может находится в одном или нескольких xml-файлах. Путь к конфигурации указывается в параметре [dictionaries\_config](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md).
+Конфигурация внешних словарей может находится в одном или нескольких xml-файлах. Путь к конфигурации указывается в параметре [dictionaries\_config](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_config).
 
-Словари могут загружаться при старте сервера или при первом использовании, в зависимости от настройки [dictionaries\_lazy\_load](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md).
+Словари могут загружаться при старте сервера или при первом использовании, в зависимости от настройки [dictionaries\_lazy\_load](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_lazy_load).
 
 Системная таблица [system.dictionaries](../../../operations/system-tables.md#system_tables-dictionaries) содержит информацию о словарях, сконфигурированных на сервере. Для каждого словаря там можно найти:
 
@@ -41,10 +41,10 @@ ClickHouse:
 
 В одном файле можно [сконфигурировать](external-dicts-dict.md) произвольное количество словарей.
 
-Если вы создаёте внешние словари [DDL-запросами](../../../sql-reference/statements/create.md#create-dictionary-query), то не задавайте конфигурацию словаря в конфигурации сервера.
+Если вы создаёте внешние словари [DDL-запросами](../../statements/create.md#create-dictionary-query), то не задавайте конфигурацию словаря в конфигурации сервера.
 
 !!! attention "Внимание"
-    Можно преобразовывать значения по небольшому словарю, описав его в запросе `SELECT` (см. функцию [transform](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md)). Эта функциональность не связана с внешними словарями.
+    Можно преобразовывать значения по небольшому словарю, описав его в запросе `SELECT` (см. функцию [transform](../../../sql-reference/functions/other-functions.md)). Эта функциональность не связана с внешними словарями.
 
 ## Смотрите также {#ext-dicts-see-also}
 
@@ -53,6 +53,6 @@ ClickHouse:
 -   [Обновление словарей](external-dicts-dict-lifetime.md)
 -   [Источники внешних словарей](external-dicts-dict-sources.md)
 -   [Ключ и поля словаря](external-dicts-dict-structure.md)
--   [Функции для работы с внешними словарями](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md#ext_dict_functions)
+-   [Функции для работы с внешними словарями](../../../sql-reference/functions/ext-dict-functions.md)
 
 [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts/) <!--hide-->

docs/ru/sql-reference/index.md

@@ -10,7 +10,7 @@ toc_title: hidden
 -   [SELECT](statements/select/index.md)
 -   [INSERT INTO](statements/insert-into.md)
 -   [CREATE](statements/create.md)
--   [ALTER](statements/alter.md)
+-   [ALTER](statements/alter.md#query_language_queries_alter)
 -   [Прочие виды запросов](statements/misc.md)
 
 [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/) <!--hide-->

docs/ru/sql-reference/statements/system.md

@@ -1,9 +1,12 @@
 # Запросы SYSTEM {#query-language-system}
 
+-   [RELOAD EMBEDDED DICTIONARIES](#query_language-system-reload-emdedded-dictionaries) 
 -   [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries)
 -   [RELOAD DICTIONARY](#query_language-system-reload-dictionary)
 -   [DROP DNS CACHE](#query_language-system-drop-dns-cache)
 -   [DROP MARK CACHE](#query_language-system-drop-mark-cache)
+-   [DROP UNCOMPRESSED CACHE](#query_language-system-drop-uncompressed-cache) 
+-   [DROP COMPILED EXPRESSION CACHE](#query_language-system-drop-compiled-expression-cache)
 -   [FLUSH LOGS](#query_language-system-flush_logs)
 -   [RELOAD CONFIG](#query_language-system-reload-config)
 -   [SHUTDOWN](#query_language-system-shutdown)
@@ -13,6 +16,24 @@
 -   [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends)
 -   [STOP MERGES](#query_language-system-stop-merges)
 -   [START MERGES](#query_language-system-start-merges)
+-   [STOP TTL MERGES](#query_language-stop-ttl-merges) 
+-   [START TTL MERGES](#query_language-start-ttl-merges) 
+-   [STOP MOVES](#query_language-stop-moves) 
+-   [START MOVES](#query_language-start-moves) 
+-   [STOP FETCHES](#query_language-system-stop-fetches)
+-   [START FETCHES](#query_language-system-start-fetches)
+-   [STOP REPLICATED SENDS](#query_language-system-start-replicated-sends)
+-   [START REPLICATED SENDS](#query_language-system-start-replicated-sends)
+-   [STOP REPLICATION QUEUES](#query_language-system-stop-replication-queues)
+-   [START REPLICATION QUEUES](#query_language-system-start-replication-queues)
+-   [SYNC REPLICA](#query_language-system-sync-replica)
+-   [RESTART REPLICA](#query_language-system-restart-replica)
+-   [RESTART REPLICAS](#query_language-system-restart-replicas) 
+
+## RELOAD EMBEDDED DICTIONARIES] {#query_language-system-reload-emdedded-dictionaries} 
+Перегружет все [Встроенные словари](../dictionaries/internal-dicts.md).
+По умолчанию встроенные словари выключены. 
+Всегда возвращает `Ok.`, вне зависимости от результата обновления встроенных словарей.
    
 ## RELOAD DICTIONARIES {#query_language-system-reload-dictionaries}
 
@@ -40,6 +61,16 @@ SELECT name, status FROM system.dictionaries;
 
 Сбрасывает кеш «засечек» (`mark cache`). Используется при разработке ClickHouse и тестах производительности.
 
+## DROP UNCOMPRESSED CACHE {#query_language-system-drop-uncompressed-cache}
+
+Сбрасывает кеш не сжатых данных. Используется при разработке ClickHouse и тестах производительности.
+Для управления кешем не сжатых данных используйте следующие настройки уровня сервера [uncompressed_cache_size](../../operations/server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) и настройки уровня запрос/пользователь/профиль [use_uncompressed_cache](../../operations/settings/settings.md#setting-use_uncompressed_cache)
+
+
+## DROP COMPILED EXPRESSION CACHE {#query_language-system-drop-compiled-expression-cache}
+Сбрасывает кеш скомпилированных выражений. Используется при разработке ClickHouse и тестах производительности.
+Компилированные выражения используются когда включена настройка уровня запрос/пользователь/профиль [compile](../../operations/settings/settings.md#compile)
+
 ## FLUSH LOGS {#query_language-system-flush_logs}
 
 Записывает буферы логов в системные таблицы (например system.query\_log). Позволяет не ждать 7.5 секунд при отладке.
@@ -84,6 +115,10 @@ SYSTEM FLUSH DISTRIBUTED [db.]<distributed_table_name>
 SYSTEM START DISTRIBUTED SENDS [db.]<distributed_table_name>
 ```
 
+## Managing MergeTree Tables {#query-language-system-mergetree}
+
+ClickHouse может управлять фоновыми процессами в [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) таблицах.
+
 ### STOP MERGES {#query_language-system-stop-merges}
 
 Позволяет остановить фоновые мержи для таблиц семейства MergeTree:
@@ -103,4 +138,110 @@ SYSTEM STOP MERGES [[db.]merge_tree_family_table_name]
 SYSTEM START MERGES [[db.]merge_tree_family_table_name]
 ```
 
+### STOP TTL MERGES {#query_language-stop-ttl-merges}
+
+Позволяет остановить фоновые процессы удаления старых данных основанные на [выражениях TTL](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl) для таблиц семейства MergeTree:
+Возвращает `Ok.` даже если указана несуществующая таблица или таблица имеет тип отличный от MergeTree. Возвращает ошибку если указана не существующая база данных:
+
+``` sql
+SYSTEM STOP TTL MERGES [[db.]merge_tree_family_table_name]
+```
+
+### START TTL MERGES {#query_language-start-ttl-merges} 
+
+Запускает фоновые процессы удаления старых данных основанные на [выражениях TTL](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl) для таблиц семейства MergeTree:
+Возвращает `Ok.` даже если указана несуществующая таблица или таблица имеет тип отличный от MergeTree. Возвращает ошибку если указана не существующая база данных:
+
+``` sql
+SYSTEM START TTL MERGES [[db.]merge_tree_family_table_name]
+```
+
+### STOP MOVES {#query_language-stop-moves} 
+
+Позволяет остановить фоновые процессы переноса данных основанные [табличных выражениях TTL с использованием TO VOLUME или TO DISK](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) for tables in the MergeTree family:
+Возвращает `Ok.` даже если указана несуществующая таблица или таблица имеет тип отличный от MergeTree. Возвращает ошибку если указана не существующая база данных:
+
+``` sql
+SYSTEM STOP MOVES [[db.]merge_tree_family_table_name]
+```
+
+### START MOVES {#query_language-start-moves} 
+
+Запускает фоновые процессы переноса данных основанные [табличных выражениях TTL с использованием TO VOLUME или TO DISK](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) for tables in the MergeTree family:
+Возвращает `Ok.` даже если указана несуществующая таблица или таблица имеет тип отличный от MergeTree. Возвращает ошибку если указана не существующая база данных:
+
+``` sql
+SYSTEM STOP MOVES [[db.]merge_tree_family_table_name]
+```
+
+## Managing ReplicatedMergeTree Tables {#query-language-system-replicated}
+
+ClickHouse может управлять фоновыми процессами связанными c репликацией в таблицах семейства [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md).
+
+### STOP FETCHES {#query_language-system-stop-fetches}
+Позволяет остановить фоновые процессы синхронизации новыми вставленными кусками данных с другими репликами в кластере для таблиц семейства `ReplicatedMergeTree`:
+Всегда возвращает `Ok.` вне зависимости от типа таблицы и даже если таблица или база данных не существет.
+
+``` sql
+SYSTEM STOP FETCHES [[db.]replicated_merge_tree_family_table_name]
+```
+
+### START FETCHES {#query_language-system-start-fetches}
+Позволяет запустить фоновые процессы синхронизации новыми вставленными кусками данных с другими репликами в кластере для таблиц семейства `ReplicatedMergeTree`:
+Всегда возвращает `Ok.` вне зависимости от типа таблицы и даже если таблица или база данных не существет.
+
+``` sql
+SYSTEM START FETCHES [[db.]replicated_merge_tree_family_table_name]
+```
+
+### STOP REPLICATED SENDS {#query_language-system-start-replicated-sends}
+Позволяет остановить фоновые процессы отсылки новых вставленных кусков данных другим репликам в кластере для таблиц семейства `ReplicatedMergeTree`:
+
+``` sql
+SYSTEM STOP REPLICATED SENDS [[db.]replicated_merge_tree_family_table_name]
+```
+
+### START REPLICATED SENDS {#query_language-system-start-replicated-sends}
+Позволяет запустить фоновые процессы отсылки новых вставленных кусков данных другим репликам в кластере для таблиц семейства `ReplicatedMergeTree`:
+
+``` sql
+SYSTEM START REPLICATED SENDS [[db.]replicated_merge_tree_family_table_name]
+```
+
+### STOP REPLICATION QUEUES {#query_language-system-stop-replication-queues}
+Останавливает фоновые процессы разбора заданий из очереди репликации которая хранится в Zookeeper для таблиц семейства `ReplicatedMergeTree`. Возможные типы заданий - merges, fetches, mutation, DDL запросы с ON CLUSTER: 
+
+``` sql
+SYSTEM STOP REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name]
+```
+
+### START REPLICATION QUEUES {#query_language-system-start-replication-queues}
+Запускает фоновые процессы разбора заданий из очереди репликации которая хранится в Zookeeper для таблиц семейства `ReplicatedMergeTree`. Возможные типы заданий - merges, fetches, mutation, DDL запросы с ON CLUSTER: 
+
+``` sql
+SYSTEM START REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name]
+```
+
+### SYNC REPLICA {#query_language-system-sync-replica}
+Ждет когда таблица семейства `ReplicatedMergeTree` будет синхронизирована с другими репликами в кластере, будет работать до достижения `receive_timeout`, если синхронизация для таблицы отключена в настоящий момент времени:  
+
+``` sql
+SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name
+```
+
+### RESTART REPLICA {#query_language-system-restart-replica}
+Реинициализация состояния Zookeeper сессий для таблицы семейства `ReplicatedMergeTree`, сравнивает текущее состояние с тем что хранится в Zookeeper как источник правды и добавляет задачи Zookeeper очередь если необходимо  
+Инициализация очереди репликации на основе данных ZooKeeper, происходит так же как при attach table. На короткое время таблица станет недоступной для любых операций.
+
+``` sql
+SYSTEM RESTART QUEUES [db.]replicated_merge_tree_family_table_name
+```
+
+### RESTART REPLICAS {#query_language-system-restart-replicas}
+Реинициализация состояния Zookeeper сессий для всех `ReplicatedMergeTree` таблиц, сравнивает текущее состояние с тем что хранится в Zookeeper как источник правды и добавляет задачи Zookeeper очередь если необходимо
+
+``` sql
+SYSTEM RESTART QUEUES [db.]replicated_merge_tree_family_table_name
+```
+
 [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/system/) <!--hide-->

docs/tools/requirements.txt

@@ -2,7 +2,7 @@ Babel==2.8.0
 backports-abc==0.5
 backports.functools-lru-cache==1.6.1
 beautifulsoup4==4.9.1
-certifi==2020.4.5.1
+certifi==2020.4.5.2
 chardet==3.0.4
 click==7.1.2
 closure==20191111
@@ -13,7 +13,7 @@ idna==2.9
 Jinja2==2.11.2
 jinja2-highlight==0.6.1
 jsmin==2.2.2
-livereload==2.6.1
+livereload==2.6.2
 Markdown==3.2.1
 MarkupSafe==1.1.1
 mkdocs==1.1.2
@@ -22,7 +22,7 @@ mkdocs-macros-plugin==0.4.9
 nltk==3.5
 nose==1.3.7
 protobuf==3.12.2
-numpy==1.18.4
+numpy==1.18.5
 Pygments==2.5.2
 pymdown-extensions==7.1
 python-slugify==1.2.6

docs/tools/translate/requirements.txt

@@ -1,5 +1,5 @@
 Babel==2.8.0
-certifi==2020.4.5.1
+certifi==2020.4.5.2
 chardet==3.0.4
 googletrans==2.4.0
 idna==2.9

docs/tr/development/developer-instruction.md

@@ -141,7 +141,7 @@ Resmi Yandex şu anda GCC'Yİ kullanıyor çünkü biraz daha iyi performansa sa
 
 Ubuntu run GCC yüklemek için: `sudo apt install gcc g++`
 
-Gcc sürümünü kontrol edin: `gcc --version`. 9'un altındaysa, buradaki talimatları izleyin: https://clickhouse.tech / docs/TR/development / build / \#ınstall-gcc-9.
+Gcc sürümünü kontrol edin: `gcc --version`. 9'un altındaysa, buradaki talimatları izleyin: https://clickhouse.tech/docs/tr/development/build/#install-gcc-9.
 
 Mac OS X build sadece Clang için desteklenir. Sadece koş `brew install llvm`
 
@@ -249,7 +249,7 @@ Kod stili Kılavuzu: https://clickhouse.tech / doscs / TR / development / style/
 
 Yazma testleri: https://clickhouse.teknoloji / doscs / TR / geliştirme / testler/
 
-Görevlerin listesi: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md
+Görevlerin listesi: https://github.com/ClickHouse/ClickHouse/contribute
 
 # Test Verileri {#test-data}
 

docs/zh/development/developer-instruction.md

@@ -129,7 +129,7 @@ Yandex官方当前使用GCC构建ClickHouse，因为它生成的机器代码性
 
 在Ubuntu上安装GCC，请执行：`sudo apt install gcc g++`
 
-请使用`gcc --version`查看gcc的版本。如果gcc版本低于9，请参考此处的指示：https://clickhouse.tech/docs/en/development/build/\#install-gcc-9 。
+请使用`gcc --version`查看gcc的版本。如果gcc版本低于9，请参考此处的指示：https://clickhouse.tech/docs/zh/development/build/#an-zhuang-gcc-9 。
 
 在Mac OS X上安装GCC，请执行：`brew install gcc`
 
@@ -234,7 +234,7 @@ ClickHouse的架构描述可以在此处查看：https://clickhouse.tech/docs/en
 
 编写测试用例：https://clickhouse.tech/docs/en/development/tests/
 
-任务列表：https://github.com/ClickHouse/ClickHouse/blob/master/tests/instructions/easy\_tasks\_sorted\_en.md
+任务列表：https://github.com/ClickHouse/ClickHouse/contribute
 
 # 测试数据 {#ce-shi-shu-ju}
 

docs/zh/index.md

@@ -4,7 +4,7 @@ ClickHouse是一个用于联机分析(OLAP)的列式数据库管理系统(DBMS)
 
 在传统的行式数据库系统中，数据按如下顺序存储：
 
-| 行  | 小心点      | JavaEnable | 标题       | GoodEvent | 活动时间            |
+| row  | watchID      | JavaEnable | title      | GoodEvent | EventTime            |
 |-----|-------------|------------|------------|-----------|---------------------|
 | \#0 | 89354350662 | 1          | 投资者关系 | 1         | 2016-05-18 05:19:20 |
 | \#1 | 90329509958 | 0          | 联系我们   | 1         | 2016-05-18 08:10:20 |
@@ -18,23 +18,23 @@ ClickHouse是一个用于联机分析(OLAP)的列式数据库管理系统(DBMS)
 
 在列式数据库系统中，数据按如下的顺序存储：
 
-| 行:         | \#0                 | \#1                 | \#2                 | \#N |
+| row:         | \#0                 | \#1                 | \#2                 | \#N |
 |-------------|---------------------|---------------------|---------------------|-----|
-| 小心点:     | 89354350662         | 90329509958         | 89953706054         | …   |
+| watchID:     | 89354350662         | 90329509958         | 89953706054         | …   |
 | JavaEnable: | 1                   | 0                   | 1                   | …   |
-| 标题:       | 投资者关系          | 联系我们            | 任务                | …   |
+| title:       | 投资者关系          | 联系我们            | 任务                | …   |
 | GoodEvent:  | 1                   | 1                   | 1                   | …   |
-| 活动时间:   | 2016-05-18 05:19:20 | 2016-05-18 08:10:20 | 2016-05-18 07:38:00 | …   |
+| EventTime:   | 2016-05-18 05:19:20 | 2016-05-18 08:10:20 | 2016-05-18 07:38:00 | …   |
 
-该示例中只展示了数据在列式数据库中数据的排列顺序。
+该示例中只展示了数据在列式数据库中数据的排列方式。
 对于存储而言，列式数据库总是将同一列的数据存储在一起，不同列的数据也总是分开存储。
 
 常见的列式数据库有： Vertica、 Paraccel (Actian Matrix，Amazon Redshift)、 Sybase IQ、 Exasol、 Infobright、 InfiniDB、 MonetDB (VectorWise， Actian Vector)、 LucidDB、 SAP HANA、 Google Dremel、 Google PowerDrill、 Druid、 kdb+。
 {: .灰色 }
 
-不同的存储方式适合不同的场景，这里的查询场景包括： 进行了哪些查询，多久查询一次以及各类查询的比例； 每种查询读取多少数据————行、列和字节；读取数据和写入数据之间的关系；使用的数据集大小以及如何使用本地的数据集；是否使用事务,以及它们是如何进行隔离的；数据的复制机制与数据的完整性要求；每种类型的查询要求的延迟与吞吐量等等。
+不同的数据存储方式适用不同的业务场景，数据访问的场景包括：进行了何种查询、多久查询一次以及各类查询的比例； 每种查询读取多少数据————行、列和字节；读取数据和写入数据之间的关系；使用的数据集大小以及如何使用本地的数据集；是否使用事务,以及它们是如何进行隔离的；数据的复制机制与数据的完整性要求；每种类型的查询要求的延迟与吞吐量等等。
 
-系统负载越高，根据使用场景进行定制化就越重要，并且定制将会变的越精细。没有一个系统同样适用于明显不同的场景。如果系统适用于广泛的场景，在负载高的情况下，所有的场景可以会被公平但低效处理，或者高效处理一小部分场景。
+系统负载越高，依据使用场景进行定制化就越重要，并且定制将会变的越精细。没有一个系统能够同时适用所有明显不同的业务场景。如果系统适用于广泛的场景，在负载高的情况下，要兼顾所有的场景，那么将不得不做出选择。是要平衡还是要效率？
 
 ## OLAP场景的关键特征 {#olapchang-jing-de-guan-jian-te-zheng}
 
@@ -52,7 +52,7 @@ ClickHouse是一个用于联机分析(OLAP)的列式数据库管理系统(DBMS)
 -   每一个查询除了一个大表外都很小
 -   查询结果明显小于源数据，换句话说，数据被过滤或聚合后能够被盛放在单台服务器的内存中
 
-很容易可以看出，OLAP场景与其他流行场景(例如,OLTP或K/V)有很大的不同， 因此想要使用OLTP或Key-Value数据库去高效的处理分析查询是没有意义的，例如，使用OLAP数据库去处理分析请求通常要优于使用MongoDB或Redis去处理分析请求。
+很容易可以看出，OLAP场景与其他通常业务场景(例如,OLTP或K/V)有很大的不同， 因此想要使用OLTP或Key-Value数据库去高效的处理分析查询场景，并不是非常完美的适用方案。例如，使用OLAP数据库去处理分析请求通常要优于使用MongoDB或Redis去处理分析请求。
 
 ## 列式数据库更适合OLAP场景的原因 {#lie-shi-shu-ju-ku-geng-gua-he-olapchang-jing-de-yuan-yin}
 

docs/zh/operations/utilities/clickhouse-copier.md

@@ -24,7 +24,7 @@
 该实用程序应手动运行:
 
 ``` bash
-clickhouse-copier copier --daemon --config zookeeper.xml --task-path /task/path --base-dir /path/to/dir
+clickhouse-copier --daemon --config zookeeper.xml --task-path /task/path --base-dir /path/to/dir
 ```
 
 参数:

docs/zh/sql-reference/aggregate-functions/parametric-functions.md

@@ -313,7 +313,7 @@ ORDER BY level ASC
 └───────┴───┘
 ```
 
-## 保留 {#retention}
+## Retention {#retention}
 
 该函数将一组条件作为参数，类型为1到32个参数 `UInt8` 表示事件是否满足特定条件。
 任何条件都可以指定为参数（如 [WHERE](../../sql-reference/statements/select/where.md#select-where)).

docs/zh/sql-reference/data-types/lowcardinality.md

@@ -0,0 +1,59 @@
+---
+toc_priority: 51
+toc_title: 低基数类型
+---
+
+# 低基数类型 {#lowcardinality-data-type}
+
+把其它数据类型转变为字典编码类型。
+
+## 语法 {#lowcardinality-syntax}
+
+```sql
+LowCardinality(data_type)
+```
+
+**参数**
+
+- `data_type` — [String](string.md), [FixedString](fixedstring.md), [Date](date.md), [DateTime](datetime.md)，包括数字类型，但是[Decimal](decimal.md)除外。对一些数据类型来说，`LowCardinality` 并不高效，详查[allow_suspicious_low_cardinality_types](../../operations/settings/settings.md#allow_suspicious_low_cardinality_types)设置描述。
+
+## 描述 {#lowcardinality-dscr}
+
+`LowCardinality` 是一种改变数据存储和数据处理方法的概念。 ClickHouse会把 `LowCardinality` 所在的列进行[dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder)。对很多应用来说，处理字典编码的数据可以显著的增加[SELECT](../statements/select/index.md)查询速度。
+
+使用 `LowCarditality` 数据类型的效率依赖于数据的多样性。如果一个字典包含少于10000个不同的值，那么ClickHouse可以进行更高效的数据存储和处理。反之如果字典多于10000，效率会表现的更差。
+
+当使用字符类型的时候，可以考虑使用 `LowCardinality` 代替[Enum](enum.md)。 `LowCardinality` 通常更加灵活和高效。
+
+## 例子
+
+创建一个 `LowCardinality` 类型的列：
+
+```sql
+CREATE TABLE lc_t
+(
+    `id` UInt16, 
+    `strings` LowCardinality(String)
+)
+ENGINE = MergeTree()
+ORDER BY id
+```
+
+## 相关的设置和函数
+
+设置:
+
+- [low_cardinality_max_dictionary_size](../../operations/settings/settings.md#low_cardinality_max_dictionary_size)
+- [low_cardinality_use_single_dictionary_for_part](../../operations/settings/settings.md#low_cardinality_use_single_dictionary_for_part)
+- [low_cardinality_allow_in_native_format](../../operations/settings/settings.md#low_cardinality_allow_in_native_format)
+- [allow_suspicious_low_cardinality_types](../../operations/settings/settings.md#allow_suspicious_low_cardinality_types)
+
+函数:
+
+- [toLowCardinality](../functions/type-conversion-functions.md#tolowcardinality)
+
+## 参考
+
+- [高效低基数类型](https://www.altinity.com/blog/2019/3/27/low-cardinality).
+- [使用低基数类型减少ClickHouse的存储成本 – 来自Instana工程师的分享](https://www.instana.com/blog/reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer/).
+- [字符优化 (俄语视频分享)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [英语分享](https://github.com/yandex/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf).

docs/zh/sql-reference/functions/arithmetic-functions.md

@@ -1,3 +1,8 @@
+---
+toc_priority: 35
+toc_title: 算术函数
+---
+
 # 算术函数 {#suan-zhu-han-shu}
 
 对于所有算术函数，结果类型为结果适合的最小数字类型（如果存在这样的类型）。最小数字类型是根据数字的位数，是否有符号以及是否是浮点类型而同时进行的。如果没有足够的位，则采用最高位类型。

docs/zh/sql-reference/functions/comparison-functions.md

@@ -1,3 +1,8 @@
+---
+toc_priority: 36
+toc_title: 比较函数
+---
+
 # 比较函数 {#bi-jiao-han-shu}
 
 比较函数始终返回0或1（UInt8）。
@@ -15,18 +20,16 @@
 
 字符串按字节进行比较。较短的字符串小于以其开头并且至少包含一个字符的所有字符串。
 
-注意。直到1.1.54134版本，有符号和无符号数字的比较方式与C++相同。换句话说，在SELECT 9223372036854775807 ＆gt; -1 等情况下，您可能会得到错误的结果。 此行为在版本1.1.54134中已更改，现在在数学上是正确的。
-
 ## 等于，a=b和a==b 运算符 {#equals-a-b-and-a-b-operator}
 
-## notEquals,a! 运算符=b和a `<>` b {#notequals-a-operator-b-and-a-b}
+## 不等于，a!=b和a<>b 运算符 {#notequals-a-operator-b-and-a-b}
 
-## 少, `< operator` {#less-operator}
+## 少, < 运算符 {#less-operator}
 
-## 更大, `> operator` {#greater-operator}
+## 大于, > 运算符 {#greater-operator}
 
-## 出租等级, `<= operator` {#lessorequals-operator}
+## 小于等于, <= 运算符 {#lessorequals-operator}
 
-## 伟大的等级, `>= operator` {#greaterorequals-operator}
+## 大于等于, >= 运算符 {#greaterorequals-operator}
 
 [来源文章](https://clickhouse.tech/docs/en/query_language/functions/comparison_functions/) <!--hide-->

docs/zh/sql-reference/functions/index.md

@@ -1,3 +1,9 @@
+---
+toc_folder_title: 函数
+toc_priority: 32
+toc_title: 简介
+---
+
 # 函数 {#han-shu}
 
 ClickHouse中至少存在两种类型的函数 - 常规函数（它们称之为«函数»）和聚合函数。 常规函数的工作就像分别为每一行执行一次函数计算一样（对于每一行，函数的结果不依赖于其他行）。 聚合函数则从各行累积一组值（即函数的结果以来整个结果集）。

docs/zh/sql-reference/functions/logical-functions.md

@@ -1,15 +1,20 @@
+---
+toc_priority: 37
+toc_title: 逻辑函数
+---
+
 # 逻辑函数 {#luo-ji-han-shu}
 
 逻辑函数可以接受任何数字类型的参数，并返回UInt8类型的0或1。
 
 当向函数传递零时，函数将判定为«false»，否则，任何其他非零的值都将被判定为«true»。
 
-## 和，和运营商 {#and-and-operator}
+## 和，`AND` 运算符 {#and-and-operator}
 
-## 或，或运营商 {#or-or-operator}
+## 或，`OR` 运算符 {#or-or-operator}
 
-## 不是，不是运营商 {#not-not-operator}
+## 非，`NOT` 运算符 {#not-not-operator}
 
-## 异或 {#xor}
+## 异或，`XOR` 运算符 {#xor}
 
 [来源文章](https://clickhouse.tech/docs/en/query_language/functions/logical_functions/) <!--hide-->

docs/zh/sql-reference/functions/type-conversion-functions.md

@@ -1,16 +1,230 @@
+---
+toc_priority: 38
+toc_title: 类型转换函数
+---
+
 # 类型转换函数 {#lei-xing-zhuan-huan-han-shu}
 
-## toUInt8,toUInt16,toUInt32,toUInt64 {#touint8-touint16-touint32-touint64}
+## 数值类型转换常见的问题 {#numeric-conversion-issues}
 
-## toInt8,toInt16,toInt32,toInt64 {#toint8-toint16-toint32-toint64}
+当你把一个值从一个类型转换为另外一个类型的时候，你需要注意的是这是一个不安全的操作，可能导致数据的丢失。数据丢失一般发生在你将一个大的数据类型转换为小的数据类型的时候，或者你把两个不同的数据类型相互转换的时候。
 
-## toFloat32,toFloat64 {#tofloat32-tofloat64}
+ClickHouse和[C++](https://en.cppreference.com/w/cpp/language/implicit_conversion)有相同的类型转换行为。
 
-## 今天，今天 {#todate-todatetime}
+## toInt(8\|16\|32\|64) {#touint8-touint16-touint32-touint64}
 
-## toUInt8OrZero,toUInt16OrZero,toUInt32OrZero,toUInt64OrZero,toInt8OrZero,toInt16OrZero,toInt32OrZero,toInt64OrZero,toFloat32OrZero,toFloat64OrZero,toDateOrZero,toDateTimeOrZero {#touint8orzero-touint16orzero-touint32orzero-touint64orzero-toint8orzero-toint16orzero-toint32orzero-toint64orzero-tofloat32orzero-tofloat64orzero-todateorzero-todatetimeorzero}
+转换一个输入值为[Int](../../sql-reference/data-types/int-uint.md)类型。这个函数包括：
+
+-   `toInt8(expr)` — 结果为`Int8`数据类型。
+-   `toInt16(expr)` — 结果为`Int16`数据类型。
+-   `toInt32(expr)` — 结果为`Int32`数据类型。
+-   `toInt64(expr)` — 结果为`Int64`数据类型。
+
+**参数**
+
+-   `expr` — [表达式](../syntax.md#syntax-expressions)返回一个数字或者代表数值类型的字符串。不支持二进制、八进制、十六进制的数字形式，有效数字之前的0也会被忽略。
+
+**返回值**
+
+整形在`Int8`, `Int16`, `Int32`，或者 `Int64` 的数据类型。
+
+函数使用[rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero)原则，这意味着会截断丢弃小数部分的数值。
+
+[NaN and Inf](../../sql-reference/data-types/float.md#data_type-float-nan-inf)转换是不确定的。具体使用的时候，请参考[数值类型转换常见的问题](#numeric-conversion-issues)。
+
+**例子**
+
+``` sql
+SELECT toInt64(nan), toInt32(32), toInt16('16'), toInt8(8.8)
+```
+
+``` text
+┌─────────toInt64(nan)─┬─toInt32(32)─┬─toInt16('16')─┬─toInt8(8.8)─┐
+│ -9223372036854775808 │          32 │            16 │           8 │
+└──────────────────────┴─────────────┴───────────────┴─────────────┘
+```
+
+## toInt(8\|16\|32\|64)OrZero {#toint8163264orzero}
+
+这个函数需要一个字符类型的入参，然后尝试把它转为`Int (8 | 16 | 32 | 64)`，如果转换失败直接返回0。
+
+**例子**
+
+``` sql
+select toInt64OrZero('123123'), toInt8OrZero('123qwe123')
+```
+
+``` text
+┌─toInt64OrZero('123123')─┬─toInt8OrZero('123qwe123')─┐
+│                  123123 │                         0 │
+└─────────────────────────┴───────────────────────────┘
+```
+## toInt(8\|16\|32\|64)OrNull {#toint8163264ornull}
+
+这个函数需要一个字符类型的入参，然后尝试把它转为`Int (8 | 16 | 32 | 64)`，如果转换失败直接返回`NULL`。
+
+**例子**
+
+``` sql
+select toInt64OrNull('123123'), toInt8OrNull('123qwe123')
+```
+
+``` text
+┌─toInt64OrNull('123123')─┬─toInt8OrNull('123qwe123')─┐
+│                  123123 │                      ᴺᵁᴸᴸ │
+└─────────────────────────┴───────────────────────────┘
+```
+
+## toUInt(8\|16\|32\|64) {#touint8163264}
+
+转换一个输入值到[UInt](../../sql-reference/data-types/int-uint.md)类型。 这个函数包括：
+
+-   `toUInt8(expr)` — 结果为`UInt8`数据类型。
+-   `toUInt16(expr)` — 结果为`UInt16`数据类型。
+-   `toUInt32(expr)` — 结果为`UInt32`数据类型。
+-   `toUInt64(expr)` — 结果为`UInt64`数据类型。
+
+**参数**
+
+-   `expr` — [表达式](../syntax.md#syntax-expressions)返回一个数字或者代表数值类型的字符串。不支持二进制、八进制、十六进制的数字形式，有效数字之前的0也会被忽略。
+
+**返回值**
+
+整形在`UInt8`, `UInt16`, `UInt32`，或者 `UInt64` 的数据类型。
+
+函数使用[rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero)原则，这意味着会截断丢弃小数部分的数值。
+
+对于负数和[NaN and Inf](../../sql-reference/data-types/float.md#data_type-float-nan-inf)来说转换的结果是不确定的。如果你传入一个负数，比如：`'-32'`，ClickHouse会抛出异常。具体使用的时候，请参考[数值类型转换常见的问题](#numeric-conversion-issues)。
+
+**例子**
+
+``` sql
+SELECT toUInt64(nan), toUInt32(-32), toUInt16('16'), toUInt8(8.8)
+```
+
+``` text
+┌───────toUInt64(nan)─┬─toUInt32(-32)─┬─toUInt16('16')─┬─toUInt8(8.8)─┐
+│ 9223372036854775808 │    4294967264 │             16 │            8 │
+└─────────────────────┴───────────────┴────────────────┴──────────────┘
+```
+
+## toUInt(8\|16\|32\|64)OrZero {#touint8163264orzero}
+
+## toUInt(8\|16\|32\|64)OrNull {#touint8163264ornull}
+
+## toFloat(32\|64) {#tofloat3264}
+
+## toFloat(32\|64)OrZero {#tofloat3264orzero}
+
+## toFloat(32\|64)OrNull {#tofloat3264ornull}
+
+## toDate {#todate}
+
+## toDateOrZero {#todateorzero}
+
+## toDateOrNull {#todateornull}
+
+## toDateTime {#todatetime}
+
+## toDateTimeOrZero {#todatetimeorzero}
+
+## toDateTimeOrNull {#todatetimeornull}
+
+## toDecimal(32\|64\|128) {#todecimal3264128}
+
+转换 `value` 到[Decimal](../../sql-reference/data-types/decimal.md)类型的值，其中精度为`S`。`value`可以是一个数字或者一个字符串。`S` 指定小数位的精度。
+
+-   `toDecimal32(value, S)`
+-   `toDecimal64(value, S)`
+-   `toDecimal128(value, S)`
+
+## toDecimal(32\|64\|128)OrNull {#todecimal3264128ornull}
+
+转换一个输入的字符到[Nullable(Decimal(P,S))](../../sql-reference/data-types/decimal.md)类型的数据。这个函数包括：
+
+-   `toDecimal32OrNull(expr, S)` — 结果为`Nullable(Decimal32(S))`数据类型。
+-   `toDecimal64OrNull(expr, S)` — 结果为`Nullable(Decimal64(S))`数据类型。
+-   `toDecimal128OrNull(expr, S)` — 结果为`Nullable(Decimal128(S))`数据类型。
+
+如果在解析输入值发生错误的时候你希望得到一个`NULL`值而不是抛出异常，你可以使用该函数。
+
+**参数**
+
+-   `expr` — [表达式](../syntax.md#syntax-expressions)返回一个[String](../../sql-reference/data-types/string.md)类型的数据。 ClickHouse倾向于文本类型的表示带小数类型的数值，比如`'1.111'`。
+-   `S` — 小数位的精度。
+
+**返回值**
+
+`Nullable(Decimal(P,S))`类型的数据，包括：
+
+-   如果有的话，小数位`S`。
+-   如果解析错误或者输入的数字的小数位多于`S`,那结果为`NULL`。
+
+**例子**
+
+``` sql
+SELECT toDecimal32OrNull(toString(-1.111), 5) AS val, toTypeName(val)
+```
+
+``` text
+┌──────val─┬─toTypeName(toDecimal32OrNull(toString(-1.111), 5))─┐
+│ -1.11100 │ Nullable(Decimal(9, 5))                            │
+└──────────┴────────────────────────────────────────────────────┘
+```
+
+``` sql
+SELECT toDecimal32OrNull(toString(-1.111), 2) AS val, toTypeName(val)
+```
+
+``` text
+┌──val─┬─toTypeName(toDecimal32OrNull(toString(-1.111), 2))─┐
+│ ᴺᵁᴸᴸ │ Nullable(Decimal(9, 2))                            │
+└──────┴────────────────────────────────────────────────────┘
+```
+
+## toDecimal(32\|64\|128)OrZero {#todecimal3264128orzero}
+
+转换输入值为[Decimal(P,S)](../../sql-reference/data-types/decimal.md)类型数据。这个函数包括：
+
+-   `toDecimal32OrZero( expr, S)` — 结果为`Decimal32(S)` 数据类型。
+-   `toDecimal64OrZero( expr, S)` — 结果为`Decimal64(S)` 数据类型。
+-   `toDecimal128OrZero( expr, S)` — 结果为`Decimal128(S)` 数据类型。
+
+当解析错误的时候，你不需要抛出异常而希望得到`0`值，你可以使用该函数。
+
+**参数**
+
+-   `expr` — [表达式](../syntax.md#syntax-expressions)返回一个[String](../../sql-reference/data-types/string.md)类型的数据。 ClickHouse倾向于文本类型的表示带小数类型的数值，比如`'1.111'`。
+-   `S` — 小数位的精度。
+
+**返回值**
+
+A value in the `Nullable(Decimal(P,S))` data type. The value contains:
+
+-   如果有的话，小数位`S`。
+-   如果解析错误或者输入的数字的小数位多于`S`,那结果为小数位精度为`S`的`0`。
+**例子**
+
+``` sql
+SELECT toDecimal32OrZero(toString(-1.111), 5) AS val, toTypeName(val)
+```
+
+``` text
+┌──────val─┬─toTypeName(toDecimal32OrZero(toString(-1.111), 5))─┐
+│ -1.11100 │ Decimal(9, 5)                                      │
+└──────────┴────────────────────────────────────────────────────┘
+```
+
+``` sql
+SELECT toDecimal32OrZero(toString(-1.111), 2) AS val, toTypeName(val)
+```
+
+``` text
+┌──val─┬─toTypeName(toDecimal32OrZero(toString(-1.111), 2))─┐
+│ 0.00 │ Decimal(9, 2)                                      │
+└──────┴────────────────────────────────────────────────────┘
+```
 
-## toUInt8OrNull,toUInt16OrNull,toUInt32OrNull,toUInt64OrNull,toInt8OrNull,toInt16OrNull,toInt32OrNull,toInt64OrNull,toFloat32OrNull,toFloat64OrNull,toDateOrNull,toDateTimeOrNull {#touint8ornull-touint16ornull-touint32ornull-touint64ornull-toint8ornull-toint16ornull-toint32ornull-toint64ornull-tofloat32ornull-tofloat64ornull-todateornull-todatetimeornull}
 
 ## toString {#tostring}
 
@@ -47,10 +261,6 @@ SELECT
 
 另请参阅`toUnixTimestamp`函数。
 
-## toDecimal32(value,S),toDecimal64(value,S),toDecimal128(value,S) {#todecimal32value-s-todecimal64value-s-todecimal128value-s}
-
-将`value`转换为精度为`S`的[十进制](../../sql-reference/functions/type-conversion-functions.md)。`value`可以是数字或字符串。`S`参数为指定的小数位数。
-
 ## toFixedString(s,N) {#tofixedstrings-n}
 
 将String类型的参数转换为FixedString(N)类型的值（具有固定长度N的字符串）。N必须是一个常量。
@@ -78,17 +288,19 @@ SELECT toFixedString('foo\0bar', 8) AS s, toStringCutToZero(s) AS s_cut
     │ foo\0bar\0 │ foo   │
     └────────────┴───────┘
 
-## reinterpretAsUInt8,reinterpretAsUInt16,reinterpretAsUInt32,reinterpretAsUInt64 {#reinterpretasuint8-reinterpretasuint16-reinterpretasuint32-reinterpretasuint64}
+## reinterpretAsUInt(8\|16\|32\|64) {#reinterpretasuint8163264}
 
-## reinterpretAsInt8,reinterpretAsInt16,reinterpretAsInt32,reinterpretAsInt64 {#reinterpretasint8-reinterpretasint16-reinterpretasint32-reinterpretasint64}
+## reinterpretAsInt(8\|16\|32\|64) {#reinterpretasint8163264}
 
-## reinterpretAsFloat32,reinterpretAsFloat64 {#reinterpretasfloat32-reinterpretasfloat64}
+## reinterpretAsFloat(32\|64) {#reinterpretasfloat3264}
 
-## 重新解释日期，重新解释日期时间 {#reinterpretasdate-reinterpretasdatetime}
+## reinterpretAsDate {#reinterpretasdate}
+
+## reinterpretAsDateTime {#reinterpretasdatetime}
 
 这些函数接受一个字符串，并将放在字符串开头的字节解释为主机顺序中的数字（little endian）。如果字符串不够长，则函数就像使用必要数量的空字节填充字符串一样。如果字符串比需要的长，则忽略额外的字节。Date被解释为Unix时间戳的天数，DateTime被解释为Unix时间戳。
 
-## 重新解释字符串 {#reinterpretasstring}
+## reinterpretAsString {#reinterpretasstring}
 
 此函数接受数字、Date或DateTime，并返回一个字符串，其中包含表示主机顺序（小端）的相应值的字节。从末尾删除空字节。例如，UInt32类型值255是一个字节长的字符串。
 
@@ -96,7 +308,7 @@ SELECT toFixedString('foo\0bar', 8) AS s, toStringCutToZero(s) AS s_cut
 
 此函数接受数字、Date或DateTime，并返回包含表示主机顺序（小端）的相应值的字节的FixedString。从末尾删除空字节。例如，UInt32类型值255是一个长度为一个字节的FixedString。
 
-## 演员(x,t) {#type_conversion_function-cast}
+## CAST(x, T) {#type_conversion_function-cast}
 
 将’x’转换为’t’数据类型。还支持语法CAST（x AS t）
 
@@ -133,10 +345,32 @@ SELECT
     │ Nullable(UInt16)                        │
     └─────────────────────────────────────────┘
 
-## 每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每天每 {#function-tointerval}
+## toInterval(Year\|Quarter\|Month\|Week\|Day\|Hour\|Minute\|Second) {#function-tointerval}
 
-将数字类型参数转换为Interval类型（时间区间）。
+把一个数值类型的值转换为[Interval](../../sql-reference/data-types/special-data-types/interval.md)类型的数据。
-Interval类型实际上是非常有用的，您可以使用此类型的数据直接与Date或DateTime执行算术运算。同时，ClickHouse为Interval类型数据的声明提供了更方便的语法。例如：
+
+**语法**
+
+``` sql
+toIntervalSecond(number)
+toIntervalMinute(number)
+toIntervalHour(number)
+toIntervalDay(number)
+toIntervalWeek(number)
+toIntervalMonth(number)
+toIntervalQuarter(number)
+toIntervalYear(number)
+```
+
+**参数**
+
+-   `number` — 正整数，持续的时间。
+
+**返回值**
+
+-   时间的`Interval`值。
+
+**例子**
 
 ``` sql
 WITH
@@ -148,22 +382,257 @@ SELECT
     date + interval_to_week
 ```
 
+``` text
 ┌─plus(date, interval_week)─┬─plus(date, interval_to_week)─┐
 │                2019-01-08 │                   2019-01-08 │
 └───────────────────────────┴──────────────────────────────┘
+```
 
-## parsedatetimebestefort {#type_conversion_functions-parsedatetimebesteffort}
+## parseDateTimeBestEffort {#parsedatetimebesteffort}
 
-将数字类型参数解析为Date或DateTime类型。
+把[String](../../sql-reference/data-types/string.md)类型的时间日期转换为[DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime)数据类型。
-与toDate和toDateTime不同，parseDateTimeBestEffort可以进行更复杂的日期格式。
-有关详细信息，请参阅链接：[复杂日期格式](https://xkcd.com/1179/)。
 
-## parsedatetimebestefortornull {#parsedatetimebesteffortornull}
+该函数可以解析[ISO 8601](https://en.wikipedia.org/wiki/ISO_8601)，[RFC 1123 - 5.2.14 RFC-822 Date and Time Specification](https://tools.ietf.org/html/rfc1123#page-55)或者ClickHouse的一些别的时间日期格式。
 
-与[parsedatetimebestefort](#type_conversion_functions-parsedatetimebesteffort)相同，但它遇到无法处理的日期格式时返回null。
+**语法**
 
-## parsedatetimebestefortorzero {#parsedatetimebesteffortorzero}
+``` sql
+parseDateTimeBestEffort(time_string [, time_zone]);
+```
 
-与[parsedatetimebestefort](#type_conversion_functions-parsedatetimebesteffort)相同，但它遇到无法处理的日期格式时返回零Date或零DateTime。
+**参数**
+
+-   `time_string` — 字符类型的时间和日期。
+-   `time_zone` — 字符类型的时区。
+
+**非标准格式的支持**
+
+-   9位或者10位的数字时间，[unix timestamp](https://en.wikipedia.org/wiki/Unix_time).
+-   时间和日期组成的字符串： `YYYYMMDDhhmmss`, `DD/MM/YYYY hh:mm:ss`, `DD-MM-YY hh:mm`, `YYYY-MM-DD hh:mm:ss`等。
+-   只有日期的字符串： `YYYY`, `YYYYMM`, `YYYY*MM`, `DD/MM/YYYY`, `DD-MM-YY` 等。
+-   只有天和时间： `DD`, `DD hh`, `DD hh:mm`。这种情况下 `YYYY-MM` 默认为 `2000-01`。
+-   包含时间日期以及时区信息： `YYYY-MM-DD hh:mm:ss ±h:mm`等。例如： `2020-12-12 17:36:00 -5:00`。
+
+对于所有的格式来说，这个函数通过全称或者第一个三个字符的月份名称来解析月份，比如：`24/DEC/18`, `24-Dec-18`, `01-September-2018`。
+
+**返回值**
+
+-   `DateTime`类型数据。
+
+**例子**
+
+查询:
+
+``` sql
+SELECT parseDateTimeBestEffort('12/12/2020 12:12:57')
+AS parseDateTimeBestEffort;
+```
+
+结果:
+
+``` text
+┌─parseDateTimeBestEffort─┐
+│     2020-12-12 12:12:57 │
+└─────────────────────────┘
+```
+
+查询:
+
+``` sql
+SELECT parseDateTimeBestEffort('Sat, 18 Aug 2018 07:22:16 GMT', 'Europe/Moscow')
+AS parseDateTimeBestEffort
+```
+
+结果:
+
+``` text
+┌─parseDateTimeBestEffort─┐
+│     2018-08-18 10:22:16 │
+└─────────────────────────┘
+```
+
+查询:
+
+``` sql
+SELECT parseDateTimeBestEffort('1284101485')
+AS parseDateTimeBestEffort
+```
+
+结果:
+
+``` text
+┌─parseDateTimeBestEffort─┐
+│     2015-07-07 12:04:41 │
+└─────────────────────────┘
+```
+
+查询:
+
+``` sql
+SELECT parseDateTimeBestEffort('2018-12-12 10:12:12')
+AS parseDateTimeBestEffort
+```
+
+结果:
+
+``` text
+┌─parseDateTimeBestEffort─┐
+│     2018-12-12 10:12:12 │
+└─────────────────────────┘
+```
+
+查询:
+
+``` sql
+SELECT parseDateTimeBestEffort('10 20:19')
+```
+
+结果:
+
+``` text
+┌─parseDateTimeBestEffort('10 20:19')─┐
+│                 2000-01-10 20:19:00 │
+└─────────────────────────────────────┘
+```
+
+**除此之外**
+
+-   [ISO 8601 announcement by @xkcd](https://xkcd.com/1179/)
+-   [RFC 1123](https://tools.ietf.org/html/rfc1123)
+-   [toDate](#todate)
+-   [toDateTime](#todatetime)
+
+## parseDateTimeBestEffortOrNull {#parsedatetimebesteffortornull}
+
+这个函数和[parseDateTimeBestEffort](#parsedatetimebesteffort)基本一致，除了无法解析返回结果为`NULL`。
+
+## parseDateTimeBestEffortOrZero {#parsedatetimebesteffortorzero}
+
+这个函数和[parseDateTimeBestEffort](#parsedatetimebesteffort)基本一致，除了无法解析返回结果为`0`。
+
+## toLowCardinality {#tolowcardinality}
+
+把输入值转换为[LowCardianlity](../data-types/lowcardinality.md)的相同类型的数据。
+
+如果要把`LowCardinality`类型的数据转换为其他类型，使用[CAST](#type_conversion_function-cast)函数。比如：`CAST(x as String)`。
+
+**语法**
+
+```sql
+toLowCardinality(expr)
+```
+
+**参数**
+
+- `expr` — [表达式](../syntax.md#syntax-expressions)为[支持的数据类型](../data-types/index.md#data_types)的一种。
+
+
+**返回值**
+
+- `expr`的结果。
+
+类型： `LowCardinality(expr_result_type)`
+
+**例子**
+
+查询:
+
+```sql
+SELECT toLowCardinality('1')
+```
+
+结果:
+
+```text
+┌─toLowCardinality('1')─┐
+│ 1                     │
+└───────────────────────┘
+```
+
+
+## toUnixTimestamp64Milli
+## toUnixTimestamp64Micro
+## toUnixTimestamp64Nano
+
+把一个`DateTime64`类型的数据转换为`Int64`类型的数据，结果包含固定亚秒的精度。输入的值是变大还是变低依赖于输入的精度。需要注意的是输出的值是一个UTC的时间戳, 不是同一个时区的`DateTime64`值。
+
+**语法**
+
+``` sql
+toUnixTimestamp64Milli(value)
+```
+
+**参数**
+
+-   `value` — 任何精度的DateTime64类型的数据。
+
+**返回值**
+
+-   `value` `Int64`类型数据。
+
+**例子**
+
+查询:
+
+``` sql
+WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64
+SELECT toUnixTimestamp64Milli(dt64)
+```
+
+结果:
+
+``` text
+┌─toUnixTimestamp64Milli(dt64)─┐
+│                1568650812345 │
+└──────────────────────────────┘
+```
+
+``` sql
+WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64
+SELECT toUnixTimestamp64Nano(dt64)
+```
+
+结果:
+
+``` text
+┌─toUnixTimestamp64Nano(dt64)─┐
+│         1568650812345678000 │
+└─────────────────────────────┘
+```
+
+## fromUnixTimestamp64Milli
+## fromUnixTimestamp64Micro
+## fromUnixTimestamp64Nano
+
+把`Int64`类型的数据转换为`DateTime64`类型的数据，结果包含固定的亚秒精度和可选的时区。 输入的值是变大还是变低依赖于输入的精度。需要注意的是输入的值是一个UTC的时间戳, 不是一个包含时区的时间戳。
+
+
+**语法**
+
+``` sql
+fromUnixTimestamp64Milli(value [, ti])
+```
+
+**参数**
+
+-   `value` — `Int64`类型的数据，可以是任意精度。
+-   `timezone` — `String`类型的时区
+
+**返回值**
+
+-   `value` DateTime64`类型的数据。
+
+**例子**
+
+``` sql
+WITH CAST(1234567891011, 'Int64') AS i64
+SELECT fromUnixTimestamp64Milli(i64, 'UTC')
+```
+
+``` text
+┌─fromUnixTimestamp64Milli(i64, 'UTC')─┐
+│              2009-02-13 23:31:31.011 │
+└──────────────────────────────────────┘
+```
 
 [来源文章](https://clickhouse.tech/docs/en/query_language/functions/type_conversion_functions/) <!--hide-->

programs/CMakeLists.txt

@@ -207,7 +207,7 @@ if (TARGET clickhouse-server AND TARGET copy-headers)
 endif ()
 
 if (ENABLE_TESTS AND USE_GTEST)
-    set (CLICKHOUSE_ALL_TESTS_TARGETS local_date_time_comparison unit_tests_libcommon unit_tests_dbms hashing_write_buffer hashing_read_buffer in_join_subqueries_preprocessor)
+    set (CLICKHOUSE_UNIT_TESTS_TARGETS unit_tests_libcommon unit_tests_dbms)
-    add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_ALL_TESTS_TARGETS})
+    add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_UNIT_TESTS_TARGETS})
     add_dependencies(clickhouse-bundle clickhouse-tests)
 endif()

programs/benchmark/CMakeLists.txt

@@ -1,5 +1,12 @@
-set(CLICKHOUSE_BENCHMARK_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Benchmark.cpp)
+set (CLICKHOUSE_BENCHMARK_SOURCES Benchmark.cpp)
-set(CLICKHOUSE_BENCHMARK_LINK PRIVATE dbms clickhouse_aggregate_functions clickhouse_common_config ${Boost_PROGRAM_OPTIONS_LIBRARY})
+
+set (CLICKHOUSE_BENCHMARK_LINK
+    PRIVATE
+        boost::program_options
+        clickhouse_aggregate_functions
+        clickhouse_common_config
+        dbms
+)
 
 clickhouse_program_add(benchmark)
 

programs/client/CMakeLists.txt

@@ -1,10 +1,19 @@
 set (CLICKHOUSE_CLIENT_SOURCES
-    ${CMAKE_CURRENT_SOURCE_DIR}/Client.cpp
+    Client.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/ConnectionParameters.cpp
+    ConnectionParameters.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/Suggest.cpp
+    Suggest.cpp
 )
 
-set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${Boost_PROGRAM_OPTIONS_LIBRARY})
+set (CLICKHOUSE_CLIENT_LINK
+    PRIVATE
+        boost::program_options
+        clickhouse_aggregate_functions
+        clickhouse_common_config
+        clickhouse_common_io
+        clickhouse_functions
+        clickhouse_parsers
+        string_utils
+)
 
 # Always use internal readpassphrase
 add_subdirectory(readpassphrase)

programs/client/Client.cpp

@@ -75,6 +75,7 @@
 #include <Storages/ColumnsDescription.h>
 #include <common/argsToConfig.h>
 #include <Common/TerminalSize.h>
+#include <Common/UTF8Helpers.h>
 
 #if !defined(ARCADIA_BUILD)
 #    include <Common/config_version.h>
@@ -357,6 +358,78 @@ private:
         return false;
     }
 
+#if USE_REPLXX
+    static void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors)
+    {
+        using namespace replxx;
+
+        static const std::unordered_map<TokenType, Replxx::Color> token_to_color =
+        {
+            { TokenType::Whitespace, Replxx::Color::DEFAULT },
+            { TokenType::Comment, Replxx::Color::GRAY },
+            { TokenType::BareWord, Replxx::Color::DEFAULT },
+            { TokenType::Number, Replxx::Color::GREEN },
+            { TokenType::StringLiteral, Replxx::Color::CYAN },
+            { TokenType::QuotedIdentifier, Replxx::Color::MAGENTA },
+            { TokenType::OpeningRoundBracket, Replxx::Color::BROWN },
+            { TokenType::ClosingRoundBracket, Replxx::Color::BROWN },
+            { TokenType::OpeningSquareBracket, Replxx::Color::BROWN },
+            { TokenType::ClosingSquareBracket, Replxx::Color::BROWN },
+            { TokenType::OpeningCurlyBrace, Replxx::Color::INTENSE },
+            { TokenType::ClosingCurlyBrace, Replxx::Color::INTENSE },
+
+            { TokenType::Comma, Replxx::Color::INTENSE },
+            { TokenType::Semicolon, Replxx::Color::INTENSE },
+            { TokenType::Dot, Replxx::Color::INTENSE },
+            { TokenType::Asterisk, Replxx::Color::INTENSE },
+            { TokenType::Plus, Replxx::Color::INTENSE },
+            { TokenType::Minus, Replxx::Color::INTENSE },
+            { TokenType::Slash, Replxx::Color::INTENSE },
+            { TokenType::Percent, Replxx::Color::INTENSE },
+            { TokenType::Arrow, Replxx::Color::INTENSE },
+            { TokenType::QuestionMark, Replxx::Color::INTENSE },
+            { TokenType::Colon, Replxx::Color::INTENSE },
+            { TokenType::Equals, Replxx::Color::INTENSE },
+            { TokenType::NotEquals, Replxx::Color::INTENSE },
+            { TokenType::Less, Replxx::Color::INTENSE },
+            { TokenType::Greater, Replxx::Color::INTENSE },
+            { TokenType::LessOrEquals, Replxx::Color::INTENSE },
+            { TokenType::GreaterOrEquals, Replxx::Color::INTENSE },
+            { TokenType::Concatenation, Replxx::Color::INTENSE },
+            { TokenType::At, Replxx::Color::INTENSE },
+
+            { TokenType::EndOfStream, Replxx::Color::DEFAULT },
+
+            { TokenType::Error, Replxx::Color::RED },
+            { TokenType::ErrorMultilineCommentIsNotClosed, Replxx::Color::RED },
+            { TokenType::ErrorSingleQuoteIsNotClosed, Replxx::Color::RED },
+            { TokenType::ErrorDoubleQuoteIsNotClosed, Replxx::Color::RED },
+            { TokenType::ErrorSinglePipeMark, Replxx::Color::RED },
+            { TokenType::ErrorWrongNumber, Replxx::Color::RED },
+            { TokenType::ErrorMaxQuerySizeExceeded, Replxx::Color::RED }
+        };
+
+        const Replxx::Color unknown_token_color = Replxx::Color::RED;
+
+        Lexer lexer(query.data(), query.data() + query.size());
+        size_t pos = 0;
+
+        for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken())
+        {
+            size_t utf8_len = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(token.begin), token.size());
+            for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index)
+            {
+                if (token_to_color.find(token.type) != token_to_color.end())
+                    colors[pos + code_point_index] = token_to_color.at(token.type);
+                else
+                    colors[pos + code_point_index] = unknown_token_color;
+            }
+
+            pos += utf8_len;
+        }
+    }
+#endif
+
     int mainImpl()
     {
         UseSSL use_ssl;
@@ -502,7 +575,18 @@ private:
             LineReader::Patterns query_delimiters = {";", "\\G"};
 
 #if USE_REPLXX
-            ReplxxLineReader lr(Suggest::instance(), history_file, config().has("multiline"), query_extenders, query_delimiters);
+            replxx::Replxx::highlighter_callback_t highlight_callback{};
+            if (config().getBool("highlight"))
+                highlight_callback = highlight;
+
+            ReplxxLineReader lr(
+                Suggest::instance(),
+                history_file,
+                config().has("multiline"),
+                query_extenders,
+                query_delimiters,
+                highlight_callback);
+
 #elif defined(USE_READLINE) && USE_READLINE
             ReadlineLineReader lr(Suggest::instance(), history_file, config().has("multiline"), query_extenders, query_delimiters);
 #else
@@ -1766,6 +1850,7 @@ public:
             ("echo", "in batch mode, print query before execution")
             ("max_client_network_bandwidth", po::value<int>(), "the maximum speed of data exchange over the network for the client in bytes per second.")
             ("compression", po::value<bool>(), "enable or disable compression")
+            ("highlight", po::value<bool>()->default_value(true), "enable or disable basic syntax highlight in interactive command line")
             ("log-level", po::value<std::string>(), "client log level")
             ("server_logs_file", po::value<std::string>(), "put server logs into specified file")
         ;
@@ -1912,6 +1997,8 @@ public:
             config().setBool("disable_suggestion", true);
         if (options.count("suggestion_limit"))
             config().setInt("suggestion_limit", options["suggestion_limit"].as<int>());
+        if (options.count("highlight"))
+            config().setBool("highlight", options["highlight"].as<bool>());
 
         argsToConfig(common_arguments, config(), 100);
 

programs/client/Suggest.cpp

@@ -114,6 +114,8 @@ void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeo
             << " UNION ALL "
             "SELECT DISTINCT name FROM system.tables LIMIT " << limit_str
             << " UNION ALL "
+            "SELECT DISTINCT name FROM system.dictionaries LIMIT " << limit_str
+            << " UNION ALL "
             "SELECT DISTINCT name FROM system.columns LIMIT " << limit_str;
     }
 

programs/compressor/CMakeLists.txt

@@ -1,7 +1,12 @@
 # Also in utils
 
-set(CLICKHOUSE_COMPRESSOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Compressor.cpp)
+set (CLICKHOUSE_COMPRESSOR_SOURCES Compressor.cpp)
-set(CLICKHOUSE_COMPRESSOR_LINK PRIVATE dbms clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY})
+
-#set(CLICKHOUSE_COMPRESSOR_INCLUDE SYSTEM PRIVATE ...)
+set (CLICKHOUSE_COMPRESSOR_LINK
+    PRIVATE
+        boost::program_options
+        clickhouse_parsers
+        dbms
+)
 
 clickhouse_program_add(compressor)

programs/extract-from-config/CMakeLists.txt

@@ -1,5 +1,11 @@
-set(CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ExtractFromConfig.cpp)
+set (CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES ExtractFromConfig.cpp)
-set(CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK PRIVATE clickhouse_common_config clickhouse_common_io clickhouse_common_zookeeper ${Boost_PROGRAM_OPTIONS_LIBRARY})
+
-#set(CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE SYSTEM PRIVATE ...)
+set (CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK
+    PRIVATE
+        boost::program_options
+        clickhouse_common_config
+        clickhouse_common_io
+        clickhouse_common_zookeeper
+)
 
 clickhouse_program_add(extract-from-config)

programs/format/CMakeLists.txt

@@ -1,5 +1,11 @@
-set(CLICKHOUSE_FORMAT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Format.cpp)
+set (CLICKHOUSE_FORMAT_SOURCES Format.cpp)
-set(CLICKHOUSE_FORMAT_LINK PRIVATE dbms clickhouse_common_io clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY})
+
-#set(CLICKHOUSE_FORMAT_INCLUDE SYSTEM PRIVATE ...)
+set (CLICKHOUSE_FORMAT_LINK
+    PRIVATE
+        boost::program_options
+        clickhouse_common_io
+        clickhouse_parsers
+        dbms
+)
 
 clickhouse_program_add(format)

programs/local/CMakeLists.txt

@@ -1,6 +1,17 @@
-set(CLICKHOUSE_LOCAL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/LocalServer.cpp)
+set (CLICKHOUSE_LOCAL_SOURCES LocalServer.cpp)
-set(CLICKHOUSE_LOCAL_LINK PRIVATE clickhouse_storages_system clickhouse_dictionaries clickhouse_common_config clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions clickhouse_parsers clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
+
-#set(CLICKHOUSE_LOCAL_INCLUDE SYSTEM PRIVATE ...)
+set (CLICKHOUSE_LOCAL_LINK
+    PRIVATE
+        boost::program_options
+        clickhouse_aggregate_functions
+        clickhouse_common_config
+        clickhouse_common_io
+        clickhouse_dictionaries
+        clickhouse_functions
+        clickhouse_parsers
+        clickhouse_storages_system
+        clickhouse_table_functions
+)
 
 clickhouse_program_add(local)
 

programs/obfuscator/CMakeLists.txt

@@ -1,5 +1,9 @@
-set(CLICKHOUSE_OBFUSCATOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Obfuscator.cpp)
+set (CLICKHOUSE_OBFUSCATOR_SOURCES Obfuscator.cpp)
-set(CLICKHOUSE_OBFUSCATOR_LINK PRIVATE dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
+
-#set(CLICKHOUSE_OBFUSCATOR_INCLUDE SYSTEM PRIVATE ...)
+set (CLICKHOUSE_OBFUSCATOR_LINK
+    PRIVATE
+        boost::program_options
+        dbms
+)
 
 clickhouse_program_add(obfuscator)

programs/server/Server.cpp

@@ -17,6 +17,7 @@
 #include <common/phdr_cache.h>
 #include <common/ErrorHandlers.h>
 #include <common/getMemoryAmount.h>
+#include <common/errnoToString.h>
 #include <common/coverage.h>
 #include <Common/ClickHouseRevision.h>
 #include <Common/DNSResolver.h>
@@ -125,6 +126,7 @@ namespace ErrorCodes
     extern const int FAILED_TO_GETPWUID;
     extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA;
     extern const int NETWORK_ERROR;
+    extern const int UNKNOWN_ELEMENT_IN_CONFIG;
 }
 
 
@@ -210,6 +212,52 @@ void Server::defineOptions(Poco::Util::OptionSet & options)
     BaseDaemon::defineOptions(options);
 }
 
+
+/// Check that there is no user-level settings at the top level in config.
+/// This is a common source of mistake (user don't know where to write user-level setting).
+void checkForUserSettingsAtTopLevel(const Poco::Util::AbstractConfiguration & config, const std::string & path)
+{
+    if (config.getBool("skip_check_for_incorrect_settings", false))
+        return;
+
+    Settings settings;
+    for (const auto & setting : settings)
+    {
+        std::string name = setting.getName().toString();
+        if (config.has(name))
+        {
+            throw Exception(fmt::format("A setting '{}' appeared at top level in config {}."
+                " But it is user-level setting that should be located in users.xml inside <profiles> section for specific profile."
+                " You can add it to <profiles><default> if you want to change default value of this setting."
+                " You can also disable the check - specify <skip_check_for_incorrect_settings>1</skip_check_for_incorrect_settings>"
+                " in the main configuration file.",
+                name, path),
+                ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
+        }
+    }
+}
+
+void checkForUsersNotInMainConfig(
+    const Poco::Util::AbstractConfiguration & config,
+    const std::string & config_path,
+    const std::string & users_config_path,
+    Poco::Logger * log)
+{
+    if (config.getBool("skip_check_for_incorrect_settings", false))
+        return;
+
+    if (config.has("users") || config.has("profiles") || config.has("quotas"))
+    {
+        /// We cannot throw exception here, because we have support for obsolete 'conf.d' directory
+        /// (that does not correspond to config.d or users.d) but substitute configuration to both of them.
+
+        LOG_ERROR(log, "The <users>, <profiles> and <quotas> elements should be located in users config file: {} not in main config {}."
+            " Also note that you should place configuration changes to the appropriate *.d directory like 'users.d'.",
+            users_config_path, config_path);
+    }
+}
+
+
 int Server::main(const std::vector<std::string> & /*args*/)
 {
     Poco::Logger * log = &logger();
@@ -269,6 +317,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
         config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
     }
 
+    checkForUserSettingsAtTopLevel(config(), config_path);
+
     const auto memory_amount = getMemoryAmount();
 
 #if defined(OS_LINUX)
@@ -473,13 +523,16 @@ int Server::main(const std::vector<std::string> & /*args*/)
         SensitiveDataMasker::setInstance(std::make_unique<SensitiveDataMasker>(config(), "query_masking_rules"));
     }
 
-    auto main_config_reloader = std::make_unique<ConfigReloader>(config_path,
+    auto main_config_reloader = std::make_unique<ConfigReloader>(
+        config_path,
         include_from_path,
         config().getString("path", ""),
         std::move(main_config_zk_node_cache),
         main_config_zk_changed_event,
         [&](ConfigurationPtr config)
         {
+            checkForUserSettingsAtTopLevel(*config, config_path);
+
             // FIXME logging-related things need synchronization -- see the 'Logger * log' saved
             // in a lot of places. For now, disable updating log configuration without server restart.
             //setTextLog(global_context->getTextLog());
@@ -508,12 +561,21 @@ int Server::main(const std::vector<std::string> & /*args*/)
         if (Poco::File(config_dir + users_config_path).exists())
             users_config_path = config_dir + users_config_path;
     }
-    auto users_config_reloader = std::make_unique<ConfigReloader>(users_config_path,
+
+    if (users_config_path != config_path)
+        checkForUsersNotInMainConfig(config(), config_path, users_config_path, log);
+
+    auto users_config_reloader = std::make_unique<ConfigReloader>(
+        users_config_path,
         include_from_path,
         config().getString("path", ""),
         zkutil::ZooKeeperNodeCache([&] { return global_context->getZooKeeper(); }),
         std::make_shared<Poco::Event>(),
-        [&](ConfigurationPtr config) { global_context->setUsersConfig(config); },
+        [&](ConfigurationPtr config)
+        {
+            global_context->setUsersConfig(config);
+            checkForUserSettingsAtTopLevel(*config, users_config_path);
+        },
         /* already_loaded = */ false);
 
     /// Reload config in SYSTEM RELOAD CONFIG query.

programs/server/config.xml

@@ -1,6 +1,9 @@
 <?xml version="1.0"?>
 <!--
   NOTE: User and query level settings are set up in "users.xml" file.
+  If you have accidentially specified user-level settings here, server won't start.
+  You can either move the settings to the right place inside "users.xml" file
+   or add <skip_check_for_incorrect_settings>1</skip_check_for_incorrect_settings> here.
 -->
 <yandex>
     <logger>

src/Access/AllowedClientHosts.cpp

@@ -8,6 +8,7 @@
 #include <ext/scope_guard.h>
 #include <boost/algorithm/string/replace.hpp>
 #include <ifaddrs.h>
+#include <Common/DNSResolver.h>
 
 
 namespace DB
@@ -44,66 +45,22 @@ namespace
         return IPSubnet(toIPv6(subnet.getPrefix()), subnet.getMask());
     }
 
-
-    /// Helper function for isAddressOfHost().
-    bool isAddressOfHostImpl(const IPAddress & address, const String & host)
-    {
-        IPAddress addr_v6 = toIPv6(address);
-
-        /// Resolve by hand, because Poco don't use AI_ALL flag but we need it.
-        addrinfo * ai_begin = nullptr;
-        SCOPE_EXIT(
-        {
-            if (ai_begin)
-                freeaddrinfo(ai_begin);
-        });
-
-        addrinfo hints;
-        memset(&hints, 0, sizeof(hints));
-        hints.ai_family = AF_UNSPEC;
-        hints.ai_flags |= AI_V4MAPPED | AI_ALL;
-
-        int err = getaddrinfo(host.c_str(), nullptr, &hints, &ai_begin);
-        if (err)
-            throw Exception("Cannot getaddrinfo(" + host + "): " + gai_strerror(err), ErrorCodes::DNS_ERROR);
-
-        for (const addrinfo * ai = ai_begin; ai; ai = ai->ai_next)
-        {
-            if (ai->ai_addrlen && ai->ai_addr)
-            {
-                if (ai->ai_family == AF_INET)
-                {
-                    const auto & sin = *reinterpret_cast<const sockaddr_in *>(ai->ai_addr);
-                    if (addr_v6 == toIPv6(IPAddress(&sin.sin_addr, sizeof(sin.sin_addr))))
-                    {
-                        return true;
-                    }
-                }
-                else if (ai->ai_family == AF_INET6)
-                {
-                    const auto & sin = *reinterpret_cast<const sockaddr_in6*>(ai->ai_addr);
-                    if (addr_v6 == IPAddress(&sin.sin6_addr, sizeof(sin.sin6_addr), sin.sin6_scope_id))
-                    {
-                        return true;
-                    }
-                }
-            }
-        }
-
-        return false;
-    }
-
-    auto & getIsAddressOfHostCache()
-    {
-        static SimpleCache<decltype(isAddressOfHostImpl), isAddressOfHostImpl> cache;
-        return cache;
-    }
-
     /// Whether a specified address is one of the addresses of a specified host.
     bool isAddressOfHost(const IPAddress & address, const String & host)
     {
-        /// We need to cache DNS requests.
+        IPAddress addr_v6 = toIPv6(address);
-        return getIsAddressOfHostCache()(address, host);
+
+        auto host_addresses = DNSResolver::instance().resolveHostAll(host);
+
+        for (const auto & addr : host_addresses)
+        {
+            if (addr.family() == IPAddress::Family::IPv4 && addr_v6 == toIPv6(addr))
+                return true;
+            else if (addr.family() == IPAddress::Family::IPv6 && addr_v6 == addr)
+                return true;
+        }
+
+        return false;
     }
 
     /// Helper function for isAddressOfLocalhost().
@@ -147,16 +104,10 @@ namespace
         return boost::range::find(local_addresses, toIPv6(address)) != local_addresses.end();
     }
 
-    /// Helper function for getHostByAddress().
+    /// Returns the host name by its address.
-    String getHostByAddressImpl(const IPAddress & address)
+    String getHostByAddress(const IPAddress & address)
     {
-        Poco::Net::SocketAddress sock_addr(address, 0);
+        String host = DNSResolver::instance().reverseResolve(address);
-
-        /// Resolve by hand, because Poco library doesn't have such functionality.
-        char host[1024];
-        int err = getnameinfo(sock_addr.addr(), sock_addr.length(), host, sizeof(host), nullptr, 0, NI_NAMEREQD);
-        if (err)
-            throw Exception("Cannot getnameinfo(" + address.toString() + "): " + gai_strerror(err), ErrorCodes::DNS_ERROR);
 
         /// Check that PTR record is resolved back to client address
         if (!isAddressOfHost(address, host))
@@ -165,19 +116,6 @@ namespace
         return host;
     }
 
-    auto & getHostByAddressCache()
-    {
-        static SimpleCache<decltype(getHostByAddressImpl), &getHostByAddressImpl> cache;
-        return cache;
-    }
-
-    /// Returns the host name by its address.
-    String getHostByAddress(const IPAddress & address)
-    {
-        /// We need to cache DNS requests.
-        return getHostByAddressCache()(address);
-    }
-
 
     void parseLikePatternIfIPSubnet(const String & pattern, IPSubnet & subnet, IPAddress::Family address_family)
     {
@@ -376,10 +314,4 @@ bool AllowedClientHosts::contains(const IPAddress & client_address) const
     return false;
 }
 
-void AllowedClientHosts::dropDNSCaches()
-{
-    getIsAddressOfHostCache().drop();
-    getHostByAddressCache().drop();
-}
-
 }

src/Access/AllowedClientHosts.h

@@ -114,8 +114,6 @@ public:
     friend bool operator ==(const AllowedClientHosts & lhs, const AllowedClientHosts & rhs);
     friend bool operator !=(const AllowedClientHosts & lhs, const AllowedClientHosts & rhs) { return !(lhs == rhs); }
 
-    static void dropDNSCaches();
-
 private:
     std::vector<IPAddress> addresses;
     std::vector<IPSubnet> subnets;

src/Access/IAccessStorage.cpp

@@ -12,10 +12,10 @@ namespace DB
 {
 namespace ErrorCodes
 {
-    extern const int BAD_CAST;
     extern const int ACCESS_ENTITY_ALREADY_EXISTS;
     extern const int ACCESS_ENTITY_NOT_FOUND;
     extern const int ACCESS_STORAGE_READONLY;
+    extern const int LOGICAL_ERROR;
 }
 
 
@@ -403,7 +403,7 @@ void IAccessStorage::throwBadCast(const UUID & id, EntityType type, const String
 {
     throw Exception(
         "ID {" + toString(id) + "}: " + outputEntityTypeAndName(type, name) + " expected to be of type " + toString(required_type),
-        ErrorCodes::BAD_CAST);
+        ErrorCodes::LOGICAL_ERROR);
 }
 
 

src/AggregateFunctions/AggregateFunctionMLMethod.h

@@ -15,7 +15,6 @@ namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-    extern const int BAD_CAST;
 }
 
 /**
@@ -381,7 +380,7 @@ public:
         auto * column = typeid_cast<ColumnFloat64 *>(&to);
         if (!column)
             throw Exception("Cast of column of predictions is incorrect. getReturnTypeToPredict must return same value as it is casted to",
-                            ErrorCodes::BAD_CAST);
+                            ErrorCodes::LOGICAL_ERROR);
 
         this->data(place).predict(column->getData(), block, offset, limit, arguments, context);
     }

src/AggregateFunctions/IAggregateFunction.h

@@ -150,6 +150,8 @@ public:
     virtual void addBatchSinglePlaceNotNull(
         size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena * arena) const = 0;
 
+    virtual void addBatchSinglePlaceFromInterval(size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena) const = 0;
+
     /** In addition to addBatch, this method collects multiple rows of arguments into array "places"
       *  as long as they are between offsets[i-1] and offsets[i]. This is used for arrayReduce and
       *  -Array combinator. It might also be used generally to break data dependency when array
@@ -214,6 +216,12 @@ public:
                 static_cast<const Derived *>(this)->add(place, columns, i, arena);
     }
 
+    void addBatchSinglePlaceFromInterval(size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena) const override
+    {
+        for (size_t i = batch_begin; i < batch_end; ++i)
+            static_cast<const Derived *>(this)->add(place, columns, i, arena);
+    }
+
     void addBatchArray(
         size_t batch_size, AggregateDataPtr * places, size_t place_offset, const IColumn ** columns, const UInt64 * offsets, Arena * arena)
         const override

src/AggregateFunctions/parseAggregateFunctionParameters.cpp

@@ -27,8 +27,12 @@ Array getAggregateFunctionParametersArray(const ASTPtr & expression_list, const
         const auto * literal = parameters[i]->as<ASTLiteral>();
         if (!literal)
         {
-            throw Exception("Parameters to aggregate functions must be literals" + (error_context.empty() ? "" : " (in " + error_context +")"),
+            throw Exception(
-                        ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS);
+                ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS,
+                "Parameters to aggregate functions must be literals. "
+                "Got parameter '{}'{}",
+                parameters[i]->formatForErrorMessage(),
+                (error_context.empty() ? "" : " (in " + error_context +")"));
         }
 
         params_row[i] = literal->value;

src/CMakeLists.txt

@@ -161,12 +161,12 @@ add_object_library(clickhouse_processors_merges_algorithms Processors/Merges/Alg
 
 if (MAKE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES)
     add_library (dbms STATIC ${dbms_headers} ${dbms_sources})
-    target_link_libraries (dbms PRIVATE jemalloc)
+    target_link_libraries (dbms PRIVATE jemalloc libdivide)
     set (all_modules dbms)
 else()
     add_library (dbms SHARED ${dbms_headers} ${dbms_sources})
     target_link_libraries (dbms PUBLIC ${all_modules})
-    target_link_libraries (clickhouse_interpreters PRIVATE jemalloc)
+    target_link_libraries (clickhouse_interpreters PRIVATE jemalloc libdivide)
     list (APPEND all_modules dbms)
     # force all split libs to be linked
     set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-as-needed")
@@ -184,6 +184,9 @@ macro (dbms_target_link_libraries)
     endforeach ()
 endmacro ()
 
+dbms_target_include_directories (PUBLIC ${ClickHouse_SOURCE_DIR}/src ${ClickHouse_BINARY_DIR}/src)
+target_include_directories (clickhouse_common_io PUBLIC ${ClickHouse_SOURCE_DIR}/src ${ClickHouse_BINARY_DIR}/src)
+
 if (USE_EMBEDDED_COMPILER)
     dbms_target_link_libraries (PRIVATE ${REQUIRED_LLVM_LIBRARIES})
     dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
@@ -240,8 +243,8 @@ target_link_libraries(clickhouse_common_io
             ${EXECINFO_LIBRARIES}
             cpuid
         PUBLIC
-            ${Boost_PROGRAM_OPTIONS_LIBRARY}
+            boost::program_options
-            ${Boost_SYSTEM_LIBRARY}
+            boost::system
             ${CITYHASH_LIBRARIES}
             ${ZLIB_LIBRARIES}
             pcg_random
@@ -264,18 +267,18 @@ endif()
 dbms_target_link_libraries (
     PRIVATE
         ${BTRIE_LIBRARIES}
-        ${Boost_PROGRAM_OPTIONS_LIBRARY}
+        boost::filesystem
-        ${Boost_FILESYSTEM_LIBRARY}
+        boost::program_options
-        ${LZ4_LIBRARY}
-        clickhouse_parsers
         clickhouse_common_config
         clickhouse_common_zookeeper
         clickhouse_dictionaries_embedded
+        clickhouse_parsers
+        lz4
         Poco::JSON
         string_utils
     PUBLIC
-        ${Boost_SYSTEM_LIBRARY}
         ${MYSQLXX_LIBRARY}
+        boost::system
         clickhouse_common_io
 )
 
@@ -284,10 +287,6 @@ dbms_target_include_directories(PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/Core/include)
 
 dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR})
 
-if (NOT USE_INTERNAL_LZ4_LIBRARY AND LZ4_INCLUDE_DIR)
-    dbms_target_include_directories(SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR})
-endif ()
-
 if (ZSTD_LIBRARY)
     dbms_target_link_libraries(PRIVATE ${ZSTD_LIBRARY})
     if (NOT USE_INTERNAL_ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR)
@@ -295,10 +294,6 @@ if (ZSTD_LIBRARY)
     endif ()
 endif()
 
-if (NOT USE_INTERNAL_BOOST_LIBRARY)
-    target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
-endif ()
-
 if (USE_ICU)
     dbms_target_link_libraries (PRIVATE ${ICU_LIBRARIES})
     dbms_target_include_directories (SYSTEM PRIVATE ${ICU_INCLUDE_DIRS})
@@ -329,8 +324,6 @@ if (USE_LDAP)
     dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${OPENLDAP_INCLUDE_DIR})
     dbms_target_link_libraries (PRIVATE ${OPENLDAP_LIBRARIES})
 endif ()
-
-dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
 dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR})
 
 if (USE_PROTOBUF)
@@ -364,9 +357,6 @@ if (USE_CASSANDRA)
     dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${CASS_INCLUDE_DIR})
 endif()
 
-dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR})
-target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR})
-
 target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR})
 
 target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${MSGPACK_INCLUDE_DIR})

src/Columns/ColumnAggregateFunction.cpp

@@ -6,6 +6,7 @@
 #include <IO/WriteBufferFromArena.h>
 #include <IO/WriteBufferFromString.h>
 #include <IO/Operators.h>
+#include <Common/FieldVisitors.h>
 #include <Common/SipHash.h>
 #include <Common/AlignedBuffer.h>
 #include <Common/typeid_cast.h>
@@ -27,6 +28,51 @@ namespace ErrorCodes
 }
 
 
+static std::string getTypeString(const AggregateFunctionPtr & func)
+{
+    WriteBufferFromOwnString stream;
+    stream << "AggregateFunction(" << func->getName();
+    const auto & parameters = func->getParameters();
+    const auto & argument_types = func->getArgumentTypes();
+
+    if (!parameters.empty())
+    {
+        stream << '(';
+        for (size_t i = 0; i < parameters.size(); ++i)
+        {
+            if (i)
+                stream << ", ";
+            stream << applyVisitor(FieldVisitorToString(), parameters[i]);
+        }
+        stream << ')';
+    }
+
+    for (const auto & argument_type : argument_types)
+        stream << ", " << argument_type->getName();
+
+    stream << ')';
+    return stream.str();
+}
+
+
+ColumnAggregateFunction::ColumnAggregateFunction(const AggregateFunctionPtr & func_)
+    : func(func_), type_string(getTypeString(func))
+{
+}
+
+ColumnAggregateFunction::ColumnAggregateFunction(const AggregateFunctionPtr & func_, const ConstArenas & arenas_)
+    : foreign_arenas(arenas_), func(func_), type_string(getTypeString(func))
+{
+
+}
+
+void ColumnAggregateFunction::set(const AggregateFunctionPtr & func_)
+{
+    func = func_;
+    type_string = getTypeString(func);
+}
+
+
 ColumnAggregateFunction::~ColumnAggregateFunction()
 {
     if (!func->hasTrivialDestructor() && !src)
@@ -336,15 +382,10 @@ MutableColumnPtr ColumnAggregateFunction::cloneEmpty() const
     return create(func);
 }
 
-String ColumnAggregateFunction::getTypeString() const
-{
-    return DataTypeAggregateFunction(func, func->getArgumentTypes(), func->getParameters()).getName();
-}
-
 Field ColumnAggregateFunction::operator[](size_t n) const
 {
     Field field = AggregateFunctionStateData();
-    field.get<AggregateFunctionStateData &>().name = getTypeString();
+    field.get<AggregateFunctionStateData &>().name = type_string;
     {
         WriteBufferFromString buffer(field.get<AggregateFunctionStateData &>().data);
         func->serialize(data[n], buffer);
@@ -355,7 +396,7 @@ Field ColumnAggregateFunction::operator[](size_t n) const
 void ColumnAggregateFunction::get(size_t n, Field & res) const
 {
     res = AggregateFunctionStateData();
-    res.get<AggregateFunctionStateData &>().name = getTypeString();
+    res.get<AggregateFunctionStateData &>().name = type_string;
     {
         WriteBufferFromString buffer(res.get<AggregateFunctionStateData &>().data);
         func->serialize(data[n], buffer);
@@ -425,8 +466,6 @@ static void pushBackAndCreateState(ColumnAggregateFunction::Container & data, Ar
 
 void ColumnAggregateFunction::insert(const Field & x)
 {
-    String type_string = getTypeString();
-
     if (x.getType() != Field::Types::AggregateFunctionState)
         throw Exception(String("Inserting field of type ") + x.getTypeName() + " into ColumnAggregateFunction. "
                         "Expected " + Field::Types::toString(Field::Types::AggregateFunctionState), ErrorCodes::LOGICAL_ERROR);
@@ -564,7 +603,7 @@ void ColumnAggregateFunction::getExtremes(Field & min, Field & max) const
     AggregateDataPtr place = place_buffer.data();
 
     AggregateFunctionStateData serialized;
-    serialized.name = getTypeString();
+    serialized.name = type_string;
 
     func->create(place);
     try

src/Columns/ColumnAggregateFunction.h

@@ -74,6 +74,9 @@ private:
     /// Array of pointers to aggregation states, that are placed in arenas.
     Container data;
 
+    /// Name of the type to distinguish different aggregation states.
+    String type_string;
+
     ColumnAggregateFunction() {}
 
     /// Create a new column that has another column as a source.
@@ -84,29 +87,17 @@ private:
     ///  but ownership of different elements cannot be mixed by different columns.
     void ensureOwnership();
 
-    ColumnAggregateFunction(const AggregateFunctionPtr & func_)
+    ColumnAggregateFunction(const AggregateFunctionPtr & func_);
-        : func(func_)
-    {
-    }
 
     ColumnAggregateFunction(const AggregateFunctionPtr & func_,
-                            const ConstArenas & arenas_)
+                            const ConstArenas & arenas_);
-        : foreign_arenas(arenas_), func(func_)
-    {
-    }
-
 
     ColumnAggregateFunction(const ColumnAggregateFunction & src_);
 
-    String getTypeString() const;
-
 public:
     ~ColumnAggregateFunction() override;
 
-    void set(const AggregateFunctionPtr & func_)
+    void set(const AggregateFunctionPtr & func_);
-    {
-        func = func_;
-    }
 
     AggregateFunctionPtr getAggregateFunction() { return func; }
     AggregateFunctionPtr getAggregateFunction() const { return func; }
@@ -121,6 +112,7 @@ public:
 
     std::string getName() const override { return "AggregateFunction(" + func->getName() + ")"; }
     const char * getFamilyName() const override { return "AggregateFunction"; }
+    TypeIndex getDataType() const override { return TypeIndex::AggregateFunction; }
 
     MutableColumnPtr predictValues(Block & block, const ColumnNumbers & arguments, const Context & context) const;
 

src/Columns/ColumnArray.h

@@ -52,6 +52,7 @@ public:
 
     std::string getName() const override;
     const char * getFamilyName() const override { return "Array"; }
+    TypeIndex getDataType() const override { return TypeIndex::Array; }
     MutableColumnPtr cloneResized(size_t size) const override;
     size_t size() const override;
     Field operator[](size_t n) const override;