Merge branch 'master' of github.com:yandex/ClickHouse

.gitmodules

@@ -46,3 +46,6 @@
 [submodule "contrib/unixodbc"]
 	path = contrib/unixodbc
 	url = https://github.com/ClickHouse-Extras/UnixODBC.git
+[submodule "contrib/base64"]
+	path = contrib/base64
+	url = https://github.com/aklomp/base64.git

CHANGELOG.md

@@ -1,10 +1,15 @@
+## ClickHouse release 18.14.15, 2018-11-21
+
+### Bug fixes:
+* The size of memory chunk was overestimated while deserializing the column of type `Array(String)` that leads to "Memory limit exceeded" errors. The issue appeared in version 18.12.13. [#3589](https://github.com/yandex/ClickHouse/issues/3589)
+
 ## ClickHouse release 18.14.14, 2018-11-20
 
 ### Bug fixes:
-* Fixed `ON CLUSTER` queries when cluster configured as secure (flag `<secure>`). [#3599]
+* Fixed `ON CLUSTER` queries when cluster configured as secure (flag `<secure>`). [#3599](https://github.com/yandex/ClickHouse/pull/3599)
 
 ### Build changes:
-* Fixed problems (llvm-7 from system, macos) [#3582]
+* Fixed problems (llvm-7 from system, macos) [#3582](https://github.com/yandex/ClickHouse/pull/3582)
 
 ## ClickHouse release 18.14.11, 2018-10-29
 

CHANGELOG_RU.md

@@ -1,10 +1,15 @@
+## ClickHouse release 18.14.15, 2018-11-21
+
+### Исправления ошибок:
+* При чтении столбцов типа `Array(String)`, размер требуемого куска памяти оценивался слишком большим, что приводило к исключению "Memory limit exceeded" при выполнении запроса. Ошибка появилась в версии 18.12.13. [#3589](https://github.com/yandex/ClickHouse/issues/3589)
+
 ## ClickHouse release 18.14.14, 2018-11-20
 
 ### Исправления ошибок:
-* Исправлена работа запросов `ON CLUSTER` в случае, когда в конфигурации кластера включено шифрование (флаг `<secure>`). [#3599]
+* Исправлена работа запросов `ON CLUSTER` в случае, когда в конфигурации кластера включено шифрование (флаг `<secure>`). [#3599](https://github.com/yandex/ClickHouse/pull/3599)
 
 ### Улучшения процесса сборки ClickHouse:
-* Испрпавлены проблемы сборки (llvm-7 из системы, macos) [#3582]
+* Испрпавлены проблемы сборки (llvm-7 из системы, macos) [#3582](https://github.com/yandex/ClickHouse/pull/3582)
 
 ## ClickHouse release 18.14.13, 2018-11-08
 

CMakeLists.txt

@@ -256,6 +256,7 @@ include (cmake/find_capnp.cmake)
 include (cmake/find_llvm.cmake)
 include (cmake/find_cpuid.cmake)
 include (cmake/find_consistent-hashing.cmake)
+include (cmake/find_base64.cmake)
 if (ENABLE_TESTS)
     include (cmake/find_gtest.cmake)
 endif ()

README.md

@@ -7,4 +7,6 @@ ClickHouse is an open-source column-oriented database management system that all
 * [Official website](https://clickhouse.yandex/) has quick high-level overview of ClickHouse on main page.
 * [Tutorial](https://clickhouse.yandex/tutorial.html) shows how to set up and query small ClickHouse cluster.
 * [Documentation](https://clickhouse.yandex/docs/en/) provides more in-depth information.
+* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
 * [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any.
+* You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person.

cmake/dbms_generate_function.cmake

@@ -1,10 +0,0 @@
-function(generate_function_register FUNCTION_AREA)
-    foreach(FUNCTION IN LISTS ARGN)
-        configure_file (registerFunction.h.in ${FUNCTIONS_GENERATED_DIR}register${FUNCTION}.h)
-        configure_file (registerFunction.cpp.in ${FUNCTIONS_GENERATED_DIR}register${FUNCTION}.cpp)
-        set(REGISTER_HEADERS "${REGISTER_HEADERS}#include \"register${FUNCTION}.h\"\n")
-        set(REGISTER_FUNCTIONS "${REGISTER_FUNCTIONS}    register${FUNCTION}(factory);\n")
-  endforeach()
-
-  configure_file (registerFunctions_area.cpp.in ${FUNCTIONS_GENERATED_DIR}registerFunctions${FUNCTION_AREA}.cpp)
-endfunction()

cmake/find_base64.cmake

@@ -0,0 +1,12 @@
+option (ENABLE_BASE64 "Enable base64" ON)
+
+if (ENABLE_BASE64)
+    if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64")
+        message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
+    else()
+        set (BASE64_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64/include)
+        set (BASE64_LIBRARY base64)
+        set (USE_BASE64 1)
+    endif()
+endif ()
+

cmake/find_odbc.cmake

@@ -71,10 +71,10 @@ if (ENABLE_ODBC)
         )
 
         # MinGW find usually fails
-        if(MINGW)
+        if (MINGW)
             set(ODBC_INCLUDE_DIRECTORIES ".")
             set(ODBC_LIBRARIES odbc32)
-        endif()
+        endif ()
 
         include(FindPackageHandleStandardArgs)
         find_package_handle_standard_args(ODBC
@@ -82,6 +82,10 @@ if (ENABLE_ODBC)
             ODBC_INCLUDE_DIRECTORIES
             ODBC_LIBRARIES)
 
+        if (USE_STATIC_LIBRARIES)
+            list(APPEND ODBC_LIBRARIES ${LTDL_LIBRARY})
+        endif ()
+
         mark_as_advanced(ODBC_FOUND ODBC_LIBRARIES ODBC_INCLUDE_DIRECTORIES)
     endif ()
 endif ()

cmake/find_poco.cmake

@@ -93,8 +93,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
     endif ()
 
     if (OPENSSL_FOUND AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL))
-        set (Poco_NetSSL_LIBRARY PocoNetSSL)
+        set (Poco_NetSSL_LIBRARY PocoNetSSL ${OPENSSL_LIBRARIES})
-        set (Poco_Crypto_LIBRARY PocoCrypto)
+        set (Poco_Crypto_LIBRARY PocoCrypto ${OPENSSL_LIBRARIES})
     endif ()
 
     if (USE_STATIC_LIBRARIES AND USE_INTERNAL_ZLIB_LIBRARY)

cmake/lib_name.cmake

@@ -1,5 +1,4 @@
 set(DIVIDE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libdivide)
-set(CITYHASH_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/cityhash102/include)
 set(COMMON_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/libs/libcommon/include ${ClickHouse_BINARY_DIR}/libs/libcommon/include)
 set(DBMS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/dbms/src ${ClickHouse_BINARY_DIR}/dbms/src)
 set(DOUBLE_CONVERSION_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/double-conversion)

cmake/print_include_directories.cmake

@@ -10,6 +10,9 @@ list(APPEND dirs ${dirs1})
 get_property (dirs1 TARGET common PROPERTY INCLUDE_DIRECTORIES)
 list(APPEND dirs ${dirs1})
 
+get_property (dirs1 TARGET cityhash PROPERTY INCLUDE_DIRECTORIES)
+list(APPEND dirs ${dirs1})
+
 if (USE_INTERNAL_BOOST_LIBRARY)
     get_property (dirs1 TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY} PROPERTY INCLUDE_DIRECTORIES)
     list(APPEND dirs ${dirs1})

cmake/test_cpu.cmake

@@ -45,6 +45,38 @@ if (HAVE_SSE42)
     set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
 endif ()
 
+set (TEST_FLAG "-mssse3")
+set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
+check_cxx_source_compiles("
+    #include <tmmintrin.h>
+    int main() {
+        __m64 a = _mm_abs_pi8(__m64());
+        (void)a;
+        return 0;
+    }
+" HAVE_SSSE3)
+
+set (TEST_FLAG "-mavx")
+set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
+check_cxx_source_compiles("
+    #include <immintrin.h>
+    int main() {
+        auto a = _mm256_insert_epi8(__m256i(), 0, 0);
+        (void)a;
+        return 0;
+    }
+" HAVE_AVX)
+
+set (TEST_FLAG "-mavx2")
+set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
+check_cxx_source_compiles("
+    #include <immintrin.h>
+    int main() {
+        auto a = _mm256_add_epi16(__m256i(), __m256i());
+        (void)a;
+        return 0;
+    }
+" HAVE_AVX2)
 
 # gcc -dM -E -mpopcnt - < /dev/null | sort > gcc-dump-popcnt
 #define __POPCNT__ 1
@@ -65,5 +97,3 @@ if (HAVE_POPCNT AND NOT ARCH_AARCH64)
 endif ()
 
 cmake_pop_check_state ()
-
-# TODO: add here sse3 test if you want use it

contrib/CMakeLists.txt

@@ -191,3 +191,6 @@ if (USE_INTERNAL_LLVM_LIBRARY)
     add_subdirectory (llvm/llvm)
 endif ()
 
+if (USE_BASE64)
+    add_subdirectory (base64-cmake)
+endif()

contrib/base64

@@ -0,0 +1 @@
+Subproject commit a27c565d1b6c676beaf297fe503c4518185666f7

contrib/base64-cmake/.gitignore

@@ -0,0 +1 @@
+config.h

contrib/base64-cmake/CMakeLists.txt

@@ -0,0 +1,52 @@
+SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64)
+
+set(base64_compile_instructions "")
+LIST(LENGTH base64_compile_instructions 0)
+macro(cast_to_bool var instruction)
+    if (HAVE_${var})
+        set(base64_${var} 1)
+        set(base64_${var}_opt ${instruction})
+    else()
+        set(base64_${var} 0)
+    endif()
+endmacro()
+
+cast_to_bool(SSSE3 "-mssse3")
+cast_to_bool(SSE41 "-msse4.1")
+cast_to_bool(SSE42 "-msse4.2")
+cast_to_bool(AVX   "-mavx")
+cast_to_bool(AVX2  "-mavx2")
+
+# write config.h file, to include it in application
+file(READ config-header.tpl header)
+file(WRITE config.h ${header})
+file(APPEND config.h "#define HAVE_SSSE3                 ${base64_SSSE3}\n")
+file(APPEND config.h "#define HAVE_SSE41                 ${base64_SSE41}\n")
+file(APPEND config.h "#define HAVE_SSE42                 ${base64_SSE42}\n")
+file(APPEND config.h "#define HAVE_AVX                   ${base64_AVX}\n")
+file(APPEND config.h "#define HAVE_AVX2                  ${base64_AVX2}\n")
+
+set(HAVE_FAST_UNALIGNED_ACCESS 0)
+if (${base64_SSSE3} OR ${base64_SSE41} OR ${base64_SSE42} OR ${base64_AVX} OR ${base64_AVX2})
+    set(HAVE_FAST_UNALIGNED_ACCESS 1)
+endif ()
+
+file(APPEND config.h "#define HAVE_FAST_UNALIGNED_ACCESS " ${HAVE_FAST_UNALIGNED_ACCESS} "\n")
+
+add_library(base64 ${LINK_MODE}
+        ${LIBRARY_DIR}/lib/lib.c
+        ${LIBRARY_DIR}/lib/codec_choose.c
+        ${LIBRARY_DIR}/lib/arch/avx/codec.c
+        ${LIBRARY_DIR}/lib/arch/avx2/codec.c
+        ${LIBRARY_DIR}/lib/arch/generic/codec.c
+        ${LIBRARY_DIR}/lib/arch/neon32/codec.c
+        ${LIBRARY_DIR}/lib/arch/neon64/codec.c
+        ${LIBRARY_DIR}/lib/arch/sse41/codec.c
+        ${LIBRARY_DIR}/lib/arch/sse42/codec.c
+        ${LIBRARY_DIR}/lib/arch/ssse3/codec.c
+
+        ${LIBRARY_DIR}/lib/codecs.h
+        config.h)
+
+target_compile_options(base64 PRIVATE ${base64_SSSE3_opt} ${base64_SSE41_opt} ${base64_SSE42_opt} ${base64_AVX_opt} ${base64_AVX2_opt})
+target_include_directories(base64 PRIVATE ${LIBRARY_DIR}/include .)

contrib/base64-cmake/config-header.tpl

@@ -0,0 +1,2 @@
+#define HAVE_NEON32                0
+#define HAVE_NEON64                0

contrib/cityhash102/CMakeLists.txt

@@ -1,9 +1,8 @@
 add_library(cityhash
 	src/city.cc
-
 	include/citycrc.h
 	include/city.h
 	src/config.h)
 
-target_include_directories (cityhash BEFORE PUBLIC include)
+target_include_directories(cityhash BEFORE PUBLIC include)
-target_include_directories (cityhash PRIVATE src)
+target_include_directories(cityhash PRIVATE src)

dbms/CMakeLists.txt

@@ -119,7 +119,7 @@ endif ()
 
 if (USE_EMBEDDED_COMPILER)
     llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
-    target_link_libraries (dbms ${REQUIRED_LLVM_LIBRARIES})
+    target_link_libraries (dbms PRIVATE ${REQUIRED_LLVM_LIBRARIES})
     target_include_directories (dbms SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
 endif ()
 
@@ -150,33 +150,48 @@ if (NOT ARCH_ARM AND CPUID_LIBRARY)
 endif()
 
 target_link_libraries (clickhouse_common_io
+        PUBLIC
     common
+        PRIVATE
     string_utils
     widechar_width
     ${LINK_LIBRARIES_ONLY_ON_X86_64}
     ${LZ4_LIBRARY}
     ${ZSTD_LIBRARY}
     ${DOUBLE_CONVERSION_LIBRARIES}
+    pocoext
+        PUBLIC
     ${Poco_Net_LIBRARY}
     ${Poco_Util_LIBRARY}
     ${Poco_Foundation_LIBRARY}
+    ${RE2_LIBRARY}
+    ${RE2_ST_LIBRARY}
+    ${CITYHASH_LIBRARIES}
+        PRIVATE
     ${ZLIB_LIBRARIES}
     ${EXECINFO_LIBRARY}
     ${ELF_LIBRARY}
+        PUBLIC
     ${Boost_SYSTEM_LIBRARY}
+        PRIVATE
     apple_rt
     ${CMAKE_DL_LIBS}
 )
 
 target_link_libraries (dbms
+        PRIVATE
     clickhouse_parsers
     clickhouse_common_config
+        PUBLIC
     clickhouse_common_io
+    pocoext
+        PUBLIC
     ${MYSQLXX_LIBRARY}
-    ${RE2_LIBRARY}
+        PRIVATE
-    ${RE2_ST_LIBRARY}
     ${BTRIE_LIBRARIES}
     ${Boost_PROGRAM_OPTIONS_LIBRARY}
+        PUBLIC
+    ${Boost_SYSTEM_LIBRARY}
 )
 
 if (NOT USE_INTERNAL_RE2_LIBRARY)
@@ -193,8 +208,8 @@ if (Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
 endif()
 
 if (USE_POCO_SQLODBC)
-    target_link_libraries (clickhouse_common_io ${Poco_SQL_LIBRARY})
+    target_link_libraries (clickhouse_common_io PRIVATE ${Poco_SQL_LIBRARY})
-    target_link_libraries (dbms ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
+    target_link_libraries (dbms PRIVATE ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
     if (NOT USE_INTERNAL_POCO_LIBRARY)
         target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIR})
         target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR} PUBLIC ${Poco_SQL_INCLUDE_DIR})
@@ -208,48 +223,44 @@ if (Poco_Data_FOUND)
 endif()
 
 if (USE_POCO_DATAODBC)
-    target_link_libraries (clickhouse_common_io ${Poco_Data_LIBRARY})
+    target_link_libraries (clickhouse_common_io PRIVATE ${Poco_Data_LIBRARY})
-    target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
+    target_link_libraries (dbms PRIVATE ${Poco_DataODBC_LIBRARY})
     if (NOT USE_INTERNAL_POCO_LIBRARY)
         target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
     endif()
 endif()
 
 if (USE_POCO_MONGODB)
-    target_link_libraries (dbms ${Poco_MongoDB_LIBRARY})
+    target_link_libraries (dbms PRIVATE ${Poco_MongoDB_LIBRARY})
 endif()
 
 if (USE_POCO_NETSSL)
-    target_link_libraries (clickhouse_common_io ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
+    target_link_libraries (clickhouse_common_io PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
-    target_link_libraries (dbms ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
+    target_link_libraries (dbms PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
 endif()
 
-target_link_libraries (dbms ${Poco_Foundation_LIBRARY})
+target_link_libraries (dbms PRIVATE ${Poco_Foundation_LIBRARY})
 
 if (USE_ICU)
-    target_link_libraries (dbms ${ICU_LIBS})
+    target_link_libraries (dbms PRIVATE ${ICU_LIBS})
     target_include_directories (dbms SYSTEM PRIVATE ${ICU_INCLUDE_DIR})
 endif ()
 
 if (USE_CAPNP)
-    target_link_libraries (dbms ${CAPNP_LIBRARY})
+    target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARY})
     if (NOT USE_INTERNAL_CAPNP_LIBRARY)
         target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR})
     endif ()
 endif ()
 
 if (USE_RDKAFKA)
-    target_link_libraries (dbms ${RDKAFKA_LIBRARY})
+    target_link_libraries (dbms PRIVATE ${RDKAFKA_LIBRARY})
     if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
         target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
     endif ()
 endif ()
 
-target_link_libraries(dbms ${OPENSSL_CRYPTO_LIBRARY})
+target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY} Threads::Threads)
-
-target_link_libraries (dbms
-    Threads::Threads
-)
 
 target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
 target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
@@ -285,6 +296,6 @@ if (ENABLE_TESTS)
     # attach all dbms gtest sources
     grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms dbms_gtest_sources)
     add_executable(unit_tests_dbms ${dbms_gtest_sources})
-    target_link_libraries(unit_tests_dbms gtest_main dbms)
+    target_link_libraries(unit_tests_dbms PRIVATE gtest_main dbms clickhouse_common_zookeeper)
     add_check(unit_tests_dbms)
 endif ()

dbms/cmake/version.cmake

@@ -22,3 +22,5 @@ endif ()
 set (VERSION_NAME "${PROJECT_NAME}" CACHE STRING "")
 set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}" CACHE STRING "")
 set (VERSION_SO "${VERSION_STRING}" CACHE STRING "")
+
+math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000")

dbms/programs/CMakeLists.txt

@@ -48,45 +48,45 @@ else ()
         link_directories (${LLVM_LIBRARY_DIRS})
     endif ()
     add_executable (clickhouse main.cpp)
-    target_link_libraries (clickhouse clickhouse_common_io)
+    target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils)
     target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
     target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 
     if (ENABLE_CLICKHOUSE_SERVER)
-        target_link_libraries (clickhouse clickhouse-server-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-server-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_CLIENT)
-        target_link_libraries (clickhouse clickhouse-client-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-client-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_LOCAL)
-        target_link_libraries (clickhouse clickhouse-local-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-local-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_BENCHMARK)
-        target_link_libraries (clickhouse clickhouse-benchmark-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-benchmark-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_PERFORMANCE)
-        target_link_libraries (clickhouse clickhouse-performance-test-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-performance-test-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_COPIER)
-        target_link_libraries (clickhouse clickhouse-copier-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-copier-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
-        target_link_libraries (clickhouse clickhouse-extract-from-config-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-extract-from-config-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_COMPRESSOR)
-        target_link_libraries (clickhouse clickhouse-compressor-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-compressor-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_FORMAT)
-        target_link_libraries (clickhouse clickhouse-format-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-format-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_OBFUSCATOR)
-        target_link_libraries (clickhouse clickhouse-obfuscator-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-obfuscator-lib)
     endif ()
     if (USE_EMBEDDED_COMPILER)
-        target_link_libraries (clickhouse clickhouse-compiler-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-compiler-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
-        target_link_libraries (clickhouse clickhouse-odbc-bridge-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-odbc-bridge-lib)
     endif()
 
     set (CLICKHOUSE_BUNDLE)

dbms/programs/benchmark/Benchmark.cpp

@@ -42,10 +42,8 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int POCO_EXCEPTION;
-    extern const int STD_EXCEPTION;
-    extern const int UNKNOWN_EXCEPTION;
     extern const int BAD_ARGUMENTS;
+    extern const int EMPTY_DATA_PASSED;
 }
 
 class Benchmark
@@ -170,7 +168,7 @@ private:
         }
 
         if (queries.empty())
-            throw Exception("Empty list of queries.");
+            throw Exception("Empty list of queries.", ErrorCodes::EMPTY_DATA_PASSED);
 
         std::cerr << "Loaded " << queries.size() << " queries.\n";
     }

dbms/programs/benchmark/CMakeLists.txt

@@ -1,8 +1,8 @@
 add_library (clickhouse-benchmark-lib ${LINK_MODE} Benchmark.cpp)
-target_link_libraries (clickhouse-benchmark-lib clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-benchmark-lib PRIVATE clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
 target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-benchmark clickhouse-benchmark.cpp)
-    target_link_libraries (clickhouse-benchmark clickhouse-benchmark-lib clickhouse_aggregate_functions)
+    target_link_libraries (clickhouse-benchmark PRIVATE clickhouse-benchmark-lib clickhouse_aggregate_functions)
 endif ()

dbms/programs/clang/CMakeLists.txt

@@ -6,9 +6,9 @@ if (CLICKHOUSE_SPLIT_BINARY)
     if (USE_EMBEDDED_COMPILER)
         link_directories (${LLVM_LIBRARY_DIRS})
         add_executable (clickhouse-clang clickhouse-clang.cpp)
-        target_link_libraries (clickhouse-clang clickhouse-compiler-lib)
+        target_link_libraries (clickhouse-clang PRIVATE clickhouse-compiler-lib)
         add_executable (clickhouse-lld clickhouse-lld.cpp)
-        target_link_libraries (clickhouse-lld clickhouse-compiler-lib)
+        target_link_libraries (clickhouse-lld PRIVATE clickhouse-compiler-lib)
         install (TARGETS clickhouse-clang clickhouse-lld RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
     endif ()
 endif ()

dbms/programs/client/CMakeLists.txt

@@ -1,13 +1,12 @@
 add_library (clickhouse-client-lib ${LINK_MODE} Client.cpp)
-target_link_libraries (clickhouse-client-lib clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-client-lib PRIVATE clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
 if (READLINE_INCLUDE_DIR)
     target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
 endif ()
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-client clickhouse-client.cpp)
-    target_link_libraries (clickhouse-client clickhouse-client-lib)
+    target_link_libraries (clickhouse-client PRIVATE clickhouse-client-lib)
 endif ()
 
 install (FILES clickhouse-client.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-client COMPONENT clickhouse-client RENAME config.xml)
-

dbms/programs/client/Client.cpp

@@ -18,7 +18,7 @@
 #include <Poco/File.h>
 #include <Poco/Util/Application.h>
 #include <common/readline_use.h>
-#include <common/find_first_symbols.h>
+#include <common/find_symbols.h>
 #include <Common/ClickHouseRevision.h>
 #include <Common/Stopwatch.h>
 #include <Common/Exception.h>
@@ -86,9 +86,6 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int POCO_EXCEPTION;
-    extern const int STD_EXCEPTION;
-    extern const int UNKNOWN_EXCEPTION;
     extern const int NETWORK_ERROR;
     extern const int NO_DATA_TO_INSERT;
     extern const int BAD_ARGUMENTS;
@@ -529,7 +526,7 @@ private:
 
         if (max_client_network_bandwidth)
         {
-            ThrottlerPtr throttler = std::make_shared<Throttler>(max_client_network_bandwidth, 0,  "");
+            ThrottlerPtr throttler = std::make_shared<Throttler>(max_client_network_bandwidth, 0, "");
             connection->setThrottler(throttler);
         }
 

dbms/programs/client/Suggest.h

@@ -184,7 +184,7 @@ public:
             }
             catch (...)
             {
-                std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false) << "\n";
+                std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n";
             }
 
             /// Note that keyword suggestions are available even if we cannot load data from server.

dbms/programs/compressor/CMakeLists.txt

@@ -1,8 +1,8 @@
 add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp)
-target_link_libraries (clickhouse-compressor-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     # Also in utils
     add_executable (clickhouse-compressor clickhouse-compressor.cpp)
-    target_link_libraries (clickhouse-compressor clickhouse-compressor-lib)
+    target_link_libraries (clickhouse-compressor PRIVATE clickhouse-compressor-lib)
 endif ()

dbms/programs/copier/CMakeLists.txt

@@ -1,5 +1,5 @@
 add_library (clickhouse-copier-lib ${LINK_MODE} ClusterCopier.cpp)
-target_link_libraries (clickhouse-copier-lib clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions)
+target_link_libraries (clickhouse-copier-lib PRIVATE clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions daemon)
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-copier clickhouse-copier.cpp)

dbms/programs/copier/ClusterCopier.cpp

@@ -323,7 +323,7 @@ struct TaskTable
 struct TaskCluster
 {
     TaskCluster(const String & task_zookeeper_path_, const String & default_local_database_)
-        : task_zookeeper_path(task_zookeeper_path_),  default_local_database(default_local_database_) {}
+        : task_zookeeper_path(task_zookeeper_path_), default_local_database(default_local_database_) {}
 
     void loadTasks(const Poco::Util::AbstractConfiguration & config, const String & base_key = "");
 
@@ -410,8 +410,7 @@ BlockInputStreamPtr squashStreamIntoOneBlock(const BlockInputStreamPtr & stream)
     return std::make_shared<SquashingBlockInputStream>(
         stream,
         std::numeric_limits<size_t>::max(),
-        std::numeric_limits<size_t>::max()
+        std::numeric_limits<size_t>::max());
-    );
 }
 
 Block getBlockWithAllStreamData(const BlockInputStreamPtr & stream)

dbms/programs/extract-from-config/CMakeLists.txt

@@ -1,7 +1,7 @@
 add_library (clickhouse-extract-from-config-lib ${LINK_MODE} ExtractFromConfig.cpp)
-target_link_libraries (clickhouse-extract-from-config-lib clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-extract-from-config-lib PRIVATE clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-extract-from-config clickhouse-extract-from-config.cpp)
-    target_link_libraries (clickhouse-extract-from-config clickhouse-extract-from-config-lib)
+    target_link_libraries (clickhouse-extract-from-config PRIVATE clickhouse-extract-from-config-lib)
 endif ()

dbms/programs/format/CMakeLists.txt

@@ -1,6 +1,6 @@
 add_library (clickhouse-format-lib ${LINK_MODE} Format.cpp)
-target_link_libraries (clickhouse-format-lib dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-format-lib PRIVATE dbms clickhouse_common_io clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY})
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-format clickhouse-format.cpp)
-    target_link_libraries (clickhouse-format clickhouse-format-lib)
+    target_link_libraries (clickhouse-format PRIVATE clickhouse-format-lib)
 endif ()

dbms/programs/local/CMakeLists.txt

@@ -1,7 +1,7 @@
 add_library (clickhouse-local-lib ${LINK_MODE} LocalServer.cpp)
-target_link_libraries (clickhouse-local-lib clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-local-lib PRIVATE clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-local clickhouse-local.cpp)
-    target_link_libraries (clickhouse-local clickhouse-local-lib)
+    target_link_libraries (clickhouse-local PRIVATE clickhouse-local-lib)
 endif ()

dbms/programs/main.cpp

@@ -1,7 +1,7 @@
 #include <iostream>
 #include <vector>
 #include <string>
-#include <utility>  /// pair
+#include <utility> /// pair
 
 #if __has_include("config_tools.h")
 #include "config_tools.h"

dbms/programs/obfuscator/CMakeLists.txt

@@ -1,8 +1,8 @@
 add_library (clickhouse-obfuscator-lib ${LINK_MODE} Obfuscator.cpp)
-target_link_libraries (clickhouse-obfuscator-lib dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-obfuscator-lib PRIVATE dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-obfuscator clickhouse-obfuscator.cpp)
     set_target_properties(clickhouse-obfuscator PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
-    target_link_libraries (clickhouse-obfuscator clickhouse-obfuscator-lib)
+    target_link_libraries (clickhouse-obfuscator PRIVATE clickhouse-obfuscator-lib)
 endif ()

dbms/programs/odbc-bridge/CMakeLists.txt

@@ -9,23 +9,23 @@ add_library (clickhouse-odbc-bridge-lib ${LINK_MODE}
     validateODBCConnectionString.cpp
 )
 
-target_link_libraries (clickhouse-odbc-bridge-lib clickhouse_common_io daemon dbms)
+target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE clickhouse_common_io daemon dbms)
 target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
 
 if (USE_POCO_SQLODBC)
-    target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQLODBC_LIBRARY})
+    target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQLODBC_LIBRARY})
     target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR})
 endif ()
 if (Poco_SQL_FOUND)
-    target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQL_LIBRARY})
+    target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQL_LIBRARY})
 endif ()
 
 if (USE_POCO_DATAODBC)
-    target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_DataODBC_LIBRARY})
+    target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_DataODBC_LIBRARY})
     target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
 endif()
 if (Poco_Data_FOUND)
-    target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_Data_LIBRARY})
+    target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_Data_LIBRARY})
 endif ()
 
 
@@ -35,5 +35,5 @@ endif ()
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-odbc-bridge odbc-bridge.cpp)
-    target_link_libraries (clickhouse-odbc-bridge clickhouse-odbc-bridge-lib)
+    target_link_libraries (clickhouse-odbc-bridge PRIVATE clickhouse-odbc-bridge-lib)
 endif ()

dbms/programs/odbc-bridge/ColumnInfoHandler.cpp

@@ -124,9 +124,9 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques
         auto identifier_quote = getIdentifierQuote(hdbc);
         if (identifier_quote.length() == 0)
             settings.identifier_quoting_style = IdentifierQuotingStyle::None;
-        else if(identifier_quote[0] == '`')
+        else if (identifier_quote[0] == '`')
             settings.identifier_quoting_style = IdentifierQuotingStyle::Backticks;
-        else if(identifier_quote[0] == '"')
+        else if (identifier_quote[0] == '"')
             settings.identifier_quoting_style = IdentifierQuotingStyle::DoubleQuotes;
         else
             throw Exception("Can not map quote identifier '" + identifier_quote + "' to IdentifierQuotingStyle value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);

dbms/programs/odbc-bridge/HandlerFactory.cpp

@@ -25,7 +25,7 @@ Poco::Net::HTTPRequestHandler * HandlerFactory::createRequestHandler(const Poco:
 #else
             return nullptr;
 #endif
-        else if(uri.getPath() == "/identifier_quote")
+        else if (uri.getPath() == "/identifier_quote")
 #if USE_POCO_SQLODBC || USE_POCO_DATAODBC
             return new IdentifierQuoteHandler(keep_alive_timeout, context);
 #else

dbms/programs/odbc-bridge/tests/CMakeLists.txt

@@ -1,2 +1,2 @@
 add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp)
-target_link_libraries (validate-odbc-connection-string clickhouse-odbc-bridge-lib)
+target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse-odbc-bridge-lib clickhouse_common_io)

dbms/programs/odbc-bridge/validateODBCConnectionString.cpp

@@ -2,7 +2,7 @@
 #include <cstring>
 #include <algorithm>
 #include <Poco/String.h>
-#include <common/find_first_symbols.h>
+#include <common/find_symbols.h>
 #include <Common/Exception.h>
 #include <Common/StringUtils/StringUtils.h>
 #include "validateODBCConnectionString.h"

dbms/programs/performance-test/CMakeLists.txt

@@ -1,8 +1,8 @@
 add_library (clickhouse-performance-test-lib ${LINK_MODE} PerformanceTest.cpp)
-target_link_libraries (clickhouse-performance-test-lib clickhouse_common_io dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-performance-test-lib PRIVATE dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
 target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-performance-test clickhouse-performance-test.cpp)
-    target_link_libraries (clickhouse-performance-test clickhouse-performance-test-lib dbms)
+    target_link_libraries (clickhouse-performance-test PRIVATE clickhouse-performance-test-lib)
 endif ()

dbms/programs/performance-test/PerformanceTest.cpp

@@ -49,10 +49,10 @@ namespace DB
 {
 namespace ErrorCodes
 {
-    extern const int POCO_EXCEPTION;
-    extern const int STD_EXCEPTION;
-    extern const int UNKNOWN_EXCEPTION;
     extern const int NOT_IMPLEMENTED;
+    extern const int LOGICAL_ERROR;
+    extern const int BAD_ARGUMENTS;
+    extern const int FILE_DOESNT_EXIST;
 }
 
 static String pad(size_t padding)
@@ -156,7 +156,7 @@ struct StopConditionsSet
             else if (key == "average_speed_not_changing_for_ms")
                 average_speed_not_changing_for_ms.value = stop_conditions_view->getUInt64(key);
             else
-                throw DB::Exception("Met unkown stop condition: " + key);
+                throw DB::Exception("Met unkown stop condition: " + key, DB::ErrorCodes::LOGICAL_ERROR);
 
             ++initialized_count;
         }
@@ -521,7 +521,7 @@ public:
     {
         if (input_files.size() < 1)
         {
-            throw DB::Exception("No tests were specified", 0);
+            throw DB::Exception("No tests were specified", DB::ErrorCodes::BAD_ARGUMENTS);
         }
 
         std::string name;
@@ -694,7 +694,7 @@ private:
                 size_t ram_size_needed = config->getUInt64("preconditions.ram_size");
                 size_t actual_ram = getMemoryAmount();
                 if (!actual_ram)
-                    throw DB::Exception("ram_size precondition not available on this platform", ErrorCodes::NOT_IMPLEMENTED);
+                    throw DB::Exception("ram_size precondition not available on this platform", DB::ErrorCodes::NOT_IMPLEMENTED);
 
                 if (ram_size_needed > actual_ram)
                 {
@@ -868,12 +868,12 @@ private:
 
         if (!test_config->has("query") && !test_config->has("query_file"))
         {
-            throw DB::Exception("Missing query fields in test's config: " + test_name);
+            throw DB::Exception("Missing query fields in test's config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
         }
 
         if (test_config->has("query") && test_config->has("query_file"))
         {
-            throw DB::Exception("Found both query and query_file fields. Choose only one");
+            throw DB::Exception("Found both query and query_file fields. Choose only one", DB::ErrorCodes::BAD_ARGUMENTS);
         }
 
         if (test_config->has("query"))
@@ -885,7 +885,7 @@ private:
         {
             const String filename = test_config->getString("query_file");
             if (filename.empty())
-                throw DB::Exception("Empty file name");
+                throw DB::Exception("Empty file name", DB::ErrorCodes::BAD_ARGUMENTS);
 
             bool tsv = fs::path(filename).extension().string() == ".tsv";
 
@@ -909,7 +909,7 @@ private:
 
         if (queries.empty())
         {
-            throw DB::Exception("Did not find any query to execute: " + test_name);
+            throw DB::Exception("Did not find any query to execute: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
         }
 
         if (test_config->has("substitutions"))
@@ -929,7 +929,7 @@ private:
 
         if (!test_config->has("type"))
         {
-            throw DB::Exception("Missing type property in config: " + test_name);
+            throw DB::Exception("Missing type property in config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
         }
 
         String config_exec_type = test_config->getString("type");
@@ -938,7 +938,7 @@ private:
         else if (config_exec_type == "once")
             exec_type = ExecutionType::Once;
         else
-            throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name);
+            throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
 
         times_to_run = test_config->getUInt("times_to_run", 1);
 
@@ -951,7 +951,7 @@ private:
         }
 
         if (stop_conditions_template.empty())
-            throw DB::Exception("No termination conditions were found in config");
+            throw DB::Exception("No termination conditions were found in config", DB::ErrorCodes::BAD_ARGUMENTS);
 
         for (size_t i = 0; i < times_to_run * queries.size(); ++i)
             stop_conditions_by_run.push_back(stop_conditions_template);
@@ -978,7 +978,7 @@ private:
         else
         {
             if (lite_output)
-                throw DB::Exception("Specify main_metric for lite output");
+                throw DB::Exception("Specify main_metric for lite output", DB::ErrorCodes::BAD_ARGUMENTS);
         }
 
         if (metrics.size() > 0)
@@ -1023,22 +1023,14 @@ private:
         if (exec_type == ExecutionType::Loop)
         {
             for (const String & metric : metrics)
-            {
                 if (std::find(non_loop_metrics.begin(), non_loop_metrics.end(), metric) != non_loop_metrics.end())
-                {
+                   throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS);
-                    throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")");
-                }
-            }
         }
         else
         {
             for (const String & metric : metrics)
-            {
                 if (std::find(loop_metrics.begin(), loop_metrics.end(), metric) != loop_metrics.end())
-                {
+                    throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS);
-                    throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")");
-                }
-            }
         }
     }
 
@@ -1439,7 +1431,7 @@ try
         if (input_files.empty())
         {
             std::cerr << std::endl;
-            throw DB::Exception("Did not find any xml files");
+            throw DB::Exception("Did not find any xml files", DB::ErrorCodes::BAD_ARGUMENTS);
         }
         else
             std::cerr << " found " << input_files.size() << " files." << std::endl;
@@ -1454,7 +1446,7 @@ try
             fs::path file(filename);
 
             if (!fs::exists(file))
-                throw DB::Exception("File '" + filename + "' does not exist");
+                throw DB::Exception("File '" + filename + "' does not exist", DB::ErrorCodes::FILE_DOESNT_EXIST);
 
             if (fs::is_directory(file))
             {
@@ -1463,7 +1455,7 @@ try
             else
             {
                 if (file.extension().string() != ".xml")
-                    throw DB::Exception("File '" + filename + "' does not have .xml extension");
+                    throw DB::Exception("File '" + filename + "' does not have .xml extension", DB::ErrorCodes::BAD_ARGUMENTS);
                 collected_files.push_back(filename);
             }
         }

dbms/programs/server/CMakeLists.txt

@@ -10,12 +10,16 @@ add_library (clickhouse-server-lib ${LINK_MODE}
     TCPHandler.cpp
     )
 
-target_link_libraries (clickhouse-server-lib clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions)
+target_link_libraries (clickhouse-server-lib PRIVATE clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY})
+if (USE_POCO_NETSSL)
+    target_link_libraries (clickhouse-server-lib PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
+endif ()
+
 target_include_directories (clickhouse-server-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-server clickhouse-server.cpp)
-    target_link_libraries (clickhouse-server clickhouse-server-lib)
+    target_link_libraries (clickhouse-server PRIVATE clickhouse-server-lib)
     install (TARGETS clickhouse-server ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 endif ()
 

dbms/programs/server/InterserverIOHTTPHandler.cpp

@@ -19,9 +19,6 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int ABORTED;
-    extern const int POCO_EXCEPTION;
-    extern const int STD_EXCEPTION;
-    extern const int UNKNOWN_EXCEPTION;
     extern const int TOO_MANY_SIMULTANEOUS_QUERIES;
 }
 

dbms/programs/server/Server.cpp

@@ -55,6 +55,7 @@
 namespace CurrentMetrics
 {
     extern const Metric Revision;
+    extern const Metric VersionInteger;
 }
 
 namespace DB
@@ -66,6 +67,8 @@ namespace ErrorCodes
     extern const int SUPPORT_IS_DISABLED;
     extern const int ARGUMENT_OUT_OF_BOUND;
     extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
+    extern const int INVALID_CONFIG_PARAMETER;
+    extern const int SYSTEM_ERROR;
 }
 
 
@@ -73,7 +76,7 @@ static std::string getCanonicalPath(std::string && path)
 {
     Poco::trimInPlace(path);
     if (path.empty())
-        throw Exception("path configuration parameter is empty");
+        throw Exception("path configuration parameter is empty", ErrorCodes::INVALID_CONFIG_PARAMETER);
     if (path.back() != '/')
         path += '/';
     return std::move(path);
@@ -108,6 +111,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
     registerStorages();
 
     CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get());
+    CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
 
     /** Context contains all that query execution is dependent:
       *  settings, available functions, data types, aggregate functions, databases...
@@ -141,7 +145,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
             {
                 LOG_TRACE(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds.");
                 if (0 != mlockall(MCL_CURRENT))
-                    LOG_WARNING(log, "Failed mlockall: " + errnoToString());
+                    LOG_WARNING(log, "Failed mlockall: " + errnoToString(ErrorCodes::SYSTEM_ERROR));
                 else
                     LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed");
             }
@@ -555,10 +559,10 @@ int Server::main(const std::vector<std::string> & /*args*/)
                     socket.setReceiveTimeout(settings.receive_timeout);
                     socket.setSendTimeout(settings.send_timeout);
                     servers.emplace_back(new Poco::Net::TCPServer(
-                        new TCPHandlerFactory(*this, /* secure= */ true ),
+                        new TCPHandlerFactory(*this, /* secure= */ true),
-                                                                  server_pool,
+                        server_pool,
-                                                                  socket,
+                        socket,
-                                                                  new Poco::Net::TCPServerParams));
+                        new Poco::Net::TCPServerParams));
                     LOG_INFO(log, "Listening tcp_secure: " + address.toString());
 #else
                     throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
@@ -633,7 +637,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
             message << "Available RAM = " << formatReadableSizeWithBinarySuffix(memory_amount) << ";"
                 << " physical cores = " << getNumberOfPhysicalCPUCores() << ";"
                 // on ARM processors it can show only enabled at current moment cores
-                << " threads = " <<  std::thread::hardware_concurrency() << ".";
+                << " threads = " << std::thread::hardware_concurrency() << ".";
             LOG_INFO(log, message.str());
         }
 

dbms/programs/server/TCPHandler.cpp

@@ -181,7 +181,7 @@ void TCPHandler::runImpl()
                 /// Reset the input stream, as we received an empty block while receiving external table data.
                 /// So, the stream has been marked as cancelled and we can't read from it anymore.
                 state.block_in.reset();
-                state.maybe_compressed_in.reset();  /// For more accurate accounting by MemoryTracker.
+                state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker.
             });
 
             /// Processing Query
@@ -718,7 +718,7 @@ bool TCPHandler::receiveData()
             {
                 NamesAndTypesList columns = block.getNamesAndTypesList();
                 storage = StorageMemory::create(external_table_name,
-                    ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
+                    ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, ColumnComments{}});
                 storage->startup();
                 query_context.addExternalTable(external_table_name, storage);
             }

dbms/programs/server/TCPHandler.h

@@ -89,7 +89,7 @@ public:
         , connection_context(server.context())
         , query_context(server.context())
     {
-        server_display_name =  server.config().getString("display_name", getFQDNOrHostName());
+        server_display_name = server.config().getString("display_name", getFQDNOrHostName());
     }
 
     void run();

dbms/programs/server/config.xml

@@ -164,6 +164,20 @@
                 </replica>
             </shard>
         </test_shard_localhost>
+        <test_cluster_two_shards_localhost>
+             <shard>
+                 <replica>
+                     <host>localhost</host>
+                     <port>9000</port>
+                 </replica>
+             </shard>
+             <shard>
+                 <replica>
+                     <host>localhost</host>
+                     <port>9000</port>
+                 </replica>
+             </shard>
+         </test_cluster_two_shards_localhost>
         <test_shard_localhost_secure>
             <shard>
                 <replica>

dbms/programs/server/users.xml

@@ -56,7 +56,7 @@
 
                  Each element of list has one of the following forms:
                  <ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
-		     2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
+                     2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
                  <host> Hostname. Example: server01.yandex.ru.
                      To check access, DNS query is performed, and all received addresses compared to peer address.
                  <host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$

dbms/scripts/gen-bias-data.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3.4
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 
 import sys

dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp

@@ -55,10 +55,10 @@ static AggregateFunctionPtr createAggregateFunctionGroupArray(const std::string
     else if (parameters.size() == 1)
     {
         auto type = parameters[0].getType();
-        if (type != Field::Types::Int64  && type != Field::Types::UInt64)
+        if (type != Field::Types::Int64 && type != Field::Types::UInt64)
                throw Exception("Parameter for aggregate function " + name + " should be positive number", ErrorCodes::BAD_ARGUMENTS);
 
-        if ((type == Field::Types::Int64  && parameters[0].get<Int64>() < 0) ||
+        if ((type == Field::Types::Int64 && parameters[0].get<Int64>() < 0) ||
             (type == Field::Types::UInt64 && parameters[0].get<UInt64>() == 0))
             throw Exception("Parameter for aggregate function " + name + " should be positive number", ErrorCodes::BAD_ARGUMENTS);
 

dbms/src/AggregateFunctions/AggregateFunctionStatistics.h

@@ -274,7 +274,7 @@ public:
 
         ++count;
 
-        left_mean +=  left_delta / count;
+        left_mean += left_delta / count;
         right_mean += right_delta / count;
         co_moment += (left_val - left_mean) * (right_val - old_right_mean);
 

dbms/src/AggregateFunctions/AggregateFunctionUniq.cpp

@@ -130,9 +130,6 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory & factory)
 
     factory.registerFunction("uniqExact",
         createAggregateFunctionUniq<true, AggregateFunctionUniqExactData, AggregateFunctionUniqExactData<String>>);
-
-    factory.registerFunction("uniqCombined",
-        createAggregateFunctionUniq<false, AggregateFunctionUniqCombinedData, AggregateFunctionUniqCombinedData<UInt64>>);
 }
 
 }

dbms/src/AggregateFunctions/AggregateFunctionUniq.h

@@ -22,7 +22,6 @@
 #include <Common/typeid_cast.h>
 
 #include <AggregateFunctions/IAggregateFunction.h>
-#include <AggregateFunctions/UniqCombinedBiasData.h>
 #include <AggregateFunctions/UniqVariadicHash.h>
 
 
@@ -124,46 +123,6 @@ struct AggregateFunctionUniqExactData<String>
     static String getName() { return "uniqExact"; }
 };
 
-template <typename T>
-struct AggregateFunctionUniqCombinedData
-{
-    using Key = UInt32;
-    using Set = CombinedCardinalityEstimator<
-        Key,
-        HashSet<Key, TrivialHash, HashTableGrower<>>,
-        16,
-        14,
-        17,
-        TrivialHash,
-        UInt32,
-        HyperLogLogBiasEstimator<UniqCombinedBiasData>,
-        HyperLogLogMode::FullFeatured>;
-
-    Set set;
-
-    static String getName() { return "uniqCombined"; }
-};
-
-template <>
-struct AggregateFunctionUniqCombinedData<String>
-{
-    using Key = UInt64;
-    using Set = CombinedCardinalityEstimator<
-        Key,
-        HashSet<Key, TrivialHash, HashTableGrower<>>,
-        16,
-        14,
-        17,
-        TrivialHash,
-        UInt64,
-        HyperLogLogBiasEstimator<UniqCombinedBiasData>,
-        HyperLogLogMode::FullFeatured>;
-
-    Set set;
-
-    static String getName() { return "uniqCombined"; }
-};
-
 
 namespace detail
 {
@@ -199,39 +158,6 @@ template <> struct AggregateFunctionUniqTraits<Float64>
     }
 };
 
-/** Hash function for uniqCombined.
-  */
-template <typename T> struct AggregateFunctionUniqCombinedTraits
-{
-    static UInt32 hash(T x) { return static_cast<UInt32>(intHash64(x)); }
-};
-
-template <> struct AggregateFunctionUniqCombinedTraits<UInt128>
-{
-    static UInt32 hash(UInt128 x)
-    {
-        return sipHash64(x);
-    }
-};
-
-template <> struct AggregateFunctionUniqCombinedTraits<Float32>
-{
-    static UInt32 hash(Float32 x)
-    {
-        UInt64 res = ext::bit_cast<UInt64>(x);
-        return static_cast<UInt32>(intHash64(res));
-    }
-};
-
-template <> struct AggregateFunctionUniqCombinedTraits<Float64>
-{
-    static UInt32 hash(Float64 x)
-    {
-        UInt64 res = ext::bit_cast<UInt64>(x);
-        return static_cast<UInt32>(intHash64(res));
-    }
-};
-
 
 /** The structure for the delegation work to add one element to the `uniq` aggregate functions.
   * Used for partial specialization to add strings.
@@ -255,19 +181,6 @@ struct OneAdder
                 data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
             }
         }
-        else if constexpr (std::is_same_v<Data, AggregateFunctionUniqCombinedData<T>>)
-        {
-            if constexpr (!std::is_same_v<T, String>)
-            {
-                const auto & value = static_cast<const ColumnVector<T> &>(column).getData()[row_num];
-                data.set.insert(AggregateFunctionUniqCombinedTraits<T>::hash(value));
-            }
-            else
-            {
-                StringRef value = column.getDataAt(row_num);
-                data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
-            }
-        }
         else if constexpr (std::is_same_v<Data, AggregateFunctionUniqExactData<T>>)
         {
             if constexpr (!std::is_same_v<T, String>)
@@ -387,5 +300,4 @@ public:
     const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
-
 }

dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp

@@ -0,0 +1,127 @@
+#include <AggregateFunctions/AggregateFunctionUniqCombined.h>
+
+#include <AggregateFunctions/AggregateFunctionFactory.h>
+#include <AggregateFunctions/Helpers.h>
+
+#include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDateTime.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+    extern const int ARGUMENT_OUT_OF_BOUND;
+}
+
+namespace
+{
+    template <UInt8 K>
+    struct WithK
+    {
+        template <typename T>
+        using AggregateFunction = AggregateFunctionUniqCombined<T, K>;
+
+        template <bool is_exact, bool argument_is_tuple>
+        using AggregateFunctionVariadic = AggregateFunctionUniqCombinedVariadic<is_exact, argument_is_tuple, K>;
+    };
+
+    template <UInt8 K>
+    AggregateFunctionPtr createAggregateFunctionWithK(const DataTypes & argument_types)
+    {
+        /// We use exact hash function if the arguments are not contiguous in memory, because only exact hash function has support for this case.
+        bool use_exact_hash_function = !isAllArgumentsContiguousInMemory(argument_types);
+
+        if (argument_types.size() == 1)
+        {
+            const IDataType & argument_type = *argument_types[0];
+
+            AggregateFunctionPtr res(createWithNumericType<WithK<K>::template AggregateFunction>(*argument_types[0]));
+
+            WhichDataType which(argument_type);
+            if (res)
+                return res;
+            else if (which.isDate())
+                return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeDate::FieldType>>();
+            else if (which.isDateTime())
+                return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeDateTime::FieldType>>();
+            else if (which.isStringOrFixedString())
+                return std::make_shared<typename WithK<K>::template AggregateFunction<String>>();
+            else if (which.isUUID())
+                return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeUUID::FieldType>>();
+            else if (which.isTuple())
+            {
+                if (use_exact_hash_function)
+                    return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<true, true>>(argument_types);
+                else
+                    return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<false, true>>(argument_types);
+            }
+        }
+
+        /// "Variadic" method also works as a fallback generic case for a single argument.
+        if (use_exact_hash_function)
+            return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<true, false>>(argument_types);
+        else
+            return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<false, false>>(argument_types);
+    }
+
+    AggregateFunctionPtr createAggregateFunctionUniqCombined(
+        const std::string & name, const DataTypes & argument_types, const Array & params)
+    {
+        /// log2 of the number of cells in HyperLogLog.
+        /// Reasonable default value, selected to be comparable in quality with "uniq" aggregate function.
+        UInt8 precision = 17;
+
+        if (!params.empty())
+        {
+            if (params.size() != 1)
+                throw Exception(
+                    "Aggregate function " + name + " requires one parameter or less.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+            UInt64 precision_param = applyVisitor(FieldVisitorConvertToNumber<UInt64>(), params[0]);
+
+            // This range is hardcoded below
+            if (precision_param > 20 || precision_param < 12)
+                throw Exception(
+                    "Parameter for aggregate function " + name + "is out or range: [12, 20].", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
+
+            precision = precision_param;
+        }
+
+        if (argument_types.empty())
+            throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+        switch (precision)
+        {
+            case 12:
+                return createAggregateFunctionWithK<12>(argument_types);
+            case 13:
+                return createAggregateFunctionWithK<13>(argument_types);
+            case 14:
+                return createAggregateFunctionWithK<14>(argument_types);
+            case 15:
+                return createAggregateFunctionWithK<15>(argument_types);
+            case 16:
+                return createAggregateFunctionWithK<16>(argument_types);
+            case 17:
+                return createAggregateFunctionWithK<17>(argument_types);
+            case 18:
+                return createAggregateFunctionWithK<18>(argument_types);
+            case 19:
+                return createAggregateFunctionWithK<19>(argument_types);
+            case 20:
+                return createAggregateFunctionWithK<20>(argument_types);
+        }
+
+        __builtin_unreachable();
+    }
+
+} // namespace
+
+void registerAggregateFunctionUniqCombined(AggregateFunctionFactory & factory)
+{
+    factory.registerFunction("uniqCombined", createAggregateFunctionUniqCombined);
+}
+
+} // namespace DB

dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.h

@@ -0,0 +1,229 @@
+#pragma once
+
+#include <Common/CombinedCardinalityEstimator.h>
+
+#include <DataTypes/DataTypeTuple.h>
+#include <DataTypes/DataTypeUUID.h>
+#include <DataTypes/DataTypesNumber.h>
+
+#include <AggregateFunctions/IAggregateFunction.h>
+#include <AggregateFunctions/UniqCombinedBiasData.h>
+#include <AggregateFunctions/UniqVariadicHash.h>
+
+#include <ext/bit_cast.h>
+
+#include <Columns/ColumnVector.h>
+#include <Columns/ColumnsNumber.h>
+#include <Common/FieldVisitors.h>
+#include <Common/SipHash.h>
+
+#include <Common/typeid_cast.h>
+
+namespace DB
+{
+namespace detail
+{
+    /** Hash function for uniqCombined.
+     */
+    template <typename T>
+    struct AggregateFunctionUniqCombinedTraits
+    {
+        static UInt32 hash(T x)
+        {
+            return static_cast<UInt32>(intHash64(x));
+        }
+    };
+
+    template <>
+    struct AggregateFunctionUniqCombinedTraits<UInt128>
+    {
+        static UInt32 hash(UInt128 x)
+        {
+            return sipHash64(x);
+        }
+    };
+
+    template <>
+    struct AggregateFunctionUniqCombinedTraits<Float32>
+    {
+        static UInt32 hash(Float32 x)
+        {
+            UInt64 res = ext::bit_cast<UInt64>(x);
+            return static_cast<UInt32>(intHash64(res));
+        }
+    };
+
+    template <>
+    struct AggregateFunctionUniqCombinedTraits<Float64>
+    {
+        static UInt32 hash(Float64 x)
+        {
+            UInt64 res = ext::bit_cast<UInt64>(x);
+            return static_cast<UInt32>(intHash64(res));
+        }
+    };
+
+} // namespace detail
+
+
+template <typename Key, UInt8 K>
+struct AggregateFunctionUniqCombinedDataWithKey
+{
+    // TODO(ilezhankin): pre-generate values for |UniqCombinedBiasData|,
+    //                   at the moment gen-bias-data.py script doesn't work.
+
+    // We want to migrate from |HashSet| to |HyperLogLogCounter| when the sizes in memory become almost equal.
+    // The size per element in |HashSet| is sizeof(Key)*2 bytes, and the overall size of |HyperLogLogCounter| is 2^K * 6 bits.
+    // For Key=UInt32 we can calculate: 2^X * 4 * 2 ≤ 2^(K-3) * 6 ⇒ X ≤ K-4.
+    using Set = CombinedCardinalityEstimator<Key, HashSet<Key, TrivialHash, HashTableGrower<>>, 16, K - 4, K, TrivialHash, Key>;
+
+    Set set;
+};
+
+template <typename Key>
+struct AggregateFunctionUniqCombinedDataWithKey<Key, 17>
+{
+    using Set = CombinedCardinalityEstimator<Key,
+        HashSet<Key, TrivialHash, HashTableGrower<>>,
+        16,
+        13,
+        17,
+        TrivialHash,
+        Key,
+        HyperLogLogBiasEstimator<UniqCombinedBiasData>,
+        HyperLogLogMode::FullFeatured>;
+
+    Set set;
+};
+
+
+template <typename T, UInt8 K>
+struct AggregateFunctionUniqCombinedData : public AggregateFunctionUniqCombinedDataWithKey<UInt32, K>
+{
+};
+
+
+template <UInt8 K>
+struct AggregateFunctionUniqCombinedData<String, K> : public AggregateFunctionUniqCombinedDataWithKey<UInt64, K>
+{
+};
+
+
+template <typename T, UInt8 K>
+class AggregateFunctionUniqCombined final
+    : public IAggregateFunctionDataHelper<AggregateFunctionUniqCombinedData<T, K>, AggregateFunctionUniqCombined<T, K>>
+{
+public:
+    String getName() const override
+    {
+        return "uniqCombined";
+    }
+
+    DataTypePtr getReturnType() const override
+    {
+        return std::make_shared<DataTypeUInt64>();
+    }
+
+    void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
+    {
+        if constexpr (!std::is_same_v<T, String>)
+        {
+            const auto & value = static_cast<const ColumnVector<T> &>(*columns[0]).getData()[row_num];
+            this->data(place).set.insert(detail::AggregateFunctionUniqCombinedTraits<T>::hash(value));
+        }
+        else
+        {
+            StringRef value = columns[0]->getDataAt(row_num);
+            this->data(place).set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
+        }
+    }
+
+    void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
+    {
+        this->data(place).set.merge(this->data(rhs).set);
+    }
+
+    void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
+    {
+        this->data(place).set.write(buf);
+    }
+
+    void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
+    {
+        this->data(place).set.read(buf);
+    }
+
+    void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
+    {
+        static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
+    }
+
+    const char * getHeaderFilePath() const override
+    {
+        return __FILE__;
+    }
+};
+
+/** For multiple arguments. To compute, hashes them.
+  * You can pass multiple arguments as is; You can also pass one argument - a tuple.
+  * But (for the possibility of efficient implementation), you can not pass several arguments, among which there are tuples.
+  */
+template <bool is_exact, bool argument_is_tuple, UInt8 K>
+class AggregateFunctionUniqCombinedVariadic final : public IAggregateFunctionDataHelper<AggregateFunctionUniqCombinedData<UInt64, K>,
+                                                        AggregateFunctionUniqCombinedVariadic<is_exact, argument_is_tuple, K>>
+{
+private:
+    size_t num_args = 0;
+
+public:
+    explicit AggregateFunctionUniqCombinedVariadic(const DataTypes & arguments)
+    {
+        if (argument_is_tuple)
+            num_args = typeid_cast<const DataTypeTuple &>(*arguments[0]).getElements().size();
+        else
+            num_args = arguments.size();
+    }
+
+    String getName() const override
+    {
+        return "uniqCombined";
+    }
+
+    DataTypePtr getReturnType() const override
+    {
+        return std::make_shared<DataTypeUInt64>();
+    }
+
+    void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
+    {
+        this->data(place).set.insert(typename AggregateFunctionUniqCombinedData<UInt64, K>::Set::value_type(
+            UniqVariadicHash<is_exact, argument_is_tuple>::apply(num_args, columns, row_num)));
+    }
+
+    void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
+    {
+        this->data(place).set.merge(this->data(rhs).set);
+    }
+
+    void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
+    {
+        this->data(place).set.write(buf);
+    }
+
+    void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
+    {
+        this->data(place).set.read(buf);
+    }
+
+    void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
+    {
+        static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
+    }
+
+    const char * getHeaderFilePath() const override
+    {
+        return __FILE__;
+    }
+};
+
+} // namespace DB

dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h

@@ -16,10 +16,12 @@
 
 namespace DB
 {
+
 namespace ErrorCodes
 {
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
     extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
 }
 
 struct ComparePairFirst final
@@ -191,7 +193,7 @@ public:
         const auto time_arg = arguments.front().get();
         if (!WhichDataType(time_arg).isDateTime() && !WhichDataType(time_arg).isUInt32())
             throw Exception{"Illegal type " + time_arg->getName() + " of first argument of aggregate function " + getName()
-                + ", must be DateTime or UInt32"};
+                + ", must be DateTime or UInt32", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         for (const auto i : ext::range(1, arguments.size()))
         {

dbms/src/AggregateFunctions/CMakeLists.txt

@@ -20,5 +20,5 @@ list(REMOVE_ITEM clickhouse_aggregate_functions_headers
 )
 
 add_library(clickhouse_aggregate_functions ${LINK_MODE} ${clickhouse_aggregate_functions_sources})
-target_link_libraries(clickhouse_aggregate_functions dbms)
+target_link_libraries(clickhouse_aggregate_functions PRIVATE dbms)
 target_include_directories (clickhouse_aggregate_functions BEFORE PRIVATE ${COMMON_INCLUDE_DIR})

dbms/src/AggregateFunctions/registerAggregateFunctions.cpp

@@ -21,6 +21,7 @@ void registerAggregateFunctionsStatisticsSimple(AggregateFunctionFactory &);
 void registerAggregateFunctionSum(AggregateFunctionFactory &);
 void registerAggregateFunctionSumMap(AggregateFunctionFactory &);
 void registerAggregateFunctionsUniq(AggregateFunctionFactory &);
+void registerAggregateFunctionUniqCombined(AggregateFunctionFactory &);
 void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &);
 void registerAggregateFunctionTopK(AggregateFunctionFactory &);
 void registerAggregateFunctionsBitwise(AggregateFunctionFactory &);
@@ -55,6 +56,7 @@ void registerAggregateFunctions()
         registerAggregateFunctionSum(factory);
         registerAggregateFunctionSumMap(factory);
         registerAggregateFunctionsUniq(factory);
+        registerAggregateFunctionUniqCombined(factory);
         registerAggregateFunctionUniqUpTo(factory);
         registerAggregateFunctionTopK(factory);
         registerAggregateFunctionsBitwise(factory);

dbms/src/Client/CMakeLists.txt

@@ -4,3 +4,5 @@
 #add_library(clickhouse_client ${LINK_MODE} ${clickhouse_client_headers} ${clickhouse_client_sources})
 #target_link_libraries (clickhouse_client clickhouse_common_io ${Poco_Net_LIBRARY})
 #target_include_directories (clickhouse_client PRIVATE ${DBMS_INCLUDE_DIR})
+
+add_subdirectory(tests)

dbms/src/Client/Connection.cpp

@@ -56,7 +56,7 @@ void Connection::connect()
             disconnect();
 
         LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << (default_database.empty() ? "(not specified)" : default_database) << ". User: " << user
-        << (static_cast<bool>(secure) ? ". Secure" : "") << (static_cast<bool>(compression) ? "" : ". Uncompressed") );
+        << (static_cast<bool>(secure) ? ". Secure" : "") << (static_cast<bool>(compression) ? "" : ". Uncompressed"));
 
         if (static_cast<bool>(secure))
         {
@@ -108,14 +108,14 @@ void Connection::connect()
         disconnect();
 
         /// Add server address to exception. Also Exception will remember stack trace. It's a pity that more precise exception type is lost.
-        throw NetException(e.displayText(), "(" + getDescription() + ")", ErrorCodes::NETWORK_ERROR);
+        throw NetException(e.displayText() + " (" + getDescription() + ")", ErrorCodes::NETWORK_ERROR);
     }
     catch (Poco::TimeoutException & e)
     {
         disconnect();
 
         /// Add server address to exception. Also Exception will remember stack trace. It's a pity that more precise exception type is lost.
-        throw NetException(e.displayText(), "(" + getDescription() + ")", ErrorCodes::SOCKET_TIMEOUT);
+        throw NetException(e.displayText() + " (" + getDescription() + ")", ErrorCodes::SOCKET_TIMEOUT);
     }
 }
 
@@ -696,7 +696,7 @@ void Connection::setDescription()
 {
     auto resolved_address = getResolvedAddress();
     description = host + ":" + toString(resolved_address.port());
-    auto ip_address =  resolved_address.host().toString();
+    auto ip_address = resolved_address.host().toString();
 
     if (host != ip_address)
         description += ", " + ip_address;

dbms/src/Client/ConnectionPoolWithFailover.cpp

@@ -203,7 +203,7 @@ ConnectionPoolWithFailover::tryGetEntry(
             LOG_TRACE(
                     log, "Server " << result.entry->getDescription() << " has unacceptable replica delay "
                     << "for table " << table_to_check->database << "." << table_to_check->table
-                    << ": "  << delay);
+                    << ": " << delay);
             ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica);
         }
     }

dbms/src/Client/tests/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_executable(test-connect test_connect.cpp)
+target_link_libraries (test-connect dbms)

dbms/src/Client/tests/test_connect.cpp

@@ -0,0 +1,59 @@
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <iostream>
+#include <Poco/Net/StreamSocket.h>
+#include <Common/Exception.h>
+#include <IO/ReadHelpers.h>
+
+
+/** In a loop it connects to the server and immediately breaks the connection.
+  * Using the SO_LINGER option, we ensure that the connection is terminated by sending a RST packet (not FIN).
+  * This behavior causes a bug in the TCPServer implementation in the Poco library.
+  */
+int main(int argc, char ** argv)
+try
+{
+    for (size_t i = 0, num_iters = argc >= 2 ? DB::parse<size_t>(argv[1]) : 1; i < num_iters; ++i)
+    {
+        std::cerr << ".";
+
+        Poco::Net::SocketAddress address("localhost", 9000);
+
+        int fd = socket(PF_INET, SOCK_STREAM, IPPROTO_IP);
+
+        if (fd < 0)
+            DB::throwFromErrno("Cannot create socket", 0);
+
+        linger linger_value;
+        linger_value.l_onoff = 1;
+        linger_value.l_linger = 0;
+
+        if (0 != setsockopt(fd, SOL_SOCKET, SO_LINGER, &linger_value, sizeof(linger_value)))
+            DB::throwFromErrno("Cannot set linger", 0);
+
+        try
+        {
+            int res = connect(fd, address.addr(), address.length());
+
+            if (res != 0 && errno != EINPROGRESS && errno != EWOULDBLOCK)
+            {
+                close(fd);
+                DB::throwFromErrno("Cannot connect", 0);
+            }
+
+            close(fd);
+        }
+        catch (const Poco::Exception & e)
+        {
+            std::cerr << e.displayText() << "\n";
+        }
+    }
+
+    std::cerr << "\n";
+}
+catch (const Poco::Exception & e)
+{
+    std::cerr << e.displayText() << "\n";
+}

dbms/src/Columns/ColumnArray.cpp

@@ -437,11 +437,11 @@ ColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hin
     auto res = ColumnArray::create(data->cloneEmpty());
 
     const ColumnString & src_string = typeid_cast<const ColumnString &>(*data);
-    const ColumnString::Chars_t & src_chars = src_string.getChars();
+    const ColumnString::Chars & src_chars = src_string.getChars();
     const Offsets & src_string_offsets = src_string.getOffsets();
     const Offsets & src_offsets = getOffsets();
 
-    ColumnString::Chars_t & res_chars = typeid_cast<ColumnString &>(res->getData()).getChars();
+    ColumnString::Chars & res_chars = typeid_cast<ColumnString &>(res->getData()).getChars();
     Offsets & res_string_offsets = typeid_cast<ColumnString &>(res->getData()).getOffsets();
     Offsets & res_offsets = res->getOffsets();
 
@@ -697,20 +697,20 @@ ColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
     if (replicate_offsets.empty())
         return cloneEmpty();
 
-    if (typeid_cast<const ColumnUInt8 *>(data.get()))     return replicateNumber<UInt8>(replicate_offsets);
+    if (typeid_cast<const ColumnUInt8 *>(data.get()))    return replicateNumber<UInt8>(replicate_offsets);
-    if (typeid_cast<const ColumnUInt16 *>(data.get()))    return replicateNumber<UInt16>(replicate_offsets);
+    if (typeid_cast<const ColumnUInt16 *>(data.get()))   return replicateNumber<UInt16>(replicate_offsets);
-    if (typeid_cast<const ColumnUInt32 *>(data.get()))    return replicateNumber<UInt32>(replicate_offsets);
+    if (typeid_cast<const ColumnUInt32 *>(data.get()))   return replicateNumber<UInt32>(replicate_offsets);
-    if (typeid_cast<const ColumnUInt64 *>(data.get()))    return replicateNumber<UInt64>(replicate_offsets);
+    if (typeid_cast<const ColumnUInt64 *>(data.get()))   return replicateNumber<UInt64>(replicate_offsets);
-    if (typeid_cast<const ColumnInt8 *>(data.get()))      return replicateNumber<Int8>(replicate_offsets);
+    if (typeid_cast<const ColumnInt8 *>(data.get()))     return replicateNumber<Int8>(replicate_offsets);
-    if (typeid_cast<const ColumnInt16 *>(data.get()))     return replicateNumber<Int16>(replicate_offsets);
+    if (typeid_cast<const ColumnInt16 *>(data.get()))    return replicateNumber<Int16>(replicate_offsets);
-    if (typeid_cast<const ColumnInt32 *>(data.get()))     return replicateNumber<Int32>(replicate_offsets);
+    if (typeid_cast<const ColumnInt32 *>(data.get()))    return replicateNumber<Int32>(replicate_offsets);
-    if (typeid_cast<const ColumnInt64 *>(data.get()))     return replicateNumber<Int64>(replicate_offsets);
+    if (typeid_cast<const ColumnInt64 *>(data.get()))    return replicateNumber<Int64>(replicate_offsets);
-    if (typeid_cast<const ColumnFloat32 *>(data.get()))   return replicateNumber<Float32>(replicate_offsets);
+    if (typeid_cast<const ColumnFloat32 *>(data.get()))  return replicateNumber<Float32>(replicate_offsets);
-    if (typeid_cast<const ColumnFloat64 *>(data.get()))   return replicateNumber<Float64>(replicate_offsets);
+    if (typeid_cast<const ColumnFloat64 *>(data.get()))  return replicateNumber<Float64>(replicate_offsets);
-    if (typeid_cast<const ColumnString *>(data.get()))    return replicateString(replicate_offsets);
+    if (typeid_cast<const ColumnString *>(data.get()))   return replicateString(replicate_offsets);
-    if (typeid_cast<const ColumnConst *>(data.get()))     return replicateConst(replicate_offsets);
+    if (typeid_cast<const ColumnConst *>(data.get()))    return replicateConst(replicate_offsets);
-    if (typeid_cast<const ColumnNullable *>(data.get()))  return replicateNullable(replicate_offsets);
+    if (typeid_cast<const ColumnNullable *>(data.get())) return replicateNullable(replicate_offsets);
-    if (typeid_cast<const ColumnTuple *>(data.get()))     return replicateTuple(replicate_offsets);
+    if (typeid_cast<const ColumnTuple *>(data.get()))    return replicateTuple(replicate_offsets);
     return replicateGeneric(replicate_offsets);
 }
 
@@ -781,11 +781,11 @@ ColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const
     ColumnArray & res_ = static_cast<ColumnArray &>(*res);
 
     const ColumnString & src_string = typeid_cast<const ColumnString &>(*data);
-    const ColumnString::Chars_t & src_chars = src_string.getChars();
+    const ColumnString::Chars & src_chars = src_string.getChars();
     const Offsets & src_string_offsets = src_string.getOffsets();
     const Offsets & src_offsets = getOffsets();
 
-    ColumnString::Chars_t & res_chars = typeid_cast<ColumnString &>(res_.getData()).getChars();
+    ColumnString::Chars & res_chars = typeid_cast<ColumnString &>(res_.getData()).getChars();
     Offsets & res_string_offsets = typeid_cast<ColumnString &>(res_.getData()).getOffsets();
     Offsets & res_offsets = res_.getOffsets();
 

dbms/src/Columns/ColumnDecimal.cpp

@@ -21,7 +21,7 @@ namespace ErrorCodes
 }
 
 template <typename T>
-int ColumnDecimal<T>::compareAt(size_t n, size_t m, const IColumn & rhs_, int ) const
+int ColumnDecimal<T>::compareAt(size_t n, size_t m, const IColumn & rhs_, int) const
 {
     auto other = static_cast<const Self &>(rhs_);
     const T & a = data[n];

dbms/src/Columns/ColumnFixedString.cpp

@@ -248,7 +248,7 @@ ColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) con
 
     auto res = ColumnFixedString::create(n);
 
-    Chars_t & res_chars = res->chars;
+    Chars & res_chars = res->chars;
 
     res_chars.resize(n * limit);
 
@@ -274,7 +274,7 @@ ColumnPtr ColumnFixedString::indexImpl(const PaddedPODArray<Type> & indexes, siz
 
     auto res = ColumnFixedString::create(n);
 
-    Chars_t & res_chars = res->chars;
+    Chars & res_chars = res->chars;
 
     res_chars.resize(n * limit);
 
@@ -296,7 +296,7 @@ ColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const
     if (0 == col_size)
         return res;
 
-    Chars_t & res_chars = res->chars;
+    Chars & res_chars = res->chars;
     res_chars.resize(n * offsets.back());
 
     Offset curr_offset = 0;

dbms/src/Columns/ColumnFixedString.h

@@ -17,14 +17,14 @@ class ColumnFixedString final : public COWPtrHelper<IColumn, ColumnFixedString>
 public:
     friend class COWPtrHelper<IColumn, ColumnFixedString>;
 
-    using Chars_t = PaddedPODArray<UInt8>;
+    using Chars = PaddedPODArray<UInt8>;
 
 private:
     /// Bytes of rows, laid in succession. The strings are stored without a trailing zero byte.
     /** NOTE It is required that the offset and type of chars in the object be the same as that of `data in ColumnUInt8`.
       * Used in `packFixed` function (AggregationCommon.h)
       */
-    Chars_t chars;
+    Chars chars;
     /// The size of the rows.
     const size_t n;
 
@@ -138,8 +138,8 @@ public:
 
     /// Specialized part of interface, not from IColumn.
 
-    Chars_t & getChars() { return chars; }
+    Chars & getChars() { return chars; }
-    const Chars_t & getChars() const { return chars; }
+    const Chars & getChars() const { return chars; }
 
     size_t getN() const { return n; }
 };

dbms/src/Columns/ColumnFunction.cpp

@@ -183,7 +183,7 @@ void ColumnFunction::appendArgument(const ColumnWithTypeAndName & column)
     auto index = captured_columns.size();
     if (!column.type->equals(*argumnet_types[index]))
         throw Exception("Cannot capture column " + std::to_string(argumnet_types.size()) +
-                        "because it has incompatible type: got " + column.type->getName() +
+                        " because it has incompatible type: got " + column.type->getName() +
                         ", but " + argumnet_types[index]->getName() + " is expected.", ErrorCodes::LOGICAL_ERROR);
 
     captured_columns.push_back(column);

dbms/src/Columns/ColumnLowCardinality.cpp

@@ -604,7 +604,7 @@ void ColumnLowCardinality::Index::check(size_t /*max_dictionary_size*/)
 void ColumnLowCardinality::Index::checkSizeOfType()
 {
     if (size_of_type != getSizeOfIndexType(*positions, size_of_type))
-        throw Exception("Invalid size of type. Expected "  + toString(8 * size_of_type) +
+        throw Exception("Invalid size of type. Expected " + toString(8 * size_of_type) +
                         ", but positions are " + positions->getName(), ErrorCodes::LOGICAL_ERROR);
 }
 

dbms/src/Columns/ColumnString.cpp

@@ -101,7 +101,7 @@ ColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) co
 
     auto res = ColumnString::create();
 
-    Chars_t & res_chars = res->chars;
+    Chars & res_chars = res->chars;
     Offsets & res_offsets = res->offsets;
 
     filterArraysImpl<UInt8>(chars, offsets, res_chars, res_offsets, filt, result_size_hint);
@@ -126,7 +126,7 @@ ColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const
 
     auto res = ColumnString::create();
 
-    Chars_t & res_chars = res->chars;
+    Chars & res_chars = res->chars;
     Offsets & res_offsets = res->offsets;
 
     if (limit == size)
@@ -202,7 +202,7 @@ ColumnPtr ColumnString::indexImpl(const PaddedPODArray<Type> & indexes, size_t l
 
     auto res = ColumnString::create();
 
-    Chars_t & res_chars = res->chars;
+    Chars & res_chars = res->chars;
     Offsets & res_offsets = res->offsets;
 
     size_t new_chars_size = 0;
@@ -287,7 +287,7 @@ ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const
     if (0 == col_size)
         return res;
 
-    Chars_t & res_chars = res->chars;
+    Chars & res_chars = res->chars;
     Offsets & res_offsets = res->offsets;
     res_chars.reserve(chars.size() / col_size * replicate_offsets.back());
     res_offsets.reserve(replicate_offsets.back());

dbms/src/Columns/ColumnString.h

@@ -19,7 +19,7 @@ namespace DB
 class ColumnString final : public COWPtrHelper<IColumn, ColumnString>
 {
 public:
-    using Chars_t = PaddedPODArray<UInt8>;
+    using Chars = PaddedPODArray<UInt8>;
 
 private:
     friend class COWPtrHelper<IColumn, ColumnString>;
@@ -29,7 +29,7 @@ private:
 
     /// Bytes of strings, placed contiguously.
     /// For convenience, every string ends with terminating zero byte. Note that strings could contain zero bytes in the middle.
-    Chars_t chars;
+    Chars chars;
 
     size_t ALWAYS_INLINE offsetAt(size_t i) const { return i == 0 ? 0 : offsets[i - 1]; }
 
@@ -245,8 +245,8 @@ public:
     bool canBeInsideNullable() const override { return true; }
 
 
-    Chars_t & getChars() { return chars; }
+    Chars & getChars() { return chars; }
-    const Chars_t & getChars() const { return chars; }
+    const Chars & getChars() const { return chars; }
 
     Offsets & getOffsets() { return offsets; }
     const Offsets & getOffsets() const { return offsets; }

dbms/src/Columns/ColumnUnique.h

@@ -81,12 +81,14 @@ public:
     {
         return column_holder->allocatedBytes()
                + index.allocatedBytes()
-               + (cached_null_mask ? cached_null_mask->allocatedBytes() : 0);
+               + (nested_null_mask ? nested_null_mask->allocatedBytes() : 0);
     }
     void forEachSubcolumn(IColumn::ColumnCallback callback) override
     {
         callback(column_holder);
         index.setColumn(getRawColumnPtr());
+        if (is_nullable)
+            nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask);
     }
 
     const UInt64 * tryGetSavedHash() const override { return index.tryGetSavedHash(); }
@@ -100,8 +102,8 @@ private:
     ReverseIndex<UInt64, ColumnType> index;
 
     /// For DataTypeNullable, stores null map.
-    mutable ColumnPtr cached_null_mask;
+    ColumnPtr nested_null_mask;
-    mutable ColumnPtr cached_column_nullable;
+    ColumnPtr nested_column_nullable;
 
     class IncrementalHash
     {
@@ -118,6 +120,9 @@ private:
 
     mutable IncrementalHash hash;
 
+    void createNullMask();
+    void updateNullMask();
+
     static size_t numSpecialValues(bool is_nullable) { return is_nullable ? 2 : 1; }
     size_t numSpecialValues() const { return numSpecialValues(is_nullable); }
 
@@ -148,6 +153,7 @@ ColumnUnique<ColumnType>::ColumnUnique(const ColumnUnique & other)
     , index(numSpecialValues(is_nullable), 0)
 {
     index.setColumn(getRawColumnPtr());
+    createNullMask();
 }
 
 template <typename ColumnType>
@@ -158,6 +164,7 @@ ColumnUnique<ColumnType>::ColumnUnique(const IDataType & type)
     const auto & holder_type = is_nullable ? *static_cast<const DataTypeNullable &>(type).getNestedType() : type;
     column_holder = holder_type.createColumn()->cloneResized(numSpecialValues());
     index.setColumn(getRawColumnPtr());
+    createNullMask();
 }
 
 template <typename ColumnType>
@@ -172,32 +179,51 @@ ColumnUnique<ColumnType>::ColumnUnique(MutableColumnPtr && holder, bool is_nulla
         throw Exception("Holder column for ColumnUnique can't be nullable.", ErrorCodes::ILLEGAL_COLUMN);
 
     index.setColumn(getRawColumnPtr());
+    createNullMask();
+}
+
+template <typename ColumnType>
+void ColumnUnique<ColumnType>::createNullMask()
+{
+    if (is_nullable)
+    {
+        size_t size = getRawColumnPtr()->size();
+        if (!nested_null_mask)
+        {
+            ColumnUInt8::MutablePtr null_mask = ColumnUInt8::create(size, UInt8(0));
+            null_mask->getData()[getNullValueIndex()] = 1;
+            nested_null_mask = std::move(null_mask);
+            nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask);
+        }
+        else
+            throw Exception("Null mask for ColumnUnique is already created.", ErrorCodes::LOGICAL_ERROR);
+    }
+}
+
+template <typename ColumnType>
+void ColumnUnique<ColumnType>::updateNullMask()
+{
+    if (is_nullable)
+    {
+        if (!nested_null_mask)
+            throw Exception("Null mask for ColumnUnique is was not created.", ErrorCodes::LOGICAL_ERROR);
+
+        size_t size = getRawColumnPtr()->size();
+
+        if (nested_null_mask->size() != size)
+        {
+            IColumn & null_mask = nested_null_mask->assumeMutableRef();
+            static_cast<ColumnUInt8 &>(null_mask).getData().resize_fill(size);
+        }
+    }
 }
 
 template <typename ColumnType>
 const ColumnPtr & ColumnUnique<ColumnType>::getNestedColumn() const
 {
     if (is_nullable)
-    {
+        return nested_column_nullable;
-        size_t size = getRawColumnPtr()->size();
-        if (!cached_null_mask)
-        {
-            ColumnUInt8::MutablePtr null_mask = ColumnUInt8::create(size, UInt8(0));
-            null_mask->getData()[getNullValueIndex()] = 1;
-            cached_null_mask = std::move(null_mask);
-            cached_column_nullable = ColumnNullable::create(column_holder, cached_null_mask);
-        }
 
-        if (cached_null_mask->size() != size)
-        {
-            MutableColumnPtr null_mask = (*std::move(cached_null_mask)).mutate();
-            static_cast<ColumnUInt8 &>(*null_mask).getData().resize_fill(size);
-            cached_null_mask = std::move(null_mask);
-            cached_column_nullable = ColumnNullable::create(column_holder, cached_null_mask);
-        }
-
-        return cached_column_nullable;
-    }
     return column_holder;
 }
 
@@ -227,6 +253,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsert(const Field & x)
     if (pos != prev_size)
         column->popBack(1);
 
+    updateNullMask();
+
     return pos;
 }
 
@@ -260,6 +288,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsertData(const char * pos, size_t lengt
         index.insertFromLastRow();
     }
 
+    updateNullMask();
+
     return insertion_point;
 }
 
@@ -288,6 +318,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsertDataWithTerminatingZero(const char
     if (position != prev_size)
         column->popBack(1);
 
+    updateNullMask();
+
     return static_cast<size_t>(position);
 }
 
@@ -305,7 +337,7 @@ StringRef ColumnUnique<ColumnType>::serializeValueIntoArena(size_t n, Arena & ar
 
         size_t nested_size = 0;
 
-        if (n == getNullValueIndex())
+        if (n != getNullValueIndex())
             nested_size = column_holder->serializeValueIntoArena(n, arena, begin).size;
 
         return StringRef(pos, sizeof(null_flag) + nested_size);
@@ -343,6 +375,8 @@ size_t ColumnUnique<ColumnType>::uniqueDeserializeAndInsertFromArena(const char
     if (index_pos != prev_size)
         column->popBack(1);
 
+    updateNullMask();
+
     return static_cast<size_t>(index_pos);
 }
 
@@ -533,6 +567,8 @@ MutableColumnPtr ColumnUnique<ColumnType>::uniqueInsertRangeFrom(const IColumn &
     if (!positions_column)
         throw Exception("Can't find index type for ColumnUnique", ErrorCodes::LOGICAL_ERROR);
 
+    updateNullMask();
+
     return positions_column;
 }
 
@@ -577,6 +613,8 @@ IColumnUnique::IndexesWithOverflow ColumnUnique<ColumnType>::uniqueInsertRangeWi
     if (!positions_column)
         throw Exception("Can't find index type for ColumnUnique", ErrorCodes::LOGICAL_ERROR);
 
+    updateNullMask();
+
     IColumnUnique::IndexesWithOverflow indexes_with_overflow;
     indexes_with_overflow.indexes = std::move(positions_column);
     indexes_with_overflow.overflowed_keys = std::move(overflowed_keys);

dbms/src/Columns/FilterDescription.cpp

@@ -72,7 +72,7 @@ FilterDescription::FilterDescription(const IColumn & column_)
                 ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER);
 
         const NullMap & null_map = nullable_column->getNullMapData();
-        IColumn::Filter & res =  concrete_column->getData();
+        IColumn::Filter & res = concrete_column->getData();
 
         size_t size = res.size();
         for (size_t i = 0; i < size; ++i)

dbms/src/Columns/IColumn.h

@@ -377,7 +377,7 @@ protected:
 };
 
 using ColumnPtr = IColumn::Ptr;
-using MutableColumnPtr  = IColumn::MutablePtr;
+using MutableColumnPtr = IColumn::MutablePtr;
 using Columns = std::vector<ColumnPtr>;
 using MutableColumns = std::vector<MutableColumnPtr>;
 

dbms/src/Columns/ReverseIndex.h

@@ -83,7 +83,7 @@ namespace
 
 
     template <typename IndexType, typename Hash, typename HashTable, typename ColumnType, bool string_hash, bool has_base_index>
-    struct  ReverseIndexHashTableCell
+    struct ReverseIndexHashTableCell
         : public HashTableCell<IndexType, Hash, ReverseIndexHashTableState<ColumnType, string_hash, has_base_index>>
     {
         using Base = HashTableCell<IndexType, Hash, ReverseIndexHashTableState<ColumnType, string_hash, has_base_index>>;
@@ -272,7 +272,7 @@ public:
             auto hash = calcHashes();
             ptr = &hash->getData()[0];
             UInt64 * expected = nullptr;
-            if(saved_hash_ptr.compare_exchange_strong(expected, ptr))
+            if (saved_hash_ptr.compare_exchange_strong(expected, ptr))
                 saved_hash = std::move(hash);
             else
                 ptr = expected;

dbms/src/Common/ClickHouseRevision.cpp

@@ -4,4 +4,5 @@
 namespace ClickHouseRevision
 {
     unsigned get() { return VERSION_REVISION; }
+    unsigned getVersionInteger() { return VERSION_INTEGER; }
 }

dbms/src/Common/ClickHouseRevision.h

@@ -3,4 +3,5 @@
 namespace ClickHouseRevision
 {
     unsigned get();
+    unsigned getVersionInteger();
 }

dbms/src/Common/Config/CMakeLists.txt

@@ -1,9 +1,8 @@
-
 include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 
 add_headers_and_sources(clickhouse_common_config .)
 
 add_library(clickhouse_common_config ${LINK_MODE} ${clickhouse_common_config_headers} ${clickhouse_common_config_sources})
 
-target_link_libraries (clickhouse_common_config clickhouse_common_zookeeper string_utils ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY})
+target_link_libraries(clickhouse_common_config PUBLIC common PRIVATE clickhouse_common_zookeeper string_utils PUBLIC ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY})
-target_include_directories (clickhouse_common_config PRIVATE ${DBMS_INCLUDE_DIR})
+target_include_directories(clickhouse_common_config PUBLIC ${DBMS_INCLUDE_DIR})

dbms/src/Common/Config/ConfigProcessor.cpp

@@ -260,10 +260,10 @@ void ConfigProcessor::doIncludesRecursive(
 
     /// Substitute <layer> for the number extracted from the hostname only if there is an
     /// empty <layer> tag without attributes in the original file.
-    if ( node->nodeName() == "layer" &&
+    if (node->nodeName() == "layer"
-        !node->hasAttributes() &&
+        && !node->hasAttributes()
-        !node->hasChildNodes() &&
+        && !node->hasChildNodes()
-         node->nodeValue().empty())
+        && node->nodeValue().empty())
     {
         NodePtr new_node = config->createTextNode(layerFromHost());
         node->appendChild(new_node);

dbms/src/Common/CounterInFile.h

@@ -18,7 +18,15 @@
 
 #include <common/Types.h>
 
-#define SMALL_READ_WRITE_BUFFER_SIZE 16
+namespace DB
+{
+    namespace ErrorCodes
+    {
+        extern const int CANNOT_OPEN_FILE;
+        extern const int CANNOT_READ_ALL_DATA;
+        extern const int ATTEMPT_TO_READ_AFTER_EOF;
+    }
+}
 
 
 /** Stores a number in the file.
@@ -26,6 +34,9 @@
  */
 class CounterInFile
 {
+private:
+    static inline constexpr size_t SMALL_READ_WRITE_BUFFER_SIZE = 16;
+
 public:
     /// path - the name of the file, including the path
     CounterInFile(const std::string & path_) : path(path_) {}
@@ -56,13 +67,13 @@ public:
 
         int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0666);
         if (-1 == fd)
-            DB::throwFromErrno("Cannot open file " + path);
+            DB::throwFromErrno("Cannot open file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
 
         try
         {
             int flock_ret = flock(fd, LOCK_EX);
             if (-1 == flock_ret)
-                DB::throwFromErrno("Cannot lock file " + path);
+                DB::throwFromErrno("Cannot lock file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
 
             if (!file_doesnt_exists)
             {
@@ -130,7 +141,7 @@ public:
 
         int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0666);
         if (-1 == fd)
-            DB::throwFromErrno("Cannot open file " + path);
+            DB::throwFromErrno("Cannot open file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
 
         try
         {
@@ -178,6 +189,3 @@ private:
     std::string path;
     std::mutex mutex;
 };
-
-
-#undef SMALL_READ_WRITE_BUFFER_SIZE

dbms/src/Common/CurrentMetrics.cpp

@@ -39,7 +39,8 @@
     M(StorageBufferRows, "Number of rows in buffers of Buffer tables") \
     M(StorageBufferBytes, "Number of bytes in buffers of Buffer tables") \
     M(DictCacheRequests, "Number of requests in fly to data sources of dictionaries of cache type.") \
-    M(Revision, "Revision of the server. It is a number incremented for every release or release candidate.") \
+    M(Revision, "Revision of the server. It is a number incremented for every release or release candidate except patch releases.") \
+    M(VersionInteger, "Version of the server in a single integer number in base-1000. For example, version 11.22.33 is translated to 11022033.") \
     M(RWLockWaitingReaders, "Number of threads waiting for read on a table RWLock.") \
     M(RWLockWaitingWriters, "Number of threads waiting for write on a table RWLock.") \
     M(RWLockActiveReaders, "Number of threads holding read lock in a table RWLock.") \

dbms/src/Common/DNSResolver.cpp

@@ -107,7 +107,7 @@ Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host_an
 
 Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host, UInt16 port)
 {
-    return !impl->disable_cache ?  Poco::Net::SocketAddress(impl->cache_host(host), port) : Poco::Net::SocketAddress(host, port);
+    return !impl->disable_cache ? Poco::Net::SocketAddress(impl->cache_host(host), port) : Poco::Net::SocketAddress(host, port);
 }
 
 void DNSResolver::dropCache()

dbms/src/Common/ErrorCodes.cpp

@@ -397,6 +397,10 @@ namespace ErrorCodes
     extern const int CANNOT_UPDATE_COLUMN = 420;
     extern const int CANNOT_ADD_DIFFERENT_AGGREGATE_STATES = 421;
     extern const int UNSUPPORTED_URI_SCHEME = 422;
+    extern const int CANNOT_GETTIMEOFDAY = 423;
+    extern const int CANNOT_LINK = 424;
+    extern const int SYSTEM_ERROR = 425;
+    extern const int NULL_POINTER_DEREFERENCE = 426;
 
     extern const int KEEPER_EXCEPTION = 999;
     extern const int POCO_EXCEPTION = 1000;

dbms/src/Common/Exception.cpp

@@ -52,7 +52,7 @@ std::string errnoToString(int code, int e)
 
 void throwFromErrno(const std::string & s, int code, int e)
 {
-    throw ErrnoException(s + ", " + errnoToString(code, e));
+    throw ErrnoException(s + ", " + errnoToString(code, e), code, e);
 }
 
 void tryLogCurrentException(const char * log_name, const std::string & start_of_message)

dbms/src/Common/Exception.h

@@ -14,19 +14,25 @@ namespace Poco { class Logger; }
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int POCO_EXCEPTION;
+}
+
 class Exception : public Poco::Exception
 {
 public:
     Exception() {}  /// For deferred initialization.
-    Exception(const std::string & msg, int code = 0) : Poco::Exception(msg, code) {}
+    Exception(const std::string & msg, int code) : Poco::Exception(msg, code) {}
-    Exception(const std::string & msg, const std::string & arg, int code = 0) : Poco::Exception(msg, arg, code) {}
+    Exception(const std::string & msg, const Exception & nested_exception, int code)
-    Exception(const std::string & msg, const Exception & exc, int code = 0) : Poco::Exception(msg, exc, code), trace(exc.trace) {}
+        : Poco::Exception(msg, nested_exception, code), trace(nested_exception.trace) {}
-    explicit Exception(const Poco::Exception & exc) : Poco::Exception(exc.displayText()) {}
 
-    const char * name() const throw() override { return "DB::Exception"; }
+    enum CreateFromPocoTag { CreateFromPoco };
-    const char * className() const throw() override { return "DB::Exception"; }
+    Exception(CreateFromPocoTag, const Poco::Exception & exc) : Poco::Exception(exc.displayText(), ErrorCodes::POCO_EXCEPTION) {}
-    DB::Exception * clone() const override { return new DB::Exception(*this); }
+
+    Exception * clone() const override { return new Exception(*this); }
     void rethrow() const override { throw *this; }
+    const char * name() const throw() override { return "DB::Exception"; }
 
     /// Add something to the existing message.
     void addMessage(const std::string & arg) { extendedMessage(arg); }
@@ -35,6 +41,8 @@ public:
 
 private:
     StackTrace trace;
+
+    const char * className() const throw() override { return "DB::Exception"; }
 };
 
 
@@ -42,25 +50,27 @@ private:
 class ErrnoException : public Exception
 {
 public:
-    ErrnoException(const std::string & msg, int code = 0, int saved_errno_ = 0)
+    ErrnoException(const std::string & msg, int code, int saved_errno_)
         : Exception(msg, code), saved_errno(saved_errno_) {}
-    ErrnoException(const std::string & msg, const std::string & arg, int code = 0, int saved_errno_ = 0)
+
-        : Exception(msg, arg, code), saved_errno(saved_errno_) {}
+    ErrnoException * clone() const override { return new ErrnoException(*this); }
-    ErrnoException(const std::string & msg, const Exception & exc, int code = 0, int saved_errno_ = 0)
+    void rethrow() const override { throw *this; }
-        : Exception(msg, exc, code), saved_errno(saved_errno_) {}
 
     int getErrno() const { return saved_errno; }
 
 private:
     int saved_errno;
+
+    const char * name() const throw() override { return "DB::ErrnoException"; }
+    const char * className() const throw() override { return "DB::ErrnoException"; }
 };
 
 
 using Exceptions = std::vector<std::exception_ptr>;
 
 
-std::string errnoToString(int code = 0, int the_errno = errno);
+std::string errnoToString(int code, int the_errno = errno);
-[[noreturn]] void throwFromErrno(const std::string & s, int code = 0, int the_errno = errno);
+[[noreturn]] void throwFromErrno(const std::string & s, int code, int the_errno = errno);
 
 
 /** Try to write an exception to the log (and forget about it).

dbms/src/Common/FieldVisitors.h

@@ -18,7 +18,7 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
-UInt128 stringToUUID(const String & );
+UInt128 stringToUUID(const String &);
 
 
 /** StaticVisitor (and its descendants) - class with overloaded operator() for all types of fields.
@@ -38,16 +38,16 @@ typename std::decay_t<Visitor>::ResultType applyVisitor(Visitor && visitor, F &&
 {
     switch (field.getType())
     {
-        case Field::Types::Null:    return visitor(field.template get<Null>());
+        case Field::Types::Null: return visitor(field.template get<Null>());
-        case Field::Types::UInt64:  return visitor(field.template get<UInt64>());
+        case Field::Types::UInt64: return visitor(field.template get<UInt64>());
         case Field::Types::UInt128: return visitor(field.template get<UInt128>());
-        case Field::Types::Int64:   return visitor(field.template get<Int64>());
+        case Field::Types::Int64: return visitor(field.template get<Int64>());
         case Field::Types::Float64: return visitor(field.template get<Float64>());
-        case Field::Types::String:  return visitor(field.template get<String>());
+        case Field::Types::String: return visitor(field.template get<String>());
-        case Field::Types::Array:   return visitor(field.template get<Array>());
+        case Field::Types::Array: return visitor(field.template get<Array>());
-        case Field::Types::Tuple:   return visitor(field.template get<Tuple>());
+        case Field::Types::Tuple: return visitor(field.template get<Tuple>());
-        case Field::Types::Decimal32:  return visitor(field.template get<DecimalField<Decimal32>>());
+        case Field::Types::Decimal32: return visitor(field.template get<DecimalField<Decimal32>>());
-        case Field::Types::Decimal64:  return visitor(field.template get<DecimalField<Decimal64>>());
+        case Field::Types::Decimal64: return visitor(field.template get<DecimalField<Decimal64>>());
         case Field::Types::Decimal128: return visitor(field.template get<DecimalField<Decimal128>>());
 
         default:

dbms/src/Common/HashTable/HashTable.h

@@ -229,8 +229,8 @@ public:
     void setHasZero() { has_zero = true; }
     void clearHasZero() { has_zero = false; }
 
-    Cell * zeroValue()              { return reinterpret_cast<Cell*>(&zero_value_storage); }
+    Cell * zeroValue()             { return reinterpret_cast<Cell*>(&zero_value_storage); }
-    const Cell * zeroValue() const  { return reinterpret_cast<const Cell*>(&zero_value_storage); }
+    const Cell * zeroValue() const { return reinterpret_cast<const Cell*>(&zero_value_storage); }
 };
 
 template <typename Cell>
@@ -240,8 +240,8 @@ struct ZeroValueStorage<false, Cell>
     void setHasZero() { throw DB::Exception("HashTable: logical error", DB::ErrorCodes::LOGICAL_ERROR); }
     void clearHasZero() {}
 
-    Cell * zeroValue()              { return nullptr; }
+    Cell * zeroValue()             { return nullptr; }
-    const Cell * zeroValue() const  { return nullptr; }
+    const Cell * zeroValue() const { return nullptr; }
 };
 
 

dbms/src/Common/HashTable/SmallTable.h

@@ -3,6 +3,15 @@
 #include <Common/HashTable/HashMap.h>
 
 
+namespace DB
+{
+    namespace ErrorCodes
+    {
+        extern const int INCORRECT_DATA;
+    }
+}
+
+
 /** Replacement of the hash table for a small number (<10) of keys.
   * Implemented as an array with linear search.
   * The array is located inside the object.
@@ -13,7 +22,6 @@
   *  you should check if the table is not full,
   *  and do a `fallback` in this case (for example, use a real hash table).
   */
-
 template
 <
     typename Key,
@@ -86,7 +94,7 @@ public:
                 DB::readVarUInt(size, in);
 
                 if (size > capacity)
-                    throw DB::Exception("Illegal size");
+                    throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
 
                 is_initialized = true;
             }
@@ -306,7 +314,7 @@ public:
         DB::readVarUInt(new_size, rb);
 
         if (new_size > capacity)
-            throw DB::Exception("Illegal size");
+            throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
 
         for (size_t i = 0; i < new_size; ++i)
             buf[i].read(rb);
@@ -324,7 +332,7 @@ public:
         DB::readText(new_size, rb);
 
         if (new_size > capacity)
-            throw DB::Exception("Illegal size");
+            throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
 
         for (size_t i = 0; i < new_size; ++i)
         {

dbms/src/Common/HyperLogLogCounter.h

@@ -310,9 +310,9 @@ public:
     {
         /// Normalizing factor for harmonic mean.
         static constexpr double alpha_m =
-            bucket_count == 2  ? 0.351 :
+            bucket_count == 2 ? 0.351 :
-            bucket_count == 4  ? 0.532 :
+            bucket_count == 4 ? 0.532 :
-            bucket_count == 8  ? 0.626 :
+            bucket_count == 8 ? 0.626 :
             bucket_count == 16 ? 0.673 :
             bucket_count == 32 ? 0.697 :
             bucket_count == 64 ? 0.709 : 0.7213 / (1 + 1.079 / bucket_count);

dbms/src/Common/NetException.h

@@ -6,20 +6,17 @@
 namespace DB
 {
 
-class NetException : public DB::Exception
+class NetException : public Exception
 {
 public:
-    NetException(const std::string & msg, int code = 0) : DB::Exception(msg, code) {}
+    NetException(const std::string & msg, int code) : Exception(msg, code) {}
-    NetException(const std::string & msg, const std::string & arg, int code = 0) : DB::Exception(msg, arg, code) {}
-    NetException(const std::string & msg, const DB::Exception & exc, int code = 0) : DB::Exception(msg, exc, code) {}
 
-    explicit NetException(const DB::Exception & exc) : DB::Exception(exc) {}
+    NetException * clone() const override { return new NetException(*this); }
-    explicit NetException(const Poco::Exception & exc) : DB::Exception(exc.displayText()) {}
+    void rethrow() const override { throw *this; }
 
+private:
     const char * name() const throw() override { return "DB::NetException"; }
     const char * className() const throw() override { return "DB::NetException"; }
-    DB::NetException * clone() const override { return new DB::NetException(*this); }
-    void rethrow() const override { throw *this; }
 };
 
 }

dbms/src/Common/PoolWithFailoverBase.h

@@ -305,7 +305,7 @@ void PoolWithFailoverBase<TNestedPool>::reportError(const Entry & entry)
             return;
         }
     }
-    throw DB::Exception("Can't find pool to report error.");
+    throw DB::Exception("Can't find pool to report error", DB::ErrorCodes::LOGICAL_ERROR);
 }
 
 template <typename TNestedPool>

dbms/src/Common/ProfileEvents.h

@@ -37,7 +37,7 @@ namespace ProfileEvents
 
         /// Global level static initializer
         Counters(Counter * allocated_counters)
-            :  counters(allocated_counters), parent(nullptr), level(VariableContext::Global) {}
+            : counters(allocated_counters), parent(nullptr), level(VariableContext::Global) {}
 
         Counter & operator[] (Event event)
         {

dbms/src/Common/RWLockFIFO.cpp

@@ -134,7 +134,7 @@ RWLockFIFO::LockHandler RWLockFIFO::getLock(RWLockFIFO::Type type, RWLockFIFO::C
     }
 
     /// Wait a notification
-    it_group->cv.wait(lock, [&] () { return it_group == queue.begin(); } );
+    it_group->cv.wait(lock, [&] () { return it_group == queue.begin(); });
 
     it_client->start_time = time(nullptr);
     finalize_metrics();

dbms/src/Common/ShellCommand.cpp

@@ -4,9 +4,11 @@
 #include <dlfcn.h>
 #include <Common/Exception.h>
 #include <Common/ShellCommand.h>
+#include <common/logger_useful.h>
 #include <IO/WriteBufferFromVector.h>
 #include <IO/WriteHelpers.h>
 #include <port/unistd.h>
+#include <csignal>
 
 
 namespace DB
@@ -75,13 +77,27 @@ namespace
 namespace DB
 {
 
+ShellCommand::ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd, bool terminate_in_destructor_)
+    : pid(pid)
+    , terminate_in_destructor(terminate_in_destructor_)
+    , log(&Poco::Logger::get("ShellCommand"))
+    , in(in_fd)
+    , out(out_fd)
+    , err(err_fd) {}
+
 ShellCommand::~ShellCommand()
 {
-    if (!wait_called)
+    if (terminate_in_destructor)
+    {
+        int retcode = kill(pid, SIGTERM);
+        if (retcode != 0)
+            LOG_WARNING(log, "Cannot kill pid " << pid << " errno '" << errnoToString(retcode) << "'");
+    }
+    else if (!wait_called)
         tryWait();
 }
 
-std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only)
+std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor)
 {
     /** Here it is written that with a normal call `vfork`, there is a chance of deadlock in multithreaded programs,
       *  because of the resolving of characters in the shared library
@@ -128,7 +144,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, c
         _exit(int(ReturnCodes::CANNOT_EXEC));
     }
 
-    std::unique_ptr<ShellCommand> res(new ShellCommand(pid, pipe_stdin.write_fd, pipe_stdout.read_fd, pipe_stderr.read_fd));
+    std::unique_ptr<ShellCommand> res(new ShellCommand(pid, pipe_stdin.write_fd, pipe_stdout.read_fd, pipe_stderr.read_fd, terminate_in_destructor));
 
     /// Now the ownership of the file descriptors is passed to the result.
     pipe_stdin.write_fd = -1;
@@ -139,7 +155,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, c
 }
 
 
-std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command, bool pipe_stdin_only)
+std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command, bool pipe_stdin_only, bool terminate_in_destructor)
 {
     /// Arguments in non-constant chunks of memory (as required for `execv`).
     /// Moreover, their copying must be done before calling `vfork`, so after `vfork` do a minimum of things.
@@ -149,11 +165,11 @@ std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command,
 
     char * const argv[] = { argv0.data(), argv1.data(), argv2.data(), nullptr };
 
-    return executeImpl("/bin/sh", argv, pipe_stdin_only);
+    return executeImpl("/bin/sh", argv, pipe_stdin_only, terminate_in_destructor);
 }
 
 
-std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & path, const std::vector<std::string> & arguments)
+std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & path, const std::vector<std::string> & arguments, bool terminate_in_destructor)
 {
     size_t argv_sum_size = path.size() + 1;
     for (const auto & arg : arguments)
@@ -174,7 +190,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & pa
 
     argv[arguments.size() + 1] = nullptr;
 
-    return executeImpl(path.data(), argv.data(), false);
+    return executeImpl(path.data(), argv.data(), false, terminate_in_destructor);
 }
 
 

dbms/src/Common/ShellCommand.h

@@ -28,11 +28,13 @@ class ShellCommand
 private:
     pid_t pid;
     bool wait_called = false;
+    bool terminate_in_destructor;
 
-    ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd)
+    Poco::Logger * log;
-        : pid(pid), in(in_fd), out(out_fd), err(err_fd) {}
 
-    static std::unique_ptr<ShellCommand> executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only);
+    ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd, bool terminate_in_destructor_);
+
+    static std::unique_ptr<ShellCommand> executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor);
 
 public:
     WriteBufferFromFile in;        /// If the command reads from stdin, do not forget to call in.close() after writing all the data there.
@@ -41,11 +43,13 @@ public:
 
     ~ShellCommand();
 
-    /// Run the command using /bin/sh -c
+    /// Run the command using /bin/sh -c.
-    static std::unique_ptr<ShellCommand> execute(const std::string & command, bool pipe_stdin_only = false);
+    /// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
+    static std::unique_ptr<ShellCommand> execute(const std::string & command, bool pipe_stdin_only = false, bool terminate_in_destructor = false);
 
     /// Run the executable with the specified arguments. `arguments` - without argv[0].
-    static std::unique_ptr<ShellCommand> executeDirect(const std::string & path, const std::vector<std::string> & arguments);
+    /// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
+    static std::unique_ptr<ShellCommand> executeDirect(const std::string & path, const std::vector<std::string> & arguments, bool terminate_in_destructor = false);
 
     /// Wait for the process to end, throw an exception if the code is not 0 or if the process was not completed by itself.
     void wait();

dbms/src/Common/StatusFile.cpp

@@ -20,6 +20,14 @@
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int CANNOT_OPEN_FILE;
+    extern const int CANNOT_CLOSE_FILE;
+    extern const int CANNOT_TRUNCATE_FILE;
+    extern const int CANNOT_SEEK_THROUGH_FILE;
+}
+
 
 StatusFile::StatusFile(const std::string & path_)
     : path(path_)
@@ -43,7 +51,7 @@ StatusFile::StatusFile(const std::string & path_)
     fd = ::open(path.c_str(), O_WRONLY | O_CREAT, 0666);
 
     if (-1 == fd)
-        throwFromErrno("Cannot open file " + path);
+        throwFromErrno("Cannot open file " + path, ErrorCodes::CANNOT_OPEN_FILE);
 
     try
     {
@@ -51,16 +59,16 @@ StatusFile::StatusFile(const std::string & path_)
         if (-1 == flock_ret)
         {
             if (errno == EWOULDBLOCK)
-                throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running.");
+                throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running.", ErrorCodes::CANNOT_OPEN_FILE);
             else
-                throwFromErrno("Cannot lock file " + path);
+                throwFromErrno("Cannot lock file " + path, ErrorCodes::CANNOT_OPEN_FILE);
         }
 
         if (0 != ftruncate(fd, 0))
-            throwFromErrno("Cannot ftruncate " + path);
+            throwFromErrno("Cannot ftruncate " + path, ErrorCodes::CANNOT_TRUNCATE_FILE);
 
         if (0 != lseek(fd, 0, SEEK_SET))
-            throwFromErrno("Cannot lseek " + path);
+            throwFromErrno("Cannot lseek " + path, ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
 
         /// Write information about current server instance to the file.
         {
@@ -82,10 +90,10 @@ StatusFile::StatusFile(const std::string & path_)
 StatusFile::~StatusFile()
 {
     if (0 != close(fd))
-        LOG_ERROR(&Logger::get("StatusFile"), "Cannot close file " << path << ", " << errnoToString());
+        LOG_ERROR(&Logger::get("StatusFile"), "Cannot close file " << path << ", " << errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
 
     if (0 != unlink(path.c_str()))
-        LOG_ERROR(&Logger::get("StatusFile"), "Cannot unlink file " << path << ", " << errnoToString());
+        LOG_ERROR(&Logger::get("StatusFile"), "Cannot unlink file " << path << ", " << errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
 }
 
 }

dbms/src/Common/Stopwatch.h

@@ -28,15 +28,15 @@ public:
       */
     Stopwatch(clockid_t clock_type_ = CLOCK_MONOTONIC) : clock_type(clock_type_) { start(); }
 
-    void start()                        { start_ns = nanoseconds(); is_running = true; }
+    void start()                       { start_ns = nanoseconds(); is_running = true; }
-    void stop()                         { stop_ns = nanoseconds(); is_running = false; }
+    void stop()                        { stop_ns = nanoseconds(); is_running = false; }
-    void reset()                        { start_ns = 0; stop_ns = 0; is_running = false; }
+    void reset()                       { start_ns = 0; stop_ns = 0; is_running = false; }
-    void restart()                      { start(); }
+    void restart()                     { start(); }
-    UInt64 elapsed() const              { return elapsedNanoseconds(); }
+    UInt64 elapsed() const             { return elapsedNanoseconds(); }
-    UInt64 elapsedNanoseconds() const   { return is_running ? nanoseconds() - start_ns : stop_ns - start_ns; }
+    UInt64 elapsedNanoseconds() const  { return is_running ? nanoseconds() - start_ns : stop_ns - start_ns; }
-    UInt64 elapsedMicroseconds() const  { return elapsedNanoseconds() / 1000U; }
+    UInt64 elapsedMicroseconds() const { return elapsedNanoseconds() / 1000U; }
-    UInt64 elapsedMilliseconds() const  { return elapsedNanoseconds() / 1000000UL; }
+    UInt64 elapsedMilliseconds() const { return elapsedNanoseconds() / 1000000UL; }
-    double elapsedSeconds() const       { return static_cast<double>(elapsedNanoseconds()) / 1000000000ULL; }
+    double elapsedSeconds() const      { return static_cast<double>(elapsedNanoseconds()) / 1000000000ULL; }
 
 private:
     UInt64 start_ns = 0;
@@ -53,10 +53,10 @@ class AtomicStopwatch
 public:
     AtomicStopwatch(clockid_t clock_type_ = CLOCK_MONOTONIC) : clock_type(clock_type_) { restart(); }
 
-    void restart()                      { start_ns = nanoseconds(); }
+    void restart()                     { start_ns = nanoseconds(); }
-    UInt64 elapsed() const              { return nanoseconds() - start_ns; }
+    UInt64 elapsed() const             { return nanoseconds() - start_ns; }
-    UInt64 elapsedMilliseconds() const  { return elapsed() / 1000000UL; }
+    UInt64 elapsedMilliseconds() const { return elapsed() / 1000000UL; }
-    double elapsedSeconds() const       { return static_cast<double>(elapsed()) / 1000000000ULL; }
+    double elapsedSeconds() const      { return static_cast<double>(elapsed()) / 1000000000ULL; }
 
     /** If specified amount of time has passed, then restarts timer and returns true.
       * Otherwise returns false.
@@ -140,10 +140,10 @@ struct StopwatchRUsage
 {
     StopwatchRUsage() = default;
 
-    void start()    { start_ts = Timestamp::current(); is_running = true; }
+    void start()   { start_ts = Timestamp::current(); is_running = true; }
-    void stop()     { stop_ts = Timestamp::current(); is_running = false; }
+    void stop()    { stop_ts = Timestamp::current(); is_running = false; }
-    void reset()    { start_ts = Timestamp(); stop_ts = Timestamp(); is_running = false; }
+    void reset()   { start_ts = Timestamp(); stop_ts = Timestamp(); is_running = false; }
-    void restart()  { start(); }
+    void restart() { start(); }
 
     UInt64 elapsed(bool count_user = true, bool count_sys = true) const
     {

dbms/src/Common/ThreadProfileEvents.h

@@ -141,9 +141,9 @@ struct TasksStatsCounters
         if (curr.stat.version < TASKSTATS_VERSION)
             return;
 
-        profile_events.increment(ProfileEvents::OSReadChars,  safeDiff(prev.stat.read_char, curr.stat.read_char));
+        profile_events.increment(ProfileEvents::OSReadChars, safeDiff(prev.stat.read_char, curr.stat.read_char));
         profile_events.increment(ProfileEvents::OSWriteChars, safeDiff(prev.stat.write_char, curr.stat.write_char));
-        profile_events.increment(ProfileEvents::OSReadBytes,  safeDiff(prev.stat.read_bytes, curr.stat.read_bytes));
+        profile_events.increment(ProfileEvents::OSReadBytes, safeDiff(prev.stat.read_bytes, curr.stat.read_bytes));
         profile_events.increment(ProfileEvents::OSWriteBytes, safeDiff(prev.stat.write_bytes, curr.stat.write_bytes));
     }
 

dbms/src/Common/XDBCBridgeHelper.h

@@ -68,6 +68,7 @@ protected:
 public:
     using Configuration = Poco::Util::AbstractConfiguration;
 
+    Context & context;
     const Configuration & config;
 
     static constexpr inline auto DEFAULT_HOST = "localhost";
@@ -78,8 +79,8 @@ public:
     static constexpr inline auto IDENTIFIER_QUOTE_HANDLER = "/identifier_quote";
     static constexpr inline auto PING_OK_ANSWER = "Ok.";
 
-    XDBCBridgeHelper(const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_)
+    XDBCBridgeHelper(Context & global_context_, const Poco::Timespan & http_timeout_, const std::string & connection_string_)
-        : http_timeout(http_timeout_), connection_string(connection_string_), config(config_)
+        : http_timeout(http_timeout_), connection_string(connection_string_), context(global_context_), config(context.getConfigRef())
     {
         size_t bridge_port = config.getUInt(BridgeHelperMixin::configPrefix() + ".port", DEFAULT_PORT);
         std::string bridge_host = config.getString(BridgeHelperMixin::configPrefix() + ".host", DEFAULT_HOST);
@@ -210,7 +211,8 @@ private:
     /* Contains logic for instantiation of the bridge instance */
     void startBridge() const
     {
-        BridgeHelperMixin::startBridge(config, log, http_timeout);
+        auto cmd = BridgeHelperMixin::startBridge(config, log, http_timeout);
+        context.addXDBCBridgeCommand(std::move(cmd));
     }
 };
 
@@ -230,7 +232,7 @@ struct JDBCBridgeMixin
         return "JDBC";
     }
 
-    static void startBridge(const Poco::Util::AbstractConfiguration &, const Poco::Logger *, const Poco::Timespan &)
+    static std::unique_ptr<ShellCommand> startBridge(const Poco::Util::AbstractConfiguration &, const Poco::Logger *, const Poco::Timespan &)
     {
         throw Exception("jdbc-bridge is not running. Please, start it manually", ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING);
     }
@@ -253,11 +255,13 @@ struct ODBCBridgeMixin
         return "ODBC";
     }
 
-    static void startBridge(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout)
+    static std::unique_ptr<ShellCommand> startBridge(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout)
     {
         /// Path to executable folder
         Poco::Path path{config.getString("application.dir", "/usr/bin")};
 
+
+        std::vector<std::string> cmd_args;
         path.setFileName(
 #if CLICKHOUSE_SPLIT_BINARY
             "clickhouse-odbc-bridge"
@@ -268,34 +272,35 @@ struct ODBCBridgeMixin
 
         std::stringstream command;
 
-        command << path.toString() <<
+#if !CLICKHOUSE_SPLIT_BINARY
-#if CLICKHOUSE_SPLIT_BINARY
+        cmd_args.push_back("odbc-bridge");
-            " "
-#else
-            " odbc-bridge "
 #endif
-            ;
 
-        command << "--http-port " << config.getUInt(configPrefix() + ".port", DEFAULT_PORT) << ' ';
+        cmd_args.push_back("--http-port");
-        command << "--listen-host " << config.getString(configPrefix() + ".listen_host", XDBCBridgeHelper<ODBCBridgeMixin>::DEFAULT_HOST)
+        cmd_args.push_back(std::to_string(config.getUInt(configPrefix() + ".port", DEFAULT_PORT)));
-                << ' ';
+        cmd_args.push_back("--listen-host");
-        command << "--http-timeout " << http_timeout.totalMicroseconds() << ' ';
+        cmd_args.push_back(config.getString(configPrefix() + ".listen_host", XDBCBridgeHelper<ODBCBridgeMixin>::DEFAULT_HOST));
+        cmd_args.push_back("--http-timeout");
+        cmd_args.push_back(std::to_string(http_timeout.totalMicroseconds()));
         if (config.has("logger." + configPrefix() + "_log"))
-            command << "--log-path " << config.getString("logger." + configPrefix() + "_log") << ' ';
+        {
+            cmd_args.push_back("--log-path");
+            cmd_args.push_back(config.getString("logger." + configPrefix() + "_log"));
+        }
         if (config.has("logger." + configPrefix() + "_errlog"))
-            command << "--err-log-path " << config.getString("logger." + configPrefix() + "_errlog") << ' ';
+        {
+            cmd_args.push_back("--err-log-path");
+            cmd_args.push_back(config.getString("logger." + configPrefix() + "_errlog"));
+        }
         if (config.has("logger." + configPrefix() + "_level"))
-            command << "--log-level " << config.getString("logger." + configPrefix() + "_level") << ' ';
+        {
-        command << "&"; /// we don't want to wait this process
+            cmd_args.push_back("--log-level");
+            cmd_args.push_back(config.getString("logger." + configPrefix() + "_level"));
+        }
 
-        auto command_str = command.str();
+        LOG_TRACE(log, "Starting " + serviceAlias());
 
-        std::cerr << command_str << std::endl;
+        return ShellCommand::executeDirect(path.toString(), cmd_args, true);
-
-        LOG_TRACE(log, "Starting " + serviceAlias() + " with command: " << command_str);
-
-        auto cmd = ShellCommand::execute(command_str);
-        cmd->wait();
     }
 };
 }