Merge branch 'master' into master

.clang-format

@@ -1,6 +1,6 @@
 BasedOnStyle: WebKit
 Language: Cpp
-AlignAfterOpenBracket: false
+AlignAfterOpenBracket: AlwaysBreak
 BreakBeforeBraces: Custom
 BraceWrapping:
     AfterClass: true
@@ -25,7 +25,7 @@ Standard: Cpp11
 PointerAlignment: Middle
 MaxEmptyLinesToKeep: 2
 KeepEmptyLinesAtTheStartOfBlocks: false
-AllowShortFunctionsOnASingleLine: Empty
+AllowShortFunctionsOnASingleLine: InlineOnly
 AlwaysBreakTemplateDeclarations: true
 IndentCaseLabels: true
 SpaceAfterTemplateKeyword: true

.gitignore

@@ -11,6 +11,7 @@
 
 /build
 /build_*
+/build-*
 /docs/build
 /docs/edit
 /docs/tools/venv/

.gitmodules

@@ -52,3 +52,6 @@
 [submodule "contrib/boost"]
 	path = contrib/boost
 	url = https://github.com/ClickHouse-Extras/boost-extra.git
+[submodule "contrib/base64"]
+	path = contrib/base64
+	url = https://github.com/aklomp/base64.git

CHANGELOG.md

@@ -1,3 +1,16 @@
+## ClickHouse release 18.14.15, 2018-11-21
+
+### Bug fixes:
+* The size of memory chunk was overestimated while deserializing the column of type `Array(String)` that leads to "Memory limit exceeded" errors. The issue appeared in version 18.12.13. [#3589](https://github.com/yandex/ClickHouse/issues/3589)
+
+## ClickHouse release 18.14.14, 2018-11-20
+
+### Bug fixes:
+* Fixed `ON CLUSTER` queries when cluster configured as secure (flag `<secure>`). [#3599](https://github.com/yandex/ClickHouse/pull/3599)
+
+### Build changes:
+* Fixed problems (llvm-7 from system, macos) [#3582](https://github.com/yandex/ClickHouse/pull/3582)
+
 ## ClickHouse release 18.14.11, 2018-10-29
 
 ### Bug fixes:

CHANGELOG_RU.md

@@ -1,3 +1,16 @@
+## ClickHouse release 18.14.15, 2018-11-21
+
+### Исправления ошибок:
+* При чтении столбцов типа `Array(String)`, размер требуемого куска памяти оценивался слишком большим, что приводило к исключению "Memory limit exceeded" при выполнении запроса. Ошибка появилась в версии 18.12.13. [#3589](https://github.com/yandex/ClickHouse/issues/3589)
+
+## ClickHouse release 18.14.14, 2018-11-20
+
+### Исправления ошибок:
+* Исправлена работа запросов `ON CLUSTER` в случае, когда в конфигурации кластера включено шифрование (флаг `<secure>`). [#3599](https://github.com/yandex/ClickHouse/pull/3599)
+
+### Улучшения процесса сборки ClickHouse:
+* Испрпавлены проблемы сборки (llvm-7 из системы, macos) [#3582](https://github.com/yandex/ClickHouse/pull/3582)
+
 ## ClickHouse release 18.14.13, 2018-11-08
 
 ### Исправления ошибок:

CMakeLists.txt

@@ -262,6 +262,7 @@ include (cmake/find_llvm.cmake)
 include (cmake/find_cpuid.cmake)
 include (cmake/find_hdfs3.cmake)
 include (cmake/find_consistent-hashing.cmake)
+include (cmake/find_base64.cmake)
 if (ENABLE_TESTS)
     include (cmake/find_gtest.cmake)
 endif ()

README.md

@@ -2,11 +2,10 @@
 
 ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time.
 
-🎤🥂 **ClickHouse Meetup in [Amsterdam on November 15](https://events.yandex.com/events/meetings/15-11-2018/)** 🍰🔥🐻
-
 ## Useful Links
 
 * [Official website](https://clickhouse.yandex/) has quick high-level overview of ClickHouse on main page.
 * [Tutorial](https://clickhouse.yandex/tutorial.html) shows how to set up and query small ClickHouse cluster.
 * [Documentation](https://clickhouse.yandex/docs/en/) provides more in-depth information.
+* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
 * [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any.

cmake/find_base64.cmake

@@ -0,0 +1,12 @@
+option (ENABLE_BASE64 "Enable base64" ON)
+
+if (ENABLE_BASE64)
+    if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64")
+        message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
+    else()
+        set (BASE64_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64/include)
+        set (BASE64_LIBRARY base64)
+        set (USE_BASE64 1)
+    endif()
+endif ()
+

cmake/find_odbc.cmake

@@ -82,6 +82,10 @@ if (ENABLE_ODBC)
             ODBC_INCLUDE_DIRECTORIES
             ODBC_LIBRARIES)
 
+        if (USE_STATIC_LIBRARIES)
+            list(APPEND ODBC_LIBRARIES ${LTDL_LIBRARY})
+        endif ()
+
         mark_as_advanced(ODBC_FOUND ODBC_LIBRARIES ODBC_INCLUDE_DIRECTORIES)
     endif ()
 endif ()

cmake/find_poco.cmake

@@ -93,8 +93,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
     endif ()
 
     if (OPENSSL_FOUND AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL))
-        set (Poco_NetSSL_LIBRARY PocoNetSSL)
+        set (Poco_NetSSL_LIBRARY PocoNetSSL ${OPENSSL_LIBRARIES})
-        set (Poco_Crypto_LIBRARY PocoCrypto)
+        set (Poco_Crypto_LIBRARY PocoCrypto ${OPENSSL_LIBRARIES})
     endif ()
 
     if (USE_STATIC_LIBRARIES AND USE_INTERNAL_ZLIB_LIBRARY)

cmake/lib_name.cmake

@@ -1,5 +1,4 @@
 set(DIVIDE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libdivide)
-set(CITYHASH_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/cityhash102/include)
 set(COMMON_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/libs/libcommon/include ${ClickHouse_BINARY_DIR}/libs/libcommon/include)
 set(DBMS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/dbms/src ${ClickHouse_BINARY_DIR}/dbms/src)
 set(DOUBLE_CONVERSION_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/double-conversion)

cmake/print_include_directories.cmake

@@ -10,6 +10,9 @@ list(APPEND dirs ${dirs1})
 get_property (dirs1 TARGET common PROPERTY INCLUDE_DIRECTORIES)
 list(APPEND dirs ${dirs1})
 
+get_property (dirs1 TARGET cityhash PROPERTY INCLUDE_DIRECTORIES)
+list(APPEND dirs ${dirs1})
+
 if (USE_INTERNAL_BOOST_LIBRARY)
     get_property (dirs1 TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY} PROPERTY INCLUDE_DIRECTORIES)
     list(APPEND dirs ${dirs1})

cmake/test_cpu.cmake

@@ -45,6 +45,38 @@ if (HAVE_SSE42)
     set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
 endif ()
 
+set (TEST_FLAG "-mssse3")
+set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
+check_cxx_source_compiles("
+    #include <tmmintrin.h>
+    int main() {
+        __m64 a = _mm_abs_pi8(__m64());
+        (void)a;
+        return 0;
+    }
+" HAVE_SSSE3)
+
+set (TEST_FLAG "-mavx")
+set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
+check_cxx_source_compiles("
+    #include <immintrin.h>
+    int main() {
+        auto a = _mm256_insert_epi8(__m256i(), 0, 0);
+        (void)a;
+        return 0;
+    }
+" HAVE_AVX)
+
+set (TEST_FLAG "-mavx2")
+set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
+check_cxx_source_compiles("
+    #include <immintrin.h>
+    int main() {
+        auto a = _mm256_add_epi16(__m256i(), __m256i());
+        (void)a;
+        return 0;
+    }
+" HAVE_AVX2)
 
 # gcc -dM -E -mpopcnt - < /dev/null | sort > gcc-dump-popcnt
 #define __POPCNT__ 1
@@ -65,5 +97,3 @@ if (HAVE_POPCNT AND NOT ARCH_AARCH64)
 endif ()
 
 cmake_pop_check_state ()
-
-# TODO: add here sse3 test if you want use it

contrib/CMakeLists.txt

@@ -200,3 +200,7 @@ if (USE_INTERNAL_HDFS3_LIBRARY)
     endif ()
     add_subdirectory(libhdfs3-cmake)
 endif ()
+
+if (USE_BASE64)
+    add_subdirectory (base64-cmake)
+endif()

contrib/base64

@@ -0,0 +1 @@
+Subproject commit a27c565d1b6c676beaf297fe503c4518185666f7

contrib/base64-cmake/.gitignore

@@ -0,0 +1 @@
+config.h

contrib/base64-cmake/CMakeLists.txt

@@ -0,0 +1,52 @@
+SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64)
+
+set(base64_compile_instructions "")
+LIST(LENGTH base64_compile_instructions 0)
+macro(cast_to_bool var instruction)
+    if (HAVE_${var})
+        set(base64_${var} 1)
+        set(base64_${var}_opt ${instruction})
+    else()
+        set(base64_${var} 0)
+    endif()
+endmacro()
+
+cast_to_bool(SSSE3 "-mssse3")
+cast_to_bool(SSE41 "-msse4.1")
+cast_to_bool(SSE42 "-msse4.2")
+cast_to_bool(AVX   "-mavx")
+cast_to_bool(AVX2  "-mavx2")
+
+# write config.h file, to include it in application
+file(READ config-header.tpl header)
+file(WRITE config.h ${header})
+file(APPEND config.h "#define HAVE_SSSE3                 ${base64_SSSE3}\n")
+file(APPEND config.h "#define HAVE_SSE41                 ${base64_SSE41}\n")
+file(APPEND config.h "#define HAVE_SSE42                 ${base64_SSE42}\n")
+file(APPEND config.h "#define HAVE_AVX                   ${base64_AVX}\n")
+file(APPEND config.h "#define HAVE_AVX2                  ${base64_AVX2}\n")
+
+set(HAVE_FAST_UNALIGNED_ACCESS 0)
+if (${base64_SSSE3} OR ${base64_SSE41} OR ${base64_SSE42} OR ${base64_AVX} OR ${base64_AVX2})
+    set(HAVE_FAST_UNALIGNED_ACCESS 1)
+endif ()
+
+file(APPEND config.h "#define HAVE_FAST_UNALIGNED_ACCESS " ${HAVE_FAST_UNALIGNED_ACCESS} "\n")
+
+add_library(base64 ${LINK_MODE}
+        ${LIBRARY_DIR}/lib/lib.c
+        ${LIBRARY_DIR}/lib/codec_choose.c
+        ${LIBRARY_DIR}/lib/arch/avx/codec.c
+        ${LIBRARY_DIR}/lib/arch/avx2/codec.c
+        ${LIBRARY_DIR}/lib/arch/generic/codec.c
+        ${LIBRARY_DIR}/lib/arch/neon32/codec.c
+        ${LIBRARY_DIR}/lib/arch/neon64/codec.c
+        ${LIBRARY_DIR}/lib/arch/sse41/codec.c
+        ${LIBRARY_DIR}/lib/arch/sse42/codec.c
+        ${LIBRARY_DIR}/lib/arch/ssse3/codec.c
+
+        ${LIBRARY_DIR}/lib/codecs.h
+        config.h)
+
+target_compile_options(base64 PRIVATE ${base64_SSSE3_opt} ${base64_SSE41_opt} ${base64_SSE42_opt} ${base64_AVX_opt} ${base64_AVX2_opt})
+target_include_directories(base64 PRIVATE ${LIBRARY_DIR}/include .)

contrib/base64-cmake/config-header.tpl

@@ -0,0 +1,2 @@
+#define HAVE_NEON32                0
+#define HAVE_NEON64                0

contrib/cityhash102/CMakeLists.txt

@@ -1,6 +1,5 @@
 add_library(cityhash
 	src/city.cc
-
 	include/citycrc.h
 	include/city.h
 	src/config.h)

contrib/poco

@@ -1 +1 @@
-Subproject commit 566162b324e0b73eb360a1cd38077c9287cc1106
+Subproject commit 20c1d877773b6a672f1bbfe3290dfea42a117ed5

contrib/ssl

@@ -1 +1 @@
-Subproject commit 919f6f1331d500bfdd26f8bbbf88e92c0119879b
+Subproject commit dbbbcdbbd17785566f8f9c107b714f9e213d7293

dbms/CMakeLists.txt

@@ -119,7 +119,7 @@ endif ()
 
 if (USE_EMBEDDED_COMPILER)
     llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
-    target_link_libraries (dbms ${REQUIRED_LLVM_LIBRARIES})
+    target_link_libraries (dbms PRIVATE ${REQUIRED_LLVM_LIBRARIES})
     target_include_directories (dbms SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
 endif ()
 
@@ -150,34 +150,49 @@ if (NOT ARCH_ARM AND CPUID_LIBRARY)
 endif()
 
 target_link_libraries (clickhouse_common_io
+        PUBLIC
     common
+        PRIVATE
     string_utils
     widechar_width
     ${LINK_LIBRARIES_ONLY_ON_X86_64}
     ${LZ4_LIBRARY}
     ${ZSTD_LIBRARY}
     ${DOUBLE_CONVERSION_LIBRARIES}
+    pocoext
+        PUBLIC
     ${Poco_Net_LIBRARY}
     ${Poco_Util_LIBRARY}
     ${Poco_Foundation_LIBRARY}
+    ${RE2_LIBRARY}
+    ${RE2_ST_LIBRARY}
+    ${CITYHASH_LIBRARIES}
+        PRIVATE
     ${ZLIB_LIBRARIES}
     ${EXECINFO_LIBRARY}
     ${ELF_LIBRARY}
+        PUBLIC
     ${Boost_SYSTEM_LIBRARY}
+        PRIVATE
     apple_rt
     ${CMAKE_DL_LIBS}
     ${HDFS3_LIBRARY}
 )
 
 target_link_libraries (dbms
+        PRIVATE
     clickhouse_parsers
     clickhouse_common_config
+        PUBLIC
     clickhouse_common_io
+    pocoext
+        PUBLIC
     ${MYSQLXX_LIBRARY}
-    ${RE2_LIBRARY}
+        PRIVATE
-    ${RE2_ST_LIBRARY}
     ${BTRIE_LIBRARIES}
     ${Boost_PROGRAM_OPTIONS_LIBRARY}
+        PUBLIC
+    ${Boost_SYSTEM_LIBRARY}
 )
 
 if (NOT USE_INTERNAL_RE2_LIBRARY)
@@ -194,8 +209,8 @@ if (Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
 endif()
 
 if (USE_POCO_SQLODBC)
-    target_link_libraries (clickhouse_common_io ${Poco_SQL_LIBRARY})
+    target_link_libraries (clickhouse_common_io PRIVATE ${Poco_SQL_LIBRARY})
-    target_link_libraries (dbms ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
+    target_link_libraries (dbms PRIVATE ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
     if (NOT USE_INTERNAL_POCO_LIBRARY)
         target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIR})
         target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR} PUBLIC ${Poco_SQL_INCLUDE_DIR})
@@ -209,48 +224,44 @@ if (Poco_Data_FOUND)
 endif()
 
 if (USE_POCO_DATAODBC)
-    target_link_libraries (clickhouse_common_io ${Poco_Data_LIBRARY})
+    target_link_libraries (clickhouse_common_io PRIVATE ${Poco_Data_LIBRARY})
-    target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
+    target_link_libraries (dbms PRIVATE ${Poco_DataODBC_LIBRARY})
     if (NOT USE_INTERNAL_POCO_LIBRARY)
         target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
     endif()
 endif()
 
 if (USE_POCO_MONGODB)
-    target_link_libraries (dbms ${Poco_MongoDB_LIBRARY})
+    target_link_libraries (dbms PRIVATE ${Poco_MongoDB_LIBRARY})
 endif()
 
 if (USE_POCO_NETSSL)
-    target_link_libraries (clickhouse_common_io ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
+    target_link_libraries (clickhouse_common_io PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
-    target_link_libraries (dbms ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
+    target_link_libraries (dbms PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
 endif()
 
-target_link_libraries (dbms ${Poco_Foundation_LIBRARY})
+target_link_libraries (dbms PRIVATE ${Poco_Foundation_LIBRARY})
 
 if (USE_ICU)
-    target_link_libraries (dbms ${ICU_LIBS})
+    target_link_libraries (dbms PRIVATE ${ICU_LIBS})
     target_include_directories (dbms SYSTEM PRIVATE ${ICU_INCLUDE_DIR})
 endif ()
 
 if (USE_CAPNP)
-    target_link_libraries (dbms ${CAPNP_LIBRARY})
+    target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARY})
     if (NOT USE_INTERNAL_CAPNP_LIBRARY)
         target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR})
     endif ()
 endif ()
 
 if (USE_RDKAFKA)
-    target_link_libraries (dbms ${RDKAFKA_LIBRARY})
+    target_link_libraries (dbms PRIVATE ${RDKAFKA_LIBRARY})
     if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
         target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
     endif ()
 endif ()
 
-target_link_libraries(dbms ${OPENSSL_CRYPTO_LIBRARY})
+target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY} Threads::Threads)
-
-target_link_libraries (dbms
-    Threads::Threads
-)
 
 target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
 target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
@@ -286,6 +297,6 @@ if (ENABLE_TESTS)
     # attach all dbms gtest sources
     grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms dbms_gtest_sources)
     add_executable(unit_tests_dbms ${dbms_gtest_sources})
-    target_link_libraries(unit_tests_dbms gtest_main dbms)
+    target_link_libraries(unit_tests_dbms PRIVATE gtest_main dbms clickhouse_common_zookeeper)
     add_check(unit_tests_dbms)
 endif ()

dbms/cmake/version.cmake

@@ -22,3 +22,5 @@ endif ()
 set (VERSION_NAME "${PROJECT_NAME}" CACHE STRING "")
 set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}" CACHE STRING "")
 set (VERSION_SO "${VERSION_STRING}" CACHE STRING "")
+
+math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000")

dbms/programs/CMakeLists.txt

@@ -48,45 +48,45 @@ else ()
         link_directories (${LLVM_LIBRARY_DIRS})
     endif ()
     add_executable (clickhouse main.cpp)
-    target_link_libraries (clickhouse clickhouse_common_io)
+    target_link_libraries (clickhouse PRIVATE clickhouse_common_io)
     target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
     target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 
     if (ENABLE_CLICKHOUSE_SERVER)
-        target_link_libraries (clickhouse clickhouse-server-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-server-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_CLIENT)
-        target_link_libraries (clickhouse clickhouse-client-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-client-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_LOCAL)
-        target_link_libraries (clickhouse clickhouse-local-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-local-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_BENCHMARK)
-        target_link_libraries (clickhouse clickhouse-benchmark-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-benchmark-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_PERFORMANCE)
-        target_link_libraries (clickhouse clickhouse-performance-test-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-performance-test-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_COPIER)
-        target_link_libraries (clickhouse clickhouse-copier-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-copier-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
-        target_link_libraries (clickhouse clickhouse-extract-from-config-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-extract-from-config-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_COMPRESSOR)
-        target_link_libraries (clickhouse clickhouse-compressor-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-compressor-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_FORMAT)
-        target_link_libraries (clickhouse clickhouse-format-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-format-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_OBFUSCATOR)
-        target_link_libraries (clickhouse clickhouse-obfuscator-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-obfuscator-lib)
     endif ()
     if (USE_EMBEDDED_COMPILER)
-        target_link_libraries (clickhouse clickhouse-compiler-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-compiler-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
-        target_link_libraries (clickhouse clickhouse-odbc-bridge-lib)
+        target_link_libraries (clickhouse PRIVATE clickhouse-odbc-bridge-lib)
     endif()
 
     set (CLICKHOUSE_BUNDLE)

dbms/programs/benchmark/Benchmark.cpp

@@ -42,10 +42,8 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int POCO_EXCEPTION;
-    extern const int STD_EXCEPTION;
-    extern const int UNKNOWN_EXCEPTION;
     extern const int BAD_ARGUMENTS;
+    extern const int EMPTY_DATA_PASSED;
 }
 
 class Benchmark
@@ -170,7 +168,7 @@ private:
         }
 
         if (queries.empty())
-            throw Exception("Empty list of queries.");
+            throw Exception("Empty list of queries.", ErrorCodes::EMPTY_DATA_PASSED);
 
         std::cerr << "Loaded " << queries.size() << " queries.\n";
     }

dbms/programs/benchmark/CMakeLists.txt

@@ -1,8 +1,8 @@
 add_library (clickhouse-benchmark-lib ${LINK_MODE} Benchmark.cpp)
-target_link_libraries (clickhouse-benchmark-lib clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-benchmark-lib PRIVATE clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
 target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-benchmark clickhouse-benchmark.cpp)
-    target_link_libraries (clickhouse-benchmark clickhouse-benchmark-lib clickhouse_aggregate_functions)
+    target_link_libraries (clickhouse-benchmark PRIVATE clickhouse-benchmark-lib clickhouse_aggregate_functions)
 endif ()

dbms/programs/clang/CMakeLists.txt

@@ -6,9 +6,9 @@ if (CLICKHOUSE_SPLIT_BINARY)
     if (USE_EMBEDDED_COMPILER)
         link_directories (${LLVM_LIBRARY_DIRS})
         add_executable (clickhouse-clang clickhouse-clang.cpp)
-        target_link_libraries (clickhouse-clang clickhouse-compiler-lib)
+        target_link_libraries (clickhouse-clang PRIVATE clickhouse-compiler-lib)
         add_executable (clickhouse-lld clickhouse-lld.cpp)
-        target_link_libraries (clickhouse-lld clickhouse-compiler-lib)
+        target_link_libraries (clickhouse-lld PRIVATE clickhouse-compiler-lib)
         install (TARGETS clickhouse-clang clickhouse-lld RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
     endif ()
 endif ()

dbms/programs/client/CMakeLists.txt

@@ -1,13 +1,12 @@
 add_library (clickhouse-client-lib ${LINK_MODE} Client.cpp)
-target_link_libraries (clickhouse-client-lib clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-client-lib PRIVATE clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
 if (READLINE_INCLUDE_DIR)
     target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
 endif ()
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-client clickhouse-client.cpp)
-    target_link_libraries (clickhouse-client clickhouse-client-lib)
+    target_link_libraries (clickhouse-client PRIVATE clickhouse-client-lib)
 endif ()
 
 install (FILES clickhouse-client.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-client COMPONENT clickhouse-client RENAME config.xml)
-

dbms/programs/client/Client.cpp

@@ -87,9 +87,6 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int POCO_EXCEPTION;
-    extern const int STD_EXCEPTION;
-    extern const int UNKNOWN_EXCEPTION;
     extern const int NETWORK_ERROR;
     extern const int NO_DATA_TO_INSERT;
     extern const int BAD_ARGUMENTS;

dbms/programs/client/Suggest.h

@@ -184,7 +184,7 @@ public:
             }
             catch (...)
             {
-                std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false) << "\n";
+                std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n";
             }
 
             /// Note that keyword suggestions are available even if we cannot load data from server.

dbms/programs/compressor/CMakeLists.txt

@@ -1,8 +1,8 @@
 add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp)
-target_link_libraries (clickhouse-compressor-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     # Also in utils
     add_executable (clickhouse-compressor clickhouse-compressor.cpp)
-    target_link_libraries (clickhouse-compressor clickhouse-compressor-lib)
+    target_link_libraries (clickhouse-compressor PRIVATE clickhouse-compressor-lib)
 endif ()

dbms/programs/copier/CMakeLists.txt

@@ -1,5 +1,5 @@
 add_library (clickhouse-copier-lib ${LINK_MODE} ClusterCopier.cpp)
-target_link_libraries (clickhouse-copier-lib clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions)
+target_link_libraries (clickhouse-copier-lib PRIVATE clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions daemon)
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-copier clickhouse-copier.cpp)

dbms/programs/extract-from-config/CMakeLists.txt

@@ -1,7 +1,7 @@
 add_library (clickhouse-extract-from-config-lib ${LINK_MODE} ExtractFromConfig.cpp)
-target_link_libraries (clickhouse-extract-from-config-lib clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-extract-from-config-lib PRIVATE clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-extract-from-config clickhouse-extract-from-config.cpp)
-    target_link_libraries (clickhouse-extract-from-config clickhouse-extract-from-config-lib)
+    target_link_libraries (clickhouse-extract-from-config PRIVATE clickhouse-extract-from-config-lib)
 endif ()

dbms/programs/format/CMakeLists.txt

@@ -1,6 +1,6 @@
 add_library (clickhouse-format-lib ${LINK_MODE} Format.cpp)
-target_link_libraries (clickhouse-format-lib dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-format-lib PRIVATE dbms clickhouse_common_io clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY})
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-format clickhouse-format.cpp)
-    target_link_libraries (clickhouse-format clickhouse-format-lib)
+    target_link_libraries (clickhouse-format PRIVATE clickhouse-format-lib)
 endif ()

dbms/programs/local/CMakeLists.txt

@@ -1,7 +1,7 @@
 add_library (clickhouse-local-lib ${LINK_MODE} LocalServer.cpp)
-target_link_libraries (clickhouse-local-lib clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-local-lib PRIVATE clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-local clickhouse-local.cpp)
-    target_link_libraries (clickhouse-local clickhouse-local-lib)
+    target_link_libraries (clickhouse-local PRIVATE clickhouse-local-lib)
 endif ()

dbms/programs/obfuscator/CMakeLists.txt

@@ -1,8 +1,8 @@
 add_library (clickhouse-obfuscator-lib ${LINK_MODE} Obfuscator.cpp)
-target_link_libraries (clickhouse-obfuscator-lib dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-obfuscator-lib PRIVATE dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-obfuscator clickhouse-obfuscator.cpp)
     set_target_properties(clickhouse-obfuscator PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
-    target_link_libraries (clickhouse-obfuscator clickhouse-obfuscator-lib)
+    target_link_libraries (clickhouse-obfuscator PRIVATE clickhouse-obfuscator-lib)
 endif ()

dbms/programs/odbc-bridge/CMakeLists.txt

@@ -9,23 +9,23 @@ add_library (clickhouse-odbc-bridge-lib ${LINK_MODE}
     validateODBCConnectionString.cpp
 )
 
-target_link_libraries (clickhouse-odbc-bridge-lib clickhouse_common_io daemon dbms)
+target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE clickhouse_common_io daemon dbms)
 target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
 
 if (USE_POCO_SQLODBC)
-    target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQLODBC_LIBRARY})
+    target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQLODBC_LIBRARY})
     target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR})
 endif ()
 if (Poco_SQL_FOUND)
-    target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQL_LIBRARY})
+    target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQL_LIBRARY})
 endif ()
 
 if (USE_POCO_DATAODBC)
-    target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_DataODBC_LIBRARY})
+    target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_DataODBC_LIBRARY})
     target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
 endif()
 if (Poco_Data_FOUND)
-    target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_Data_LIBRARY})
+    target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_Data_LIBRARY})
 endif ()
 
 
@@ -35,5 +35,5 @@ endif ()
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-odbc-bridge odbc-bridge.cpp)
-    target_link_libraries (clickhouse-odbc-bridge clickhouse-odbc-bridge-lib)
+    target_link_libraries (clickhouse-odbc-bridge PRIVATE clickhouse-odbc-bridge-lib)
 endif ()

dbms/programs/odbc-bridge/tests/CMakeLists.txt

@@ -1,2 +1,2 @@
 add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp)
-target_link_libraries (validate-odbc-connection-string clickhouse-odbc-bridge-lib)
+target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse-odbc-bridge-lib clickhouse_common_io)

dbms/programs/performance-test/CMakeLists.txt

@@ -1,8 +1,8 @@
 add_library (clickhouse-performance-test-lib ${LINK_MODE} PerformanceTest.cpp)
-target_link_libraries (clickhouse-performance-test-lib clickhouse_common_io dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-performance-test-lib PRIVATE dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
 target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-performance-test clickhouse-performance-test.cpp)
-    target_link_libraries (clickhouse-performance-test clickhouse-performance-test-lib dbms)
+    target_link_libraries (clickhouse-performance-test PRIVATE clickhouse-performance-test-lib)
 endif ()

dbms/programs/performance-test/PerformanceTest.cpp

@@ -49,10 +49,10 @@ namespace DB
 {
 namespace ErrorCodes
 {
-    extern const int POCO_EXCEPTION;
-    extern const int STD_EXCEPTION;
-    extern const int UNKNOWN_EXCEPTION;
     extern const int NOT_IMPLEMENTED;
+    extern const int LOGICAL_ERROR;
+    extern const int BAD_ARGUMENTS;
+    extern const int FILE_DOESNT_EXIST;
 }
 
 static String pad(size_t padding)
@@ -156,7 +156,7 @@ struct StopConditionsSet
             else if (key == "average_speed_not_changing_for_ms")
                 average_speed_not_changing_for_ms.value = stop_conditions_view->getUInt64(key);
             else
-                throw DB::Exception("Met unkown stop condition: " + key);
+                throw DB::Exception("Met unkown stop condition: " + key, DB::ErrorCodes::LOGICAL_ERROR);
 
             ++initialized_count;
         }
@@ -521,7 +521,7 @@ public:
     {
         if (input_files.size() < 1)
         {
-            throw DB::Exception("No tests were specified", 0);
+            throw DB::Exception("No tests were specified", DB::ErrorCodes::BAD_ARGUMENTS);
         }
 
         std::string name;
@@ -694,7 +694,7 @@ private:
                 size_t ram_size_needed = config->getUInt64("preconditions.ram_size");
                 size_t actual_ram = getMemoryAmount();
                 if (!actual_ram)
-                    throw DB::Exception("ram_size precondition not available on this platform", ErrorCodes::NOT_IMPLEMENTED);
+                    throw DB::Exception("ram_size precondition not available on this platform", DB::ErrorCodes::NOT_IMPLEMENTED);
 
                 if (ram_size_needed > actual_ram)
                 {
@@ -868,12 +868,12 @@ private:
 
         if (!test_config->has("query") && !test_config->has("query_file"))
         {
-            throw DB::Exception("Missing query fields in test's config: " + test_name);
+            throw DB::Exception("Missing query fields in test's config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
         }
 
         if (test_config->has("query") && test_config->has("query_file"))
         {
-            throw DB::Exception("Found both query and query_file fields. Choose only one");
+            throw DB::Exception("Found both query and query_file fields. Choose only one", DB::ErrorCodes::BAD_ARGUMENTS);
         }
 
         if (test_config->has("query"))
@@ -885,7 +885,7 @@ private:
         {
             const String filename = test_config->getString("query_file");
             if (filename.empty())
-                throw DB::Exception("Empty file name");
+                throw DB::Exception("Empty file name", DB::ErrorCodes::BAD_ARGUMENTS);
 
             bool tsv = fs::path(filename).extension().string() == ".tsv";
 
@@ -909,7 +909,7 @@ private:
 
         if (queries.empty())
         {
-            throw DB::Exception("Did not find any query to execute: " + test_name);
+            throw DB::Exception("Did not find any query to execute: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
         }
 
         if (test_config->has("substitutions"))
@@ -929,7 +929,7 @@ private:
 
         if (!test_config->has("type"))
         {
-            throw DB::Exception("Missing type property in config: " + test_name);
+            throw DB::Exception("Missing type property in config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
         }
 
         String config_exec_type = test_config->getString("type");
@@ -938,7 +938,7 @@ private:
         else if (config_exec_type == "once")
             exec_type = ExecutionType::Once;
         else
-            throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name);
+            throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
 
         times_to_run = test_config->getUInt("times_to_run", 1);
 
@@ -951,7 +951,7 @@ private:
         }
 
         if (stop_conditions_template.empty())
-            throw DB::Exception("No termination conditions were found in config");
+            throw DB::Exception("No termination conditions were found in config", DB::ErrorCodes::BAD_ARGUMENTS);
 
         for (size_t i = 0; i < times_to_run * queries.size(); ++i)
             stop_conditions_by_run.push_back(stop_conditions_template);
@@ -978,7 +978,7 @@ private:
         else
         {
             if (lite_output)
-                throw DB::Exception("Specify main_metric for lite output");
+                throw DB::Exception("Specify main_metric for lite output", DB::ErrorCodes::BAD_ARGUMENTS);
         }
 
         if (metrics.size() > 0)
@@ -1023,22 +1023,14 @@ private:
         if (exec_type == ExecutionType::Loop)
         {
             for (const String & metric : metrics)
-            {
                 if (std::find(non_loop_metrics.begin(), non_loop_metrics.end(), metric) != non_loop_metrics.end())
-                {
+                   throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS);
-                    throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")");
-                }
-            }
         }
         else
         {
             for (const String & metric : metrics)
-            {
                 if (std::find(loop_metrics.begin(), loop_metrics.end(), metric) != loop_metrics.end())
-                {
+                    throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS);
-                    throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")");
-                }
-            }
         }
     }
 
@@ -1439,7 +1431,7 @@ try
         if (input_files.empty())
         {
             std::cerr << std::endl;
-            throw DB::Exception("Did not find any xml files");
+            throw DB::Exception("Did not find any xml files", DB::ErrorCodes::BAD_ARGUMENTS);
         }
         else
             std::cerr << " found " << input_files.size() << " files." << std::endl;
@@ -1454,7 +1446,7 @@ try
             fs::path file(filename);
 
             if (!fs::exists(file))
-                throw DB::Exception("File '" + filename + "' does not exist");
+                throw DB::Exception("File '" + filename + "' does not exist", DB::ErrorCodes::FILE_DOESNT_EXIST);
 
             if (fs::is_directory(file))
             {
@@ -1463,7 +1455,7 @@ try
             else
             {
                 if (file.extension().string() != ".xml")
-                    throw DB::Exception("File '" + filename + "' does not have .xml extension");
+                    throw DB::Exception("File '" + filename + "' does not have .xml extension", DB::ErrorCodes::BAD_ARGUMENTS);
                 collected_files.push_back(filename);
             }
         }

dbms/programs/server/CMakeLists.txt

@@ -10,12 +10,16 @@ add_library (clickhouse-server-lib ${LINK_MODE}
     TCPHandler.cpp
     )
 
-target_link_libraries (clickhouse-server-lib clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions)
+target_link_libraries (clickhouse-server-lib PRIVATE clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY})
+if (USE_POCO_NETSSL)
+    target_link_libraries (clickhouse-server-lib PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
+endif ()
+
 target_include_directories (clickhouse-server-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
 
 if (CLICKHOUSE_SPLIT_BINARY)
     add_executable (clickhouse-server clickhouse-server.cpp)
-    target_link_libraries (clickhouse-server clickhouse-server-lib)
+    target_link_libraries (clickhouse-server PRIVATE clickhouse-server-lib)
     install (TARGETS clickhouse-server ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 endif ()
 

dbms/programs/server/HTTPHandler.cpp

@@ -270,7 +270,6 @@ void HTTPHandler::processQuery(
     std::string query_id = params.get("query_id", "");
     context.setUser(user, password, request.clientAddress(), quota_key);
     context.setCurrentQueryId(query_id);
-    CurrentThread::attachQueryContext(context);
 
     /// The user could specify session identifier and session timeout.
     /// It allows to modify settings, create temporary tables and reuse them in subsequent requests.

dbms/programs/server/InterserverIOHTTPHandler.cpp

@@ -19,9 +19,6 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int ABORTED;
-    extern const int POCO_EXCEPTION;
-    extern const int STD_EXCEPTION;
-    extern const int UNKNOWN_EXCEPTION;
     extern const int TOO_MANY_SIMULTANEOUS_QUERIES;
 }
 

dbms/programs/server/Server.cpp

@@ -55,6 +55,7 @@
 namespace CurrentMetrics
 {
     extern const Metric Revision;
+    extern const Metric VersionInteger;
 }
 
 namespace DB
@@ -66,6 +67,8 @@ namespace ErrorCodes
     extern const int SUPPORT_IS_DISABLED;
     extern const int ARGUMENT_OUT_OF_BOUND;
     extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
+    extern const int INVALID_CONFIG_PARAMETER;
+    extern const int SYSTEM_ERROR;
 }
 
 
@@ -73,7 +76,7 @@ static std::string getCanonicalPath(std::string && path)
 {
     Poco::trimInPlace(path);
     if (path.empty())
-        throw Exception("path configuration parameter is empty");
+        throw Exception("path configuration parameter is empty", ErrorCodes::INVALID_CONFIG_PARAMETER);
     if (path.back() != '/')
         path += '/';
     return std::move(path);
@@ -108,6 +111,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
     registerStorages();
 
     CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get());
+    CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
 
     /** Context contains all that query execution is dependent:
       *  settings, available functions, data types, aggregate functions, databases...
@@ -141,7 +145,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
             {
                 LOG_TRACE(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds.");
                 if (0 != mlockall(MCL_CURRENT))
-                    LOG_WARNING(log, "Failed mlockall: " + errnoToString());
+                    LOG_WARNING(log, "Failed mlockall: " + errnoToString(ErrorCodes::SYSTEM_ERROR));
                 else
                     LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed");
             }

dbms/programs/server/TCPHandler.cpp

@@ -718,7 +718,7 @@ bool TCPHandler::receiveData()
             {
                 NamesAndTypesList columns = block.getNamesAndTypesList();
                 storage = StorageMemory::create(external_table_name,
-                    ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
+                    ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, ColumnComments{}});
                 storage->startup();
                 query_context.addExternalTable(external_table_name, storage);
             }

dbms/programs/server/config.xml

@@ -164,6 +164,20 @@
                 </replica>
             </shard>
         </test_shard_localhost>
+        <test_cluster_two_shards_localhost>
+             <shard>
+                 <replica>
+                     <host>localhost</host>
+                     <port>9000</port>
+                 </replica>
+             </shard>
+             <shard>
+                 <replica>
+                     <host>localhost</host>
+                     <port>9000</port>
+                 </replica>
+             </shard>
+         </test_cluster_two_shards_localhost>
         <test_shard_localhost_secure>
             <shard>
                 <replica>

dbms/scripts/gen-bias-data.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3.4
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 
 import sys

dbms/src/AggregateFunctions/AggregateFunctionAvg.cpp

@@ -12,7 +12,7 @@ namespace
 template <typename T>
 struct Avg
 {
-    using FieldType = std::conditional_t<IsDecimalNumber<T>, Decimal128, typename NearestFieldType<T>::Type>;
+    using FieldType = std::conditional_t<IsDecimalNumber<T>, Decimal128, NearestFieldType<T>>;
     using Function = AggregateFunctionAvg<T, AggregateFunctionAvgData<FieldType>>;
 };
 

dbms/src/AggregateFunctions/AggregateFunctionSum.cpp

@@ -14,7 +14,7 @@ template <typename T>
 struct SumSimple
 {
     /// @note It uses slow Decimal128 (cause we need such a variant). sumWithOverflow is faster for Decimal32/64
-    using ResultType = std::conditional_t<IsDecimalNumber<T>, Decimal128, typename NearestFieldType<T>::Type>;
+    using ResultType = std::conditional_t<IsDecimalNumber<T>, Decimal128, NearestFieldType<T>>;
     using AggregateDataType = AggregateFunctionSumData<ResultType>;
     using Function = AggregateFunctionSum<T, ResultType, AggregateDataType>;
 };

dbms/src/AggregateFunctions/AggregateFunctionSumMap.h

@@ -52,7 +52,7 @@ struct AggregateFunctionSumMapData
 
 template <typename T>
 class AggregateFunctionSumMap final : public IAggregateFunctionDataHelper<
-    AggregateFunctionSumMapData<typename NearestFieldType<T>::Type>, AggregateFunctionSumMap<T>>
+    AggregateFunctionSumMapData<NearestFieldType<T>>, AggregateFunctionSumMap<T>>
 {
 private:
     using ColVecType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;

dbms/src/AggregateFunctions/AggregateFunctionUniq.cpp

@@ -130,9 +130,6 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory & factory)
 
     factory.registerFunction("uniqExact",
         createAggregateFunctionUniq<true, AggregateFunctionUniqExactData, AggregateFunctionUniqExactData<String>>);
-
-    factory.registerFunction("uniqCombined",
-        createAggregateFunctionUniq<false, AggregateFunctionUniqCombinedData, AggregateFunctionUniqCombinedData<UInt64>>);
 }
 
 }

dbms/src/AggregateFunctions/AggregateFunctionUniq.h

@@ -22,7 +22,6 @@
 #include <Common/typeid_cast.h>
 
 #include <AggregateFunctions/IAggregateFunction.h>
-#include <AggregateFunctions/UniqCombinedBiasData.h>
 #include <AggregateFunctions/UniqVariadicHash.h>
 
 
@@ -124,46 +123,6 @@ struct AggregateFunctionUniqExactData<String>
     static String getName() { return "uniqExact"; }
 };
 
-template <typename T>
-struct AggregateFunctionUniqCombinedData
-{
-    using Key = UInt32;
-    using Set = CombinedCardinalityEstimator<
-        Key,
-        HashSet<Key, TrivialHash, HashTableGrower<>>,
-        16,
-        14,
-        17,
-        TrivialHash,
-        UInt32,
-        HyperLogLogBiasEstimator<UniqCombinedBiasData>,
-        HyperLogLogMode::FullFeatured>;
-
-    Set set;
-
-    static String getName() { return "uniqCombined"; }
-};
-
-template <>
-struct AggregateFunctionUniqCombinedData<String>
-{
-    using Key = UInt64;
-    using Set = CombinedCardinalityEstimator<
-        Key,
-        HashSet<Key, TrivialHash, HashTableGrower<>>,
-        16,
-        14,
-        17,
-        TrivialHash,
-        UInt64,
-        HyperLogLogBiasEstimator<UniqCombinedBiasData>,
-        HyperLogLogMode::FullFeatured>;
-
-    Set set;
-
-    static String getName() { return "uniqCombined"; }
-};
-
 
 namespace detail
 {
@@ -199,39 +158,6 @@ template <> struct AggregateFunctionUniqTraits<Float64>
     }
 };
 
-/** Hash function for uniqCombined.
-  */
-template <typename T> struct AggregateFunctionUniqCombinedTraits
-{
-    static UInt32 hash(T x) { return static_cast<UInt32>(intHash64(x)); }
-};
-
-template <> struct AggregateFunctionUniqCombinedTraits<UInt128>
-{
-    static UInt32 hash(UInt128 x)
-    {
-        return sipHash64(x);
-    }
-};
-
-template <> struct AggregateFunctionUniqCombinedTraits<Float32>
-{
-    static UInt32 hash(Float32 x)
-    {
-        UInt64 res = ext::bit_cast<UInt64>(x);
-        return static_cast<UInt32>(intHash64(res));
-    }
-};
-
-template <> struct AggregateFunctionUniqCombinedTraits<Float64>
-{
-    static UInt32 hash(Float64 x)
-    {
-        UInt64 res = ext::bit_cast<UInt64>(x);
-        return static_cast<UInt32>(intHash64(res));
-    }
-};
-
 
 /** The structure for the delegation work to add one element to the `uniq` aggregate functions.
   * Used for partial specialization to add strings.
@@ -255,19 +181,6 @@ struct OneAdder
                 data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
             }
         }
-        else if constexpr (std::is_same_v<Data, AggregateFunctionUniqCombinedData<T>>)
-        {
-            if constexpr (!std::is_same_v<T, String>)
-            {
-                const auto & value = static_cast<const ColumnVector<T> &>(column).getData()[row_num];
-                data.set.insert(AggregateFunctionUniqCombinedTraits<T>::hash(value));
-            }
-            else
-            {
-                StringRef value = column.getDataAt(row_num);
-                data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
-            }
-        }
         else if constexpr (std::is_same_v<Data, AggregateFunctionUniqExactData<T>>)
         {
             if constexpr (!std::is_same_v<T, String>)
@@ -387,5 +300,4 @@ public:
     const char * getHeaderFilePath() const override { return __FILE__; }
 };
 
-
 }

dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp

@@ -0,0 +1,127 @@
+#include <AggregateFunctions/AggregateFunctionUniqCombined.h>
+
+#include <AggregateFunctions/AggregateFunctionFactory.h>
+#include <AggregateFunctions/Helpers.h>
+
+#include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDateTime.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+    extern const int ARGUMENT_OUT_OF_BOUND;
+}
+
+namespace
+{
+    template <UInt8 K>
+    struct WithK
+    {
+        template <typename T>
+        using AggregateFunction = AggregateFunctionUniqCombined<T, K>;
+
+        template <bool is_exact, bool argument_is_tuple>
+        using AggregateFunctionVariadic = AggregateFunctionUniqCombinedVariadic<is_exact, argument_is_tuple, K>;
+    };
+
+    template <UInt8 K>
+    AggregateFunctionPtr createAggregateFunctionWithK(const DataTypes & argument_types)
+    {
+        /// We use exact hash function if the arguments are not contiguous in memory, because only exact hash function has support for this case.
+        bool use_exact_hash_function = !isAllArgumentsContiguousInMemory(argument_types);
+
+        if (argument_types.size() == 1)
+        {
+            const IDataType & argument_type = *argument_types[0];
+
+            AggregateFunctionPtr res(createWithNumericType<WithK<K>::template AggregateFunction>(*argument_types[0]));
+
+            WhichDataType which(argument_type);
+            if (res)
+                return res;
+            else if (which.isDate())
+                return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeDate::FieldType>>();
+            else if (which.isDateTime())
+                return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeDateTime::FieldType>>();
+            else if (which.isStringOrFixedString())
+                return std::make_shared<typename WithK<K>::template AggregateFunction<String>>();
+            else if (which.isUUID())
+                return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeUUID::FieldType>>();
+            else if (which.isTuple())
+            {
+                if (use_exact_hash_function)
+                    return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<true, true>>(argument_types);
+                else
+                    return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<false, true>>(argument_types);
+            }
+        }
+
+        /// "Variadic" method also works as a fallback generic case for a single argument.
+        if (use_exact_hash_function)
+            return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<true, false>>(argument_types);
+        else
+            return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<false, false>>(argument_types);
+    }
+
+    AggregateFunctionPtr createAggregateFunctionUniqCombined(
+        const std::string & name, const DataTypes & argument_types, const Array & params)
+    {
+        /// log2 of the number of cells in HyperLogLog.
+        /// Reasonable default value, selected to be comparable in quality with "uniq" aggregate function.
+        UInt8 precision = 17;
+
+        if (!params.empty())
+        {
+            if (params.size() != 1)
+                throw Exception(
+                    "Aggregate function " + name + " requires one parameter or less.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+            UInt64 precision_param = applyVisitor(FieldVisitorConvertToNumber<UInt64>(), params[0]);
+
+            // This range is hardcoded below
+            if (precision_param > 20 || precision_param < 12)
+                throw Exception(
+                    "Parameter for aggregate function " + name + "is out or range: [12, 20].", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
+
+            precision = precision_param;
+        }
+
+        if (argument_types.empty())
+            throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+        switch (precision)
+        {
+            case 12:
+                return createAggregateFunctionWithK<12>(argument_types);
+            case 13:
+                return createAggregateFunctionWithK<13>(argument_types);
+            case 14:
+                return createAggregateFunctionWithK<14>(argument_types);
+            case 15:
+                return createAggregateFunctionWithK<15>(argument_types);
+            case 16:
+                return createAggregateFunctionWithK<16>(argument_types);
+            case 17:
+                return createAggregateFunctionWithK<17>(argument_types);
+            case 18:
+                return createAggregateFunctionWithK<18>(argument_types);
+            case 19:
+                return createAggregateFunctionWithK<19>(argument_types);
+            case 20:
+                return createAggregateFunctionWithK<20>(argument_types);
+        }
+
+        __builtin_unreachable();
+    }
+
+} // namespace
+
+void registerAggregateFunctionUniqCombined(AggregateFunctionFactory & factory)
+{
+    factory.registerFunction("uniqCombined", createAggregateFunctionUniqCombined);
+}
+
+} // namespace DB

dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.h

@@ -0,0 +1,229 @@
+#pragma once
+
+#include <Common/CombinedCardinalityEstimator.h>
+
+#include <DataTypes/DataTypeTuple.h>
+#include <DataTypes/DataTypeUUID.h>
+#include <DataTypes/DataTypesNumber.h>
+
+#include <AggregateFunctions/IAggregateFunction.h>
+#include <AggregateFunctions/UniqCombinedBiasData.h>
+#include <AggregateFunctions/UniqVariadicHash.h>
+
+#include <ext/bit_cast.h>
+
+#include <Columns/ColumnVector.h>
+#include <Columns/ColumnsNumber.h>
+#include <Common/FieldVisitors.h>
+#include <Common/SipHash.h>
+
+#include <Common/typeid_cast.h>
+
+namespace DB
+{
+namespace detail
+{
+    /** Hash function for uniqCombined.
+     */
+    template <typename T>
+    struct AggregateFunctionUniqCombinedTraits
+    {
+        static UInt32 hash(T x)
+        {
+            return static_cast<UInt32>(intHash64(x));
+        }
+    };
+
+    template <>
+    struct AggregateFunctionUniqCombinedTraits<UInt128>
+    {
+        static UInt32 hash(UInt128 x)
+        {
+            return sipHash64(x);
+        }
+    };
+
+    template <>
+    struct AggregateFunctionUniqCombinedTraits<Float32>
+    {
+        static UInt32 hash(Float32 x)
+        {
+            UInt64 res = ext::bit_cast<UInt64>(x);
+            return static_cast<UInt32>(intHash64(res));
+        }
+    };
+
+    template <>
+    struct AggregateFunctionUniqCombinedTraits<Float64>
+    {
+        static UInt32 hash(Float64 x)
+        {
+            UInt64 res = ext::bit_cast<UInt64>(x);
+            return static_cast<UInt32>(intHash64(res));
+        }
+    };
+
+} // namespace detail
+
+
+template <typename Key, UInt8 K>
+struct AggregateFunctionUniqCombinedDataWithKey
+{
+    // TODO(ilezhankin): pre-generate values for |UniqCombinedBiasData|,
+    //                   at the moment gen-bias-data.py script doesn't work.
+
+    // We want to migrate from |HashSet| to |HyperLogLogCounter| when the sizes in memory become almost equal.
+    // The size per element in |HashSet| is sizeof(Key)*2 bytes, and the overall size of |HyperLogLogCounter| is 2^K * 6 bits.
+    // For Key=UInt32 we can calculate: 2^X * 4 * 2 ≤ 2^(K-3) * 6 ⇒ X ≤ K-4.
+    using Set = CombinedCardinalityEstimator<Key, HashSet<Key, TrivialHash, HashTableGrower<>>, 16, K - 4, K, TrivialHash, Key>;
+
+    Set set;
+};
+
+template <typename Key>
+struct AggregateFunctionUniqCombinedDataWithKey<Key, 17>
+{
+    using Set = CombinedCardinalityEstimator<Key,
+        HashSet<Key, TrivialHash, HashTableGrower<>>,
+        16,
+        13,
+        17,
+        TrivialHash,
+        Key,
+        HyperLogLogBiasEstimator<UniqCombinedBiasData>,
+        HyperLogLogMode::FullFeatured>;
+
+    Set set;
+};
+
+
+template <typename T, UInt8 K>
+struct AggregateFunctionUniqCombinedData : public AggregateFunctionUniqCombinedDataWithKey<UInt32, K>
+{
+};
+
+
+template <UInt8 K>
+struct AggregateFunctionUniqCombinedData<String, K> : public AggregateFunctionUniqCombinedDataWithKey<UInt64, K>
+{
+};
+
+
+template <typename T, UInt8 K>
+class AggregateFunctionUniqCombined final
+    : public IAggregateFunctionDataHelper<AggregateFunctionUniqCombinedData<T, K>, AggregateFunctionUniqCombined<T, K>>
+{
+public:
+    String getName() const override
+    {
+        return "uniqCombined";
+    }
+
+    DataTypePtr getReturnType() const override
+    {
+        return std::make_shared<DataTypeUInt64>();
+    }
+
+    void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
+    {
+        if constexpr (!std::is_same_v<T, String>)
+        {
+            const auto & value = static_cast<const ColumnVector<T> &>(*columns[0]).getData()[row_num];
+            this->data(place).set.insert(detail::AggregateFunctionUniqCombinedTraits<T>::hash(value));
+        }
+        else
+        {
+            StringRef value = columns[0]->getDataAt(row_num);
+            this->data(place).set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
+        }
+    }
+
+    void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
+    {
+        this->data(place).set.merge(this->data(rhs).set);
+    }
+
+    void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
+    {
+        this->data(place).set.write(buf);
+    }
+
+    void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
+    {
+        this->data(place).set.read(buf);
+    }
+
+    void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
+    {
+        static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
+    }
+
+    const char * getHeaderFilePath() const override
+    {
+        return __FILE__;
+    }
+};
+
+/** For multiple arguments. To compute, hashes them.
+  * You can pass multiple arguments as is; You can also pass one argument - a tuple.
+  * But (for the possibility of efficient implementation), you can not pass several arguments, among which there are tuples.
+  */
+template <bool is_exact, bool argument_is_tuple, UInt8 K>
+class AggregateFunctionUniqCombinedVariadic final : public IAggregateFunctionDataHelper<AggregateFunctionUniqCombinedData<UInt64, K>,
+                                                        AggregateFunctionUniqCombinedVariadic<is_exact, argument_is_tuple, K>>
+{
+private:
+    size_t num_args = 0;
+
+public:
+    explicit AggregateFunctionUniqCombinedVariadic(const DataTypes & arguments)
+    {
+        if (argument_is_tuple)
+            num_args = typeid_cast<const DataTypeTuple &>(*arguments[0]).getElements().size();
+        else
+            num_args = arguments.size();
+    }
+
+    String getName() const override
+    {
+        return "uniqCombined";
+    }
+
+    DataTypePtr getReturnType() const override
+    {
+        return std::make_shared<DataTypeUInt64>();
+    }
+
+    void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
+    {
+        this->data(place).set.insert(typename AggregateFunctionUniqCombinedData<UInt64, K>::Set::value_type(
+            UniqVariadicHash<is_exact, argument_is_tuple>::apply(num_args, columns, row_num)));
+    }
+
+    void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
+    {
+        this->data(place).set.merge(this->data(rhs).set);
+    }
+
+    void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
+    {
+        this->data(place).set.write(buf);
+    }
+
+    void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
+    {
+        this->data(place).set.read(buf);
+    }
+
+    void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
+    {
+        static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
+    }
+
+    const char * getHeaderFilePath() const override
+    {
+        return __FILE__;
+    }
+};
+
+} // namespace DB

dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h

@@ -16,10 +16,12 @@
 
 namespace DB
 {
+
 namespace ErrorCodes
 {
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
     extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
 }
 
 struct ComparePairFirst final
@@ -191,7 +193,7 @@ public:
         const auto time_arg = arguments.front().get();
         if (!WhichDataType(time_arg).isDateTime() && !WhichDataType(time_arg).isUInt32())
             throw Exception{"Illegal type " + time_arg->getName() + " of first argument of aggregate function " + getName()
-                + ", must be DateTime or UInt32"};
+                + ", must be DateTime or UInt32", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         for (const auto i : ext::range(1, arguments.size()))
         {

dbms/src/AggregateFunctions/CMakeLists.txt

@@ -20,5 +20,5 @@ list(REMOVE_ITEM clickhouse_aggregate_functions_headers
 )
 
 add_library(clickhouse_aggregate_functions ${LINK_MODE} ${clickhouse_aggregate_functions_sources})
-target_link_libraries(clickhouse_aggregate_functions dbms)
+target_link_libraries(clickhouse_aggregate_functions PRIVATE dbms)
 target_include_directories (clickhouse_aggregate_functions BEFORE PRIVATE ${COMMON_INCLUDE_DIR})

dbms/src/AggregateFunctions/registerAggregateFunctions.cpp

@@ -21,6 +21,7 @@ void registerAggregateFunctionsStatisticsSimple(AggregateFunctionFactory &);
 void registerAggregateFunctionSum(AggregateFunctionFactory &);
 void registerAggregateFunctionSumMap(AggregateFunctionFactory &);
 void registerAggregateFunctionsUniq(AggregateFunctionFactory &);
+void registerAggregateFunctionUniqCombined(AggregateFunctionFactory &);
 void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &);
 void registerAggregateFunctionTopK(AggregateFunctionFactory &);
 void registerAggregateFunctionsBitwise(AggregateFunctionFactory &);
@@ -55,6 +56,7 @@ void registerAggregateFunctions()
         registerAggregateFunctionSum(factory);
         registerAggregateFunctionSumMap(factory);
         registerAggregateFunctionsUniq(factory);
+        registerAggregateFunctionUniqCombined(factory);
         registerAggregateFunctionUniqUpTo(factory);
         registerAggregateFunctionTopK(factory);
         registerAggregateFunctionsBitwise(factory);

dbms/src/Client/Connection.cpp

@@ -108,14 +108,14 @@ void Connection::connect()
         disconnect();
 
         /// Add server address to exception. Also Exception will remember stack trace. It's a pity that more precise exception type is lost.
-        throw NetException(e.displayText(), "(" + getDescription() + ")", ErrorCodes::NETWORK_ERROR);
+        throw NetException(e.displayText() + " (" + getDescription() + ")", ErrorCodes::NETWORK_ERROR);
     }
     catch (Poco::TimeoutException & e)
     {
         disconnect();
 
         /// Add server address to exception. Also Exception will remember stack trace. It's a pity that more precise exception type is lost.
-        throw NetException(e.displayText(), "(" + getDescription() + ")", ErrorCodes::SOCKET_TIMEOUT);
+        throw NetException(e.displayText() + " (" + getDescription() + ")", ErrorCodes::SOCKET_TIMEOUT);
     }
 }
 

dbms/src/Columns/ColumnConst.h

@@ -209,7 +209,7 @@ public:
     Field getField() const { return getDataColumn()[0]; }
 
     template <typename T>
-    T getValue() const { return getField().safeGet<typename NearestFieldType<T>::Type>(); }
+    T getValue() const { return getField().safeGet<NearestFieldType<T>>(); }
 };
 
 }

dbms/src/Columns/ColumnDecimal.cpp

@@ -213,8 +213,8 @@ void ColumnDecimal<T>::getExtremes(Field & min, Field & max) const
 {
     if (data.size() == 0)
     {
-        min = typename NearestFieldType<T>::Type(0, scale);
+        min = NearestFieldType<T>(0, scale);
-        max = typename NearestFieldType<T>::Type(0, scale);
+        max = NearestFieldType<T>(0, scale);
         return;
     }
 
@@ -229,8 +229,8 @@ void ColumnDecimal<T>::getExtremes(Field & min, Field & max) const
             cur_max = x;
     }
 
-    min = typename NearestFieldType<T>::Type(cur_min, scale);
+    min = NearestFieldType<T>(cur_min, scale);
-    max = typename NearestFieldType<T>::Type(cur_max, scale);
+    max = NearestFieldType<T>(cur_max, scale);
 }
 
 template class ColumnDecimal<Decimal32>;

dbms/src/Columns/ColumnDecimal.h

@@ -91,7 +91,7 @@ public:
     void insertFrom(const IColumn & src, size_t n) override { data.push_back(static_cast<const Self &>(src).getData()[n]); }
     void insertData(const char * pos, size_t /*length*/) override;
     void insertDefault() override { data.push_back(T()); }
-    void insert(const Field & x) override { data.push_back(DB::get<typename NearestFieldType<T>::Type>(x)); }
+    void insert(const Field & x) override { data.push_back(DB::get<NearestFieldType<T>>(x)); }
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 
     void popBack(size_t n) override { data.resize_assume_reserved(data.size() - n); }

dbms/src/Columns/ColumnUnique.h

@@ -81,12 +81,14 @@ public:
     {
         return column_holder->allocatedBytes()
                + index.allocatedBytes()
-               + (cached_null_mask ? cached_null_mask->allocatedBytes() : 0);
+               + (nested_null_mask ? nested_null_mask->allocatedBytes() : 0);
     }
     void forEachSubcolumn(IColumn::ColumnCallback callback) override
     {
         callback(column_holder);
         index.setColumn(getRawColumnPtr());
+        if (is_nullable)
+            nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask);
     }
 
     const UInt64 * tryGetSavedHash() const override { return index.tryGetSavedHash(); }
@@ -100,8 +102,8 @@ private:
     ReverseIndex<UInt64, ColumnType> index;
 
     /// For DataTypeNullable, stores null map.
-    mutable ColumnPtr cached_null_mask;
+    ColumnPtr nested_null_mask;
-    mutable ColumnPtr cached_column_nullable;
+    ColumnPtr nested_column_nullable;
 
     class IncrementalHash
     {
@@ -118,6 +120,9 @@ private:
 
     mutable IncrementalHash hash;
 
+    void createNullMask();
+    void updateNullMask();
+
     static size_t numSpecialValues(bool is_nullable) { return is_nullable ? 2 : 1; }
     size_t numSpecialValues() const { return numSpecialValues(is_nullable); }
 
@@ -148,6 +153,7 @@ ColumnUnique<ColumnType>::ColumnUnique(const ColumnUnique & other)
     , index(numSpecialValues(is_nullable), 0)
 {
     index.setColumn(getRawColumnPtr());
+    createNullMask();
 }
 
 template <typename ColumnType>
@@ -158,6 +164,7 @@ ColumnUnique<ColumnType>::ColumnUnique(const IDataType & type)
     const auto & holder_type = is_nullable ? *static_cast<const DataTypeNullable &>(type).getNestedType() : type;
     column_holder = holder_type.createColumn()->cloneResized(numSpecialValues());
     index.setColumn(getRawColumnPtr());
+    createNullMask();
 }
 
 template <typename ColumnType>
@@ -172,32 +179,51 @@ ColumnUnique<ColumnType>::ColumnUnique(MutableColumnPtr && holder, bool is_nulla
         throw Exception("Holder column for ColumnUnique can't be nullable.", ErrorCodes::ILLEGAL_COLUMN);
 
     index.setColumn(getRawColumnPtr());
+    createNullMask();
+}
+
+template <typename ColumnType>
+void ColumnUnique<ColumnType>::createNullMask()
+{
+    if (is_nullable)
+    {
+        size_t size = getRawColumnPtr()->size();
+        if (!nested_null_mask)
+        {
+            ColumnUInt8::MutablePtr null_mask = ColumnUInt8::create(size, UInt8(0));
+            null_mask->getData()[getNullValueIndex()] = 1;
+            nested_null_mask = std::move(null_mask);
+            nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask);
+        }
+        else
+            throw Exception("Null mask for ColumnUnique is already created.", ErrorCodes::LOGICAL_ERROR);
+    }
+}
+
+template <typename ColumnType>
+void ColumnUnique<ColumnType>::updateNullMask()
+{
+    if (is_nullable)
+    {
+        if (!nested_null_mask)
+            throw Exception("Null mask for ColumnUnique is was not created.", ErrorCodes::LOGICAL_ERROR);
+
+        size_t size = getRawColumnPtr()->size();
+
+        if (nested_null_mask->size() != size)
+        {
+            IColumn & null_mask = nested_null_mask->assumeMutableRef();
+            static_cast<ColumnUInt8 &>(null_mask).getData().resize_fill(size);
+        }
+    }
 }
 
 template <typename ColumnType>
 const ColumnPtr & ColumnUnique<ColumnType>::getNestedColumn() const
 {
     if (is_nullable)
-    {
+        return nested_column_nullable;
-        size_t size = getRawColumnPtr()->size();
-        if (!cached_null_mask)
-        {
-            ColumnUInt8::MutablePtr null_mask = ColumnUInt8::create(size, UInt8(0));
-            null_mask->getData()[getNullValueIndex()] = 1;
-            cached_null_mask = std::move(null_mask);
-            cached_column_nullable = ColumnNullable::create(column_holder, cached_null_mask);
-        }
 
-        if (cached_null_mask->size() != size)
-        {
-            MutableColumnPtr null_mask = (*std::move(cached_null_mask)).mutate();
-            static_cast<ColumnUInt8 &>(*null_mask).getData().resize_fill(size);
-            cached_null_mask = std::move(null_mask);
-            cached_column_nullable = ColumnNullable::create(column_holder, cached_null_mask);
-        }
-
-        return cached_column_nullable;
-    }
     return column_holder;
 }
 
@@ -227,6 +253,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsert(const Field & x)
     if (pos != prev_size)
         column->popBack(1);
 
+    updateNullMask();
+
     return pos;
 }
 
@@ -260,6 +288,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsertData(const char * pos, size_t lengt
         index.insertFromLastRow();
     }
 
+    updateNullMask();
+
     return insertion_point;
 }
 
@@ -288,6 +318,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsertDataWithTerminatingZero(const char
     if (position != prev_size)
         column->popBack(1);
 
+    updateNullMask();
+
     return static_cast<size_t>(position);
 }
 
@@ -305,7 +337,7 @@ StringRef ColumnUnique<ColumnType>::serializeValueIntoArena(size_t n, Arena & ar
 
         size_t nested_size = 0;
 
-        if (n == getNullValueIndex())
+        if (n != getNullValueIndex())
             nested_size = column_holder->serializeValueIntoArena(n, arena, begin).size;
 
         return StringRef(pos, sizeof(null_flag) + nested_size);
@@ -343,6 +375,8 @@ size_t ColumnUnique<ColumnType>::uniqueDeserializeAndInsertFromArena(const char
     if (index_pos != prev_size)
         column->popBack(1);
 
+    updateNullMask();
+
     return static_cast<size_t>(index_pos);
 }
 
@@ -533,6 +567,8 @@ MutableColumnPtr ColumnUnique<ColumnType>::uniqueInsertRangeFrom(const IColumn &
     if (!positions_column)
         throw Exception("Can't find index type for ColumnUnique", ErrorCodes::LOGICAL_ERROR);
 
+    updateNullMask();
+
     return positions_column;
 }
 
@@ -577,6 +613,8 @@ IColumnUnique::IndexesWithOverflow ColumnUnique<ColumnType>::uniqueInsertRangeWi
     if (!positions_column)
         throw Exception("Can't find index type for ColumnUnique", ErrorCodes::LOGICAL_ERROR);
 
+    updateNullMask();
+
     IColumnUnique::IndexesWithOverflow indexes_with_overflow;
     indexes_with_overflow.indexes = std::move(positions_column);
     indexes_with_overflow.overflowed_keys = std::move(overflowed_keys);

dbms/src/Columns/ColumnVector.cpp

@@ -311,8 +311,8 @@ void ColumnVector<T>::getExtremes(Field & min, Field & max) const
             cur_max = x;
     }
 
-    min = typename NearestFieldType<T>::Type(cur_min);
+    min = NearestFieldType<T>(cur_min);
-    max = typename NearestFieldType<T>::Type(cur_max);
+    max = NearestFieldType<T>(cur_max);
 }
 
 /// Explicit template instantiations - to avoid code bloat in headers.

dbms/src/Columns/ColumnVector.h

@@ -244,7 +244,7 @@ public:
 
     void insert(const Field & x) override
     {
-        data.push_back(DB::get<typename NearestFieldType<T>::Type>(x));
+        data.push_back(DB::get<NearestFieldType<T>>(x));
     }
 
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;

dbms/src/Common/ClickHouseRevision.cpp

@@ -4,4 +4,5 @@
 namespace ClickHouseRevision
 {
     unsigned get() { return VERSION_REVISION; }
+    unsigned getVersionInteger() { return VERSION_INTEGER; }
 }

dbms/src/Common/ClickHouseRevision.h

@@ -3,4 +3,5 @@
 namespace ClickHouseRevision
 {
     unsigned get();
+    unsigned getVersionInteger();
 }

dbms/src/Common/Config/CMakeLists.txt

@@ -1,9 +1,8 @@
-
 include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 
 add_headers_and_sources(clickhouse_common_config .)
 
 add_library(clickhouse_common_config ${LINK_MODE} ${clickhouse_common_config_headers} ${clickhouse_common_config_sources})
 
-target_link_libraries (clickhouse_common_config clickhouse_common_zookeeper string_utils ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY})
+target_link_libraries(clickhouse_common_config PUBLIC common PRIVATE clickhouse_common_zookeeper string_utils PUBLIC ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY})
-target_include_directories (clickhouse_common_config PRIVATE ${DBMS_INCLUDE_DIR})
+target_include_directories(clickhouse_common_config PUBLIC ${DBMS_INCLUDE_DIR})

dbms/src/Common/CounterInFile.h

@@ -18,7 +18,15 @@
 
 #include <common/Types.h>
 
-#define SMALL_READ_WRITE_BUFFER_SIZE 16
+namespace DB
+{
+    namespace ErrorCodes
+    {
+        extern const int CANNOT_OPEN_FILE;
+        extern const int CANNOT_READ_ALL_DATA;
+        extern const int ATTEMPT_TO_READ_AFTER_EOF;
+    }
+}
 
 
 /** Stores a number in the file.
@@ -26,6 +34,9 @@
  */
 class CounterInFile
 {
+private:
+    static inline constexpr size_t SMALL_READ_WRITE_BUFFER_SIZE = 16;
+
 public:
     /// path - the name of the file, including the path
     CounterInFile(const std::string & path_) : path(path_) {}
@@ -56,13 +67,13 @@ public:
 
         int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0666);
         if (-1 == fd)
-            DB::throwFromErrno("Cannot open file " + path);
+            DB::throwFromErrno("Cannot open file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
 
         try
         {
             int flock_ret = flock(fd, LOCK_EX);
             if (-1 == flock_ret)
-                DB::throwFromErrno("Cannot lock file " + path);
+                DB::throwFromErrno("Cannot lock file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
 
             if (!file_doesnt_exists)
             {
@@ -130,7 +141,7 @@ public:
 
         int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0666);
         if (-1 == fd)
-            DB::throwFromErrno("Cannot open file " + path);
+            DB::throwFromErrno("Cannot open file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
 
         try
         {
@@ -178,6 +189,3 @@ private:
     std::string path;
     std::mutex mutex;
 };
-
-
-#undef SMALL_READ_WRITE_BUFFER_SIZE

dbms/src/Common/CurrentMetrics.cpp

@@ -39,7 +39,8 @@
     M(StorageBufferRows, "Number of rows in buffers of Buffer tables") \
     M(StorageBufferBytes, "Number of bytes in buffers of Buffer tables") \
     M(DictCacheRequests, "Number of requests in fly to data sources of dictionaries of cache type.") \
-    M(Revision, "Revision of the server. It is a number incremented for every release or release candidate.") \
+    M(Revision, "Revision of the server. It is a number incremented for every release or release candidate except patch releases.") \
+    M(VersionInteger, "Version of the server in a single integer number in base-1000. For example, version 11.22.33 is translated to 11022033.") \
     M(RWLockWaitingReaders, "Number of threads waiting for read on a table RWLock.") \
     M(RWLockWaitingWriters, "Number of threads waiting for write on a table RWLock.") \
     M(RWLockActiveReaders, "Number of threads holding read lock in a table RWLock.") \

dbms/src/Common/ErrorCodes.cpp

@@ -396,6 +396,11 @@ namespace ErrorCodes
     extern const int MULTIPLE_ASSIGNMENTS_TO_COLUMN = 419;
     extern const int CANNOT_UPDATE_COLUMN = 420;
     extern const int CANNOT_ADD_DIFFERENT_AGGREGATE_STATES = 421;
+    extern const int UNSUPPORTED_URI_SCHEME = 422;
+    extern const int CANNOT_GETTIMEOFDAY = 423;
+    extern const int CANNOT_LINK = 424;
+    extern const int SYSTEM_ERROR = 425;
+    extern const int NULL_POINTER_DEREFERENCE = 426;
 
     extern const int KEEPER_EXCEPTION = 999;
     extern const int POCO_EXCEPTION = 1000;

dbms/src/Common/Exception.cpp

@@ -52,7 +52,7 @@ std::string errnoToString(int code, int e)
 
 void throwFromErrno(const std::string & s, int code, int e)
 {
-    throw ErrnoException(s + ", " + errnoToString(code, e));
+    throw ErrnoException(s + ", " + errnoToString(code, e), code, e);
 }
 
 void tryLogCurrentException(const char * log_name, const std::string & start_of_message)

dbms/src/Common/Exception.h

@@ -14,19 +14,25 @@ namespace Poco { class Logger; }
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int POCO_EXCEPTION;
+}
+
 class Exception : public Poco::Exception
 {
 public:
     Exception() {}  /// For deferred initialization.
-    Exception(const std::string & msg, int code = 0) : Poco::Exception(msg, code) {}
+    Exception(const std::string & msg, int code) : Poco::Exception(msg, code) {}
-    Exception(const std::string & msg, const std::string & arg, int code = 0) : Poco::Exception(msg, arg, code) {}
+    Exception(const std::string & msg, const Exception & nested_exception, int code)
-    Exception(const std::string & msg, const Exception & exc, int code = 0) : Poco::Exception(msg, exc, code), trace(exc.trace) {}
+        : Poco::Exception(msg, nested_exception, code), trace(nested_exception.trace) {}
-    explicit Exception(const Poco::Exception & exc) : Poco::Exception(exc.displayText()) {}
 
-    const char * name() const throw() override { return "DB::Exception"; }
+    enum CreateFromPocoTag { CreateFromPoco };
-    const char * className() const throw() override { return "DB::Exception"; }
+    Exception(CreateFromPocoTag, const Poco::Exception & exc) : Poco::Exception(exc.displayText(), ErrorCodes::POCO_EXCEPTION) {}
-    DB::Exception * clone() const override { return new DB::Exception(*this); }
+
+    Exception * clone() const override { return new Exception(*this); }
     void rethrow() const override { throw *this; }
+    const char * name() const throw() override { return "DB::Exception"; }
 
     /// Add something to the existing message.
     void addMessage(const std::string & arg) { extendedMessage(arg); }
@@ -35,6 +41,8 @@ public:
 
 private:
     StackTrace trace;
+
+    const char * className() const throw() override { return "DB::Exception"; }
 };
 
 
@@ -42,25 +50,27 @@ private:
 class ErrnoException : public Exception
 {
 public:
-    ErrnoException(const std::string & msg, int code = 0, int saved_errno_ = 0)
+    ErrnoException(const std::string & msg, int code, int saved_errno_)
         : Exception(msg, code), saved_errno(saved_errno_) {}
-    ErrnoException(const std::string & msg, const std::string & arg, int code = 0, int saved_errno_ = 0)
+
-        : Exception(msg, arg, code), saved_errno(saved_errno_) {}
+    ErrnoException * clone() const override { return new ErrnoException(*this); }
-    ErrnoException(const std::string & msg, const Exception & exc, int code = 0, int saved_errno_ = 0)
+    void rethrow() const override { throw *this; }
-        : Exception(msg, exc, code), saved_errno(saved_errno_) {}
 
     int getErrno() const { return saved_errno; }
 
 private:
     int saved_errno;
+
+    const char * name() const throw() override { return "DB::ErrnoException"; }
+    const char * className() const throw() override { return "DB::ErrnoException"; }
 };
 
 
 using Exceptions = std::vector<std::exception_ptr>;
 
 
-std::string errnoToString(int code = 0, int the_errno = errno);
+std::string errnoToString(int code, int the_errno = errno);
-[[noreturn]] void throwFromErrno(const std::string & s, int code = 0, int the_errno = errno);
+[[noreturn]] void throwFromErrno(const std::string & s, int code, int the_errno = errno);
 
 
 /** Try to write an exception to the log (and forget about it).

dbms/src/Common/HashTable/SmallTable.h

@@ -3,6 +3,15 @@
 #include <Common/HashTable/HashMap.h>
 
 
+namespace DB
+{
+    namespace ErrorCodes
+    {
+        extern const int INCORRECT_DATA;
+    }
+}
+
+
 /** Replacement of the hash table for a small number (<10) of keys.
   * Implemented as an array with linear search.
   * The array is located inside the object.
@@ -13,7 +22,6 @@
   *  you should check if the table is not full,
   *  and do a `fallback` in this case (for example, use a real hash table).
   */
-
 template
 <
     typename Key,
@@ -86,7 +94,7 @@ public:
                 DB::readVarUInt(size, in);
 
                 if (size > capacity)
-                    throw DB::Exception("Illegal size");
+                    throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
 
                 is_initialized = true;
             }
@@ -306,7 +314,7 @@ public:
         DB::readVarUInt(new_size, rb);
 
         if (new_size > capacity)
-            throw DB::Exception("Illegal size");
+            throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
 
         for (size_t i = 0; i < new_size; ++i)
             buf[i].read(rb);
@@ -324,7 +332,7 @@ public:
         DB::readText(new_size, rb);
 
         if (new_size > capacity)
-            throw DB::Exception("Illegal size");
+            throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
 
         for (size_t i = 0; i < new_size; ++i)
         {

dbms/src/Common/NetException.h

@@ -6,20 +6,17 @@
 namespace DB
 {
 
-class NetException : public DB::Exception
+class NetException : public Exception
 {
 public:
-    NetException(const std::string & msg, int code = 0) : DB::Exception(msg, code) {}
+    NetException(const std::string & msg, int code) : Exception(msg, code) {}
-    NetException(const std::string & msg, const std::string & arg, int code = 0) : DB::Exception(msg, arg, code) {}
-    NetException(const std::string & msg, const DB::Exception & exc, int code = 0) : DB::Exception(msg, exc, code) {}
 
-    explicit NetException(const DB::Exception & exc) : DB::Exception(exc) {}
+    NetException * clone() const override { return new NetException(*this); }
-    explicit NetException(const Poco::Exception & exc) : DB::Exception(exc.displayText()) {}
+    void rethrow() const override { throw *this; }
 
+private:
     const char * name() const throw() override { return "DB::NetException"; }
     const char * className() const throw() override { return "DB::NetException"; }
-    DB::NetException * clone() const override { return new DB::NetException(*this); }
-    void rethrow() const override { throw *this; }
 };
 
 }

dbms/src/Common/PoolWithFailoverBase.h

@@ -305,7 +305,7 @@ void PoolWithFailoverBase<TNestedPool>::reportError(const Entry & entry)
             return;
         }
     }
-    throw DB::Exception("Can't find pool to report error.");
+    throw DB::Exception("Can't find pool to report error", DB::ErrorCodes::LOGICAL_ERROR);
 }
 
 template <typename TNestedPool>

dbms/src/Common/ProfileEvents.cpp

@@ -170,6 +170,7 @@
     M(OSWriteBytes, "Number of bytes written to disks or block devices. Doesn't include bytes that are in page cache dirty pages. May not include data that was written by OS asynchronously.") \
     M(OSReadChars, "Number of bytes read from filesystem, including page cache.") \
     M(OSWriteChars, "Number of bytes written to filesystem, including page cache.") \
+    M(CreatedHTTPConnections, "Total amount of created HTTP connections (closed or opened).") \
 
 namespace ProfileEvents
 {

dbms/src/Common/ShellCommand.cpp

@@ -4,9 +4,11 @@
 #include <dlfcn.h>
 #include <Common/Exception.h>
 #include <Common/ShellCommand.h>
+#include <common/logger_useful.h>
 #include <IO/WriteBufferFromVector.h>
 #include <IO/WriteHelpers.h>
 #include <port/unistd.h>
+#include <csignal>
 
 
 namespace DB
@@ -75,13 +77,27 @@ namespace
 namespace DB
 {
 
+ShellCommand::ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd, bool terminate_in_destructor_)
+    : pid(pid)
+    , terminate_in_destructor(terminate_in_destructor_)
+    , log(&Poco::Logger::get("ShellCommand"))
+    , in(in_fd)
+    , out(out_fd)
+    , err(err_fd) {}
+
 ShellCommand::~ShellCommand()
 {
-    if (!wait_called)
+    if (terminate_in_destructor)
+    {
+        int retcode = kill(pid, SIGTERM);
+        if (retcode != 0)
+            LOG_WARNING(log, "Cannot kill pid " << pid << " errno '" << errnoToString(retcode) << "'");
+    }
+    else if (!wait_called)
         tryWait();
 }
 
-std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only)
+std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor)
 {
     /** Here it is written that with a normal call `vfork`, there is a chance of deadlock in multithreaded programs,
       *  because of the resolving of characters in the shared library
@@ -128,7 +144,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, c
         _exit(int(ReturnCodes::CANNOT_EXEC));
     }
 
-    std::unique_ptr<ShellCommand> res(new ShellCommand(pid, pipe_stdin.write_fd, pipe_stdout.read_fd, pipe_stderr.read_fd));
+    std::unique_ptr<ShellCommand> res(new ShellCommand(pid, pipe_stdin.write_fd, pipe_stdout.read_fd, pipe_stderr.read_fd, terminate_in_destructor));
 
     /// Now the ownership of the file descriptors is passed to the result.
     pipe_stdin.write_fd = -1;
@@ -139,7 +155,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, c
 }
 
 
-std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command, bool pipe_stdin_only)
+std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command, bool pipe_stdin_only, bool terminate_in_destructor)
 {
     /// Arguments in non-constant chunks of memory (as required for `execv`).
     /// Moreover, their copying must be done before calling `vfork`, so after `vfork` do a minimum of things.
@@ -149,11 +165,11 @@ std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command,
 
     char * const argv[] = { argv0.data(), argv1.data(), argv2.data(), nullptr };
 
-    return executeImpl("/bin/sh", argv, pipe_stdin_only);
+    return executeImpl("/bin/sh", argv, pipe_stdin_only, terminate_in_destructor);
 }
 
 
-std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & path, const std::vector<std::string> & arguments)
+std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & path, const std::vector<std::string> & arguments, bool terminate_in_destructor)
 {
     size_t argv_sum_size = path.size() + 1;
     for (const auto & arg : arguments)
@@ -174,7 +190,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & pa
 
     argv[arguments.size() + 1] = nullptr;
 
-    return executeImpl(path.data(), argv.data(), false);
+    return executeImpl(path.data(), argv.data(), false, terminate_in_destructor);
 }
 
 

dbms/src/Common/ShellCommand.h

@@ -28,11 +28,13 @@ class ShellCommand
 private:
     pid_t pid;
     bool wait_called = false;
+    bool terminate_in_destructor;
 
-    ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd)
+    Poco::Logger * log;
-        : pid(pid), in(in_fd), out(out_fd), err(err_fd) {}
 
-    static std::unique_ptr<ShellCommand> executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only);
+    ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd, bool terminate_in_destructor_);
+
+    static std::unique_ptr<ShellCommand> executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor);
 
 public:
     WriteBufferFromFile in;        /// If the command reads from stdin, do not forget to call in.close() after writing all the data there.
@@ -41,11 +43,13 @@ public:
 
     ~ShellCommand();
 
-    /// Run the command using /bin/sh -c
+    /// Run the command using /bin/sh -c.
-    static std::unique_ptr<ShellCommand> execute(const std::string & command, bool pipe_stdin_only = false);
+    /// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
+    static std::unique_ptr<ShellCommand> execute(const std::string & command, bool pipe_stdin_only = false, bool terminate_in_destructor = false);
 
     /// Run the executable with the specified arguments. `arguments` - without argv[0].
-    static std::unique_ptr<ShellCommand> executeDirect(const std::string & path, const std::vector<std::string> & arguments);
+    /// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
+    static std::unique_ptr<ShellCommand> executeDirect(const std::string & path, const std::vector<std::string> & arguments, bool terminate_in_destructor = false);
 
     /// Wait for the process to end, throw an exception if the code is not 0 or if the process was not completed by itself.
     void wait();

dbms/src/Common/StatusFile.cpp

@@ -20,6 +20,14 @@
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int CANNOT_OPEN_FILE;
+    extern const int CANNOT_CLOSE_FILE;
+    extern const int CANNOT_TRUNCATE_FILE;
+    extern const int CANNOT_SEEK_THROUGH_FILE;
+}
+
 
 StatusFile::StatusFile(const std::string & path_)
     : path(path_)
@@ -43,7 +51,7 @@ StatusFile::StatusFile(const std::string & path_)
     fd = ::open(path.c_str(), O_WRONLY | O_CREAT, 0666);
 
     if (-1 == fd)
-        throwFromErrno("Cannot open file " + path);
+        throwFromErrno("Cannot open file " + path, ErrorCodes::CANNOT_OPEN_FILE);
 
     try
     {
@@ -51,16 +59,16 @@ StatusFile::StatusFile(const std::string & path_)
         if (-1 == flock_ret)
         {
             if (errno == EWOULDBLOCK)
-                throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running.");
+                throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running.", ErrorCodes::CANNOT_OPEN_FILE);
             else
-                throwFromErrno("Cannot lock file " + path);
+                throwFromErrno("Cannot lock file " + path, ErrorCodes::CANNOT_OPEN_FILE);
         }
 
         if (0 != ftruncate(fd, 0))
-            throwFromErrno("Cannot ftruncate " + path);
+            throwFromErrno("Cannot ftruncate " + path, ErrorCodes::CANNOT_TRUNCATE_FILE);
 
         if (0 != lseek(fd, 0, SEEK_SET))
-            throwFromErrno("Cannot lseek " + path);
+            throwFromErrno("Cannot lseek " + path, ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
 
         /// Write information about current server instance to the file.
         {
@@ -82,10 +90,10 @@ StatusFile::StatusFile(const std::string & path_)
 StatusFile::~StatusFile()
 {
     if (0 != close(fd))
-        LOG_ERROR(&Logger::get("StatusFile"), "Cannot close file " << path << ", " << errnoToString());
+        LOG_ERROR(&Logger::get("StatusFile"), "Cannot close file " << path << ", " << errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
 
     if (0 != unlink(path.c_str()))
-        LOG_ERROR(&Logger::get("StatusFile"), "Cannot unlink file " << path << ", " << errnoToString());
+        LOG_ERROR(&Logger::get("StatusFile"), "Cannot unlink file " << path << ", " << errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
 }
 
 }

dbms/src/Common/XDBCBridgeHelper.h

@@ -68,6 +68,7 @@ protected:
 public:
     using Configuration = Poco::Util::AbstractConfiguration;
 
+    Context & context;
     const Configuration & config;
 
     static constexpr inline auto DEFAULT_HOST = "localhost";
@@ -78,8 +79,8 @@ public:
     static constexpr inline auto IDENTIFIER_QUOTE_HANDLER = "/identifier_quote";
     static constexpr inline auto PING_OK_ANSWER = "Ok.";
 
-    XDBCBridgeHelper(const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_)
+    XDBCBridgeHelper(Context & global_context_, const Poco::Timespan & http_timeout_, const std::string & connection_string_)
-        : http_timeout(http_timeout_), connection_string(connection_string_), config(config_)
+        : http_timeout(http_timeout_), connection_string(connection_string_), context(global_context_), config(context.getConfigRef())
     {
         size_t bridge_port = config.getUInt(BridgeHelperMixin::configPrefix() + ".port", DEFAULT_PORT);
         std::string bridge_host = config.getString(BridgeHelperMixin::configPrefix() + ".host", DEFAULT_HOST);
@@ -210,7 +211,8 @@ private:
     /* Contains logic for instantiation of the bridge instance */
     void startBridge() const
     {
-        BridgeHelperMixin::startBridge(config, log, http_timeout);
+        auto cmd = BridgeHelperMixin::startBridge(config, log, http_timeout);
+        context.addXDBCBridgeCommand(std::move(cmd));
     }
 };
 
@@ -230,7 +232,7 @@ struct JDBCBridgeMixin
         return "JDBC";
     }
 
-    static void startBridge(const Poco::Util::AbstractConfiguration &, const Poco::Logger *, const Poco::Timespan &)
+    static std::unique_ptr<ShellCommand> startBridge(const Poco::Util::AbstractConfiguration &, const Poco::Logger *, const Poco::Timespan &)
     {
         throw Exception("jdbc-bridge is not running. Please, start it manually", ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING);
     }
@@ -253,11 +255,13 @@ struct ODBCBridgeMixin
         return "ODBC";
     }
 
-    static void startBridge(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout)
+    static std::unique_ptr<ShellCommand> startBridge(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout)
     {
         /// Path to executable folder
         Poco::Path path{config.getString("application.dir", "/usr/bin")};
 
+
+        std::vector<std::string> cmd_args;
         path.setFileName(
 #if CLICKHOUSE_SPLIT_BINARY
             "clickhouse-odbc-bridge"
@@ -268,34 +272,35 @@ struct ODBCBridgeMixin
 
         std::stringstream command;
 
-        command << path.toString() <<
+#if !CLICKHOUSE_SPLIT_BINARY
-#if CLICKHOUSE_SPLIT_BINARY
+        cmd_args.push_back("odbc-bridge");
-            " "
-#else
-            " odbc-bridge "
 #endif
-            ;
 
-        command << "--http-port " << config.getUInt(configPrefix() + ".port", DEFAULT_PORT) << ' ';
+        cmd_args.push_back("--http-port");
-        command << "--listen-host " << config.getString(configPrefix() + ".listen_host", XDBCBridgeHelper<ODBCBridgeMixin>::DEFAULT_HOST)
+        cmd_args.push_back(std::to_string(config.getUInt(configPrefix() + ".port", DEFAULT_PORT)));
-                << ' ';
+        cmd_args.push_back("--listen-host");
-        command << "--http-timeout " << http_timeout.totalMicroseconds() << ' ';
+        cmd_args.push_back(config.getString(configPrefix() + ".listen_host", XDBCBridgeHelper<ODBCBridgeMixin>::DEFAULT_HOST));
+        cmd_args.push_back("--http-timeout");
+        cmd_args.push_back(std::to_string(http_timeout.totalMicroseconds()));
         if (config.has("logger." + configPrefix() + "_log"))
-            command << "--log-path " << config.getString("logger." + configPrefix() + "_log") << ' ';
+        {
+            cmd_args.push_back("--log-path");
+            cmd_args.push_back(config.getString("logger." + configPrefix() + "_log"));
+        }
         if (config.has("logger." + configPrefix() + "_errlog"))
-            command << "--err-log-path " << config.getString("logger." + configPrefix() + "_errlog") << ' ';
+        {
+            cmd_args.push_back("--err-log-path");
+            cmd_args.push_back(config.getString("logger." + configPrefix() + "_errlog"));
+        }
         if (config.has("logger." + configPrefix() + "_level"))
-            command << "--log-level " << config.getString("logger." + configPrefix() + "_level") << ' ';
+        {
-        command << "&"; /// we don't want to wait this process
+            cmd_args.push_back("--log-level");
+            cmd_args.push_back(config.getString("logger." + configPrefix() + "_level"));
+        }
 
-        auto command_str = command.str();
+        LOG_TRACE(log, "Starting " + serviceAlias());
 
-        std::cerr << command_str << std::endl;
+        return ShellCommand::executeDirect(path.toString(), cmd_args, true);
-
-        LOG_TRACE(log, "Starting " + serviceAlias() + " with command: " << command_str);
-
-        auto cmd = ShellCommand::execute(command_str);
-        cmd->wait();
     }
 };
 }

dbms/src/Common/ZooKeeper/CMakeLists.txt

@@ -4,7 +4,8 @@ add_headers_and_sources(clickhouse_common_zookeeper .)
 
 add_library(clickhouse_common_zookeeper ${LINK_MODE} ${clickhouse_common_zookeeper_headers} ${clickhouse_common_zookeeper_sources})
 
-target_link_libraries (clickhouse_common_zookeeper clickhouse_common_io)
+target_link_libraries (clickhouse_common_zookeeper PUBLIC clickhouse_common_io common PRIVATE string_utils PUBLIC ${Poco_Util_LIBRARY})
+target_include_directories(clickhouse_common_zookeeper PUBLIC ${DBMS_INCLUDE_DIR})
 
 if (ENABLE_TESTS)
     add_subdirectory (tests)

dbms/src/Common/ZooKeeper/ZooKeeperHolder.cpp

@@ -1,5 +1,15 @@
 #include "ZooKeeperHolder.h"
 
+
+namespace DB
+{
+    namespace ErrorCodes
+    {
+        extern const int NULL_POINTER_DEREFERENCE;
+    }
+}
+
+
 using namespace zkutil;
 
 ZooKeeperHolder::UnstorableZookeeperHandler ZooKeeperHolder::getZooKeeper()
@@ -47,7 +57,7 @@ ZooKeeperHolder::UnstorableZookeeperHandler::UnstorableZookeeperHandler(ZooKeepe
 ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->()
 {
     if (zk_ptr == nullptr)
-        throw DB::Exception(nullptr_exception_message);
+        throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
 
     return zk_ptr.get();
 }
@@ -55,20 +65,20 @@ ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->()
 const ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->() const
 {
     if (zk_ptr == nullptr)
-        throw DB::Exception(nullptr_exception_message);
+        throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
     return zk_ptr.get();
 }
 
 ZooKeeper & ZooKeeperHolder::UnstorableZookeeperHandler::operator*()
 {
     if (zk_ptr == nullptr)
-        throw DB::Exception(nullptr_exception_message);
+        throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
     return *zk_ptr;
 }
 
 const ZooKeeper & ZooKeeperHolder::UnstorableZookeeperHandler::operator*() const
 {
     if (zk_ptr == nullptr)
-        throw DB::Exception(nullptr_exception_message);
+        throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
     return *zk_ptr;
 }

dbms/src/Common/ZooKeeper/tests/CMakeLists.txt

@@ -1,23 +1,23 @@
 add_executable(zkutil_test_commands zkutil_test_commands.cpp)
-target_link_libraries(zkutil_test_commands clickhouse_common_zookeeper)
+target_link_libraries(zkutil_test_commands PRIVATE clickhouse_common_zookeeper)
 
 add_executable(zkutil_test_commands_new_lib zkutil_test_commands_new_lib.cpp)
-target_link_libraries(zkutil_test_commands_new_lib clickhouse_common_zookeeper)
+target_link_libraries(zkutil_test_commands_new_lib PRIVATE clickhouse_common_zookeeper)
 
 add_executable(zkutil_test_lock zkutil_test_lock.cpp)
-target_link_libraries(zkutil_test_lock clickhouse_common_zookeeper)
+target_link_libraries(zkutil_test_lock PRIVATE clickhouse_common_zookeeper)
 
 add_executable(zkutil_expiration_test zkutil_expiration_test.cpp)
-target_link_libraries(zkutil_expiration_test clickhouse_common_zookeeper)
+target_link_libraries(zkutil_expiration_test PRIVATE clickhouse_common_zookeeper)
 
 add_executable(zkutil_test_async zkutil_test_async.cpp)
-target_link_libraries(zkutil_test_async clickhouse_common_zookeeper)
+target_link_libraries(zkutil_test_async PRIVATE clickhouse_common_zookeeper)
 
 add_executable(zkutil_zookeeper_holder zkutil_zookeeper_holder.cpp)
-target_link_libraries(zkutil_zookeeper_holder clickhouse_common_zookeeper)
+target_link_libraries(zkutil_zookeeper_holder PRIVATE clickhouse_common_zookeeper)
 
 add_executable (zk_many_watches_reconnect zk_many_watches_reconnect.cpp)
-target_link_libraries (zk_many_watches_reconnect clickhouse_common_zookeeper clickhouse_common_config)
+target_link_libraries (zk_many_watches_reconnect PRIVATE clickhouse_common_zookeeper clickhouse_common_config)
 
 add_executable (zookeeper_impl zookeeper_impl.cpp)
-target_link_libraries (zookeeper_impl clickhouse_common_zookeeper)
+target_link_libraries (zookeeper_impl PRIVATE clickhouse_common_zookeeper)

dbms/src/Common/config.h.in

@@ -16,3 +16,4 @@
 #cmakedefine01 USE_POCO_NETSSL
 #cmakedefine01 CLICKHOUSE_SPLIT_BINARY
 #cmakedefine01 ENABLE_INSERT_INFILE
+#cmakedefine01 USE_BASE64

dbms/src/Common/config_version.h.in

@@ -23,6 +23,7 @@
 #cmakedefine VERSION_FULL "@VERSION_FULL@"
 #cmakedefine VERSION_DESCRIBE "@VERSION_DESCRIBE@"
 #cmakedefine VERSION_GITHASH "@VERSION_GITHASH@"
+#cmakedefine VERSION_INTEGER @VERSION_INTEGER@
 
 #if defined(VERSION_MAJOR)
 #define DBMS_VERSION_MAJOR VERSION_MAJOR

dbms/src/Common/createHardLink.cpp

@@ -8,6 +8,12 @@
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int CANNOT_STAT;
+    extern const int CANNOT_LINK;
+}
+
 void createHardLink(const String & source_path, const String & destination_path)
 {
     if (0 != link(source_path.c_str(), destination_path.c_str()))
@@ -20,16 +26,16 @@ void createHardLink(const String & source_path, const String & destination_path)
             struct stat destination_descr;
 
             if (0 != lstat(source_path.c_str(), &source_descr))
-                throwFromErrno("Cannot stat " + source_path);
+                throwFromErrno("Cannot stat " + source_path, ErrorCodes::CANNOT_STAT);
 
             if (0 != lstat(destination_path.c_str(), &destination_descr))
-                throwFromErrno("Cannot stat " + destination_path);
+                throwFromErrno("Cannot stat " + destination_path, ErrorCodes::CANNOT_STAT);
 
             if (source_descr.st_ino != destination_descr.st_ino)
-                throwFromErrno("Destination file " + destination_path + " is already exist and have different inode.", 0, link_errno);
+                throwFromErrno("Destination file " + destination_path + " is already exist and have different inode.", ErrorCodes::CANNOT_LINK, link_errno);
         }
         else
-            throwFromErrno("Cannot link " + source_path + " to " + destination_path);
+            throwFromErrno("Cannot link " + source_path + " to " + destination_path, ErrorCodes::CANNOT_LINK);
     }
 }
 

dbms/src/Common/setThreadName.cpp

@@ -32,7 +32,7 @@ void setThreadName(const char * name)
 #else
     if (0 != prctl(PR_SET_NAME, name, 0, 0, 0))
 #endif
-        DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME, ...)");
+        DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME, ...)", DB::ErrorCodes::PTHREAD_ERROR);
 }
 
 std::string getThreadName()
@@ -48,7 +48,7 @@ std::string getThreadName()
 //        throw DB::Exception("Cannot get thread name with pthread_get_name_np()", DB::ErrorCodes::PTHREAD_ERROR);
 #else
     if (0 != prctl(PR_GET_NAME, name.data(), 0, 0, 0))
-        DB::throwFromErrno("Cannot get thread name with prctl(PR_GET_NAME)");
+        DB::throwFromErrno("Cannot get thread name with prctl(PR_GET_NAME)", DB::ErrorCodes::PTHREAD_ERROR);
 #endif
 
     name.resize(std::strlen(name.data()));

dbms/src/Common/tests/CMakeLists.txt

@@ -1,73 +1,73 @@
 add_executable (hashes_test hashes_test.cpp)
-target_link_libraries (hashes_test dbms)
+target_link_libraries (hashes_test PRIVATE dbms)
 
 add_executable (sip_hash sip_hash.cpp)
-target_link_libraries (sip_hash clickhouse_common_io)
+target_link_libraries (sip_hash PRIVATE clickhouse_common_io)
 
 add_executable (sip_hash_perf sip_hash_perf.cpp)
-target_link_libraries (sip_hash_perf clickhouse_common_io)
+target_link_libraries (sip_hash_perf PRIVATE clickhouse_common_io)
 
 add_executable (auto_array auto_array.cpp)
-target_link_libraries (auto_array clickhouse_common_io)
+target_link_libraries (auto_array PRIVATE clickhouse_common_io)
 
 add_executable (lru_cache lru_cache.cpp)
-target_link_libraries (lru_cache clickhouse_common_io)
+target_link_libraries (lru_cache PRIVATE clickhouse_common_io)
 
 add_executable (hash_table hash_table.cpp)
-target_link_libraries (hash_table clickhouse_common_io)
+target_link_libraries (hash_table PRIVATE clickhouse_common_io)
 
 add_executable (small_table small_table.cpp)
-target_link_libraries (small_table clickhouse_common_io)
+target_link_libraries (small_table PRIVATE clickhouse_common_io)
 
 add_executable (parallel_aggregation parallel_aggregation.cpp)
-target_link_libraries (parallel_aggregation clickhouse_common_io)
+target_link_libraries (parallel_aggregation PRIVATE clickhouse_common_io)
 
 add_executable (parallel_aggregation2 parallel_aggregation2.cpp)
-target_link_libraries (parallel_aggregation2 clickhouse_common_io)
+target_link_libraries (parallel_aggregation2 PRIVATE clickhouse_common_io)
 
 add_executable (int_hashes_perf int_hashes_perf.cpp AvalancheTest.cpp Random.cpp)
-target_link_libraries (int_hashes_perf clickhouse_common_io)
+target_link_libraries (int_hashes_perf PRIVATE clickhouse_common_io)
 
 add_executable (simple_cache simple_cache.cpp)
 target_include_directories (simple_cache PRIVATE ${DBMS_INCLUDE_DIR})
-target_link_libraries (simple_cache common)
+target_link_libraries (simple_cache PRIVATE common)
 
 add_executable (compact_array compact_array.cpp)
-target_link_libraries (compact_array clickhouse_common_io ${Boost_FILESYSTEM_LIBRARY})
+target_link_libraries (compact_array PRIVATE clickhouse_common_io ${Boost_FILESYSTEM_LIBRARY})
 
 add_executable (radix_sort radix_sort.cpp)
-target_link_libraries (radix_sort clickhouse_common_io)
+target_link_libraries (radix_sort PRIVATE clickhouse_common_io)
 
 add_executable (shell_command_test shell_command_test.cpp)
-target_link_libraries (shell_command_test clickhouse_common_io)
+target_link_libraries (shell_command_test PRIVATE clickhouse_common_io)
 
 add_executable (arena_with_free_lists arena_with_free_lists.cpp)
-target_link_libraries (arena_with_free_lists clickhouse_common_io)
+target_link_libraries (arena_with_free_lists PRIVATE clickhouse_common_io)
 
 add_executable (pod_array pod_array.cpp)
-target_link_libraries (pod_array clickhouse_common_io)
+target_link_libraries (pod_array PRIVATE clickhouse_common_io)
 
 add_executable (thread_creation_latency thread_creation_latency.cpp)
-target_link_libraries (thread_creation_latency clickhouse_common_io)
+target_link_libraries (thread_creation_latency PRIVATE clickhouse_common_io)
 
 add_executable (thread_pool thread_pool.cpp)
-target_link_libraries (thread_pool clickhouse_common_io)
+target_link_libraries (thread_pool PRIVATE clickhouse_common_io)
 
 add_executable (array_cache array_cache.cpp)
-target_link_libraries (array_cache clickhouse_common_io)
+target_link_libraries (array_cache PRIVATE clickhouse_common_io)
 
 add_executable (space_saving space_saving.cpp)
-target_link_libraries (space_saving clickhouse_common_io)
+target_link_libraries (space_saving PRIVATE clickhouse_common_io)
 
 add_executable (integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp)
 target_include_directories (integer_hash_tables_and_hashes SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
-target_link_libraries (integer_hash_tables_and_hashes clickhouse_common_io)
+target_link_libraries (integer_hash_tables_and_hashes PRIVATE clickhouse_common_io)
 
 add_executable (allocator allocator.cpp)
-target_link_libraries (allocator clickhouse_common_io)
+target_link_libraries (allocator PRIVATE clickhouse_common_io)
 
 add_executable (cow_columns cow_columns.cpp)
-target_link_libraries (cow_columns clickhouse_common_io)
+target_link_libraries (cow_columns PRIVATE clickhouse_common_io)
 
 add_executable (stopwatch stopwatch.cpp)
-target_link_libraries (stopwatch clickhouse_common_io)
+target_link_libraries (stopwatch PRIVATE clickhouse_common_io)

dbms/src/Common/tests/arena_with_free_lists.cpp

@@ -25,6 +25,14 @@
 
 using namespace DB;
 
+namespace DB
+{
+    namespace ErrorCodes
+    {
+        extern const int SYSTEM_ERROR;
+    }
+}
+
 
 /// Implementation of ArenaWithFreeLists, which contains a bug. Used to reproduce the bug.
 #if USE_BAD_ARENA
@@ -237,7 +245,7 @@ int main(int argc, char ** argv)
 
         rusage resource_usage;
         if (0 != getrusage(RUSAGE_SELF, &resource_usage))
-            throwFromErrno("Cannot getrusage");
+            throwFromErrno("Cannot getrusage", ErrorCodes::SYSTEM_ERROR);
 
         size_t allocated_bytes = resource_usage.ru_maxrss * 1024;
         std::cerr << "Current memory usage: " << allocated_bytes << " bytes.\n";

dbms/src/Common/tests/thread_creation_latency.cpp

@@ -22,6 +22,15 @@ void     f()         { ++x; }
 void *     g(void *)     { f(); return {}; }
 
 
+namespace DB
+{
+    namespace ErrorCodes
+    {
+        extern const int PTHREAD_ERROR;
+    }
+}
+
+
 template <typename F>
 void test(size_t n, const char * name, F && kernel)
 {
@@ -80,9 +89,9 @@ int main(int argc, char ** argv)
     {
         pthread_t thread;
         if (pthread_create(&thread, nullptr, g, nullptr))
-            DB::throwFromErrno("Cannot create thread.");
+            DB::throwFromErrno("Cannot create thread.", DB::ErrorCodes::PTHREAD_ERROR);
         if (pthread_join(thread, nullptr))
-            DB::throwFromErrno("Cannot join thread.");
+            DB::throwFromErrno("Cannot join thread.", DB::ErrorCodes::PTHREAD_ERROR);
     });
 
     test(n, "Create and destroy std::thread each iteration", []

dbms/src/Core/Defines.h

@@ -62,6 +62,9 @@
 
 #define DEFAULT_HTTP_READ_BUFFER_TIMEOUT 1800
 #define DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT 1
+/// Maximum namber of http-connections between two endpoints
+/// the number is unmotivated
+#define DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT 15
 
 // more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html
 

dbms/src/Core/Field.h

@@ -578,43 +578,54 @@ template <> struct TypeName<Array> { static std::string get() { return "Array";
 template <> struct TypeName<Tuple> { static std::string get() { return "Tuple"; } };
 
 
-template <typename T> struct NearestFieldType;
+template <typename T> struct NearestFieldTypeImpl;
 
-template <> struct NearestFieldType<UInt8>   { using Type = UInt64; };
+/// char may be signed or unsigned, and behave identically to signed char or unsigned char,
-template <> struct NearestFieldType<UInt16>  { using Type = UInt64; };
+///  but they are always three different types.
-template <> struct NearestFieldType<UInt32>  { using Type = UInt64; };
+/// signedness of char is different in Linux on x86 and Linux on ARM.
-template <> struct NearestFieldType<UInt64>  { using Type = UInt64; };
+template <> struct NearestFieldTypeImpl<char> { using Type = std::conditional_t<std::is_signed_v<char>, Int64, UInt64>; };
-#ifdef __APPLE__
+template <> struct NearestFieldTypeImpl<signed char> { using Type = Int64; };
-template <> struct NearestFieldType<time_t>  { using Type = UInt64; };
+template <> struct NearestFieldTypeImpl<unsigned char> { using Type = UInt64; };
-template <> struct NearestFieldType<size_t>  { using Type = UInt64; };
+
-#endif
+template <> struct NearestFieldTypeImpl<UInt16> { using Type = UInt64; };
-template <> struct NearestFieldType<DayNum>  { using Type = UInt64; };
+template <> struct NearestFieldTypeImpl<UInt32> { using Type = UInt64; };
-template <> struct NearestFieldType<UInt128> { using Type = UInt128; };
+
-template <> struct NearestFieldType<UUID>  { using Type = UInt128; };
+template <> struct NearestFieldTypeImpl<DayNum> { using Type = UInt64; };
-template <> struct NearestFieldType<Int8>    { using Type = Int64; };
+template <> struct NearestFieldTypeImpl<UInt128> { using Type = UInt128; };
-template <> struct NearestFieldType<Int16>   { using Type = Int64; };
+template <> struct NearestFieldTypeImpl<UUID> { using Type = UInt128; };
-template <> struct NearestFieldType<Int32>   { using Type = Int64; };
+template <> struct NearestFieldTypeImpl<Int16> { using Type = Int64; };
-template <> struct NearestFieldType<Int64>   { using Type = Int64; };
+template <> struct NearestFieldTypeImpl<Int32> { using Type = Int64; };
-template <> struct NearestFieldType<Int128>  { using Type = Int128; };
+
-template <> struct NearestFieldType<Decimal32>   { using Type = DecimalField<Decimal32>; };
+/// long and long long are always different types that may behave identically or not.
-template <> struct NearestFieldType<Decimal64>   { using Type = DecimalField<Decimal64>; };
+/// This is different on Linux and Mac.
-template <> struct NearestFieldType<Decimal128>  { using Type = DecimalField<Decimal128>; };
+template <> struct NearestFieldTypeImpl<long> { using Type = Int64; };
-template <> struct NearestFieldType<DecimalField<Decimal32>>   { using Type = DecimalField<Decimal32>; };
+template <> struct NearestFieldTypeImpl<long long> { using Type = Int64; };
-template <> struct NearestFieldType<DecimalField<Decimal64>>   { using Type = DecimalField<Decimal64>; };
+template <> struct NearestFieldTypeImpl<unsigned long> { using Type = UInt64; };
-template <> struct NearestFieldType<DecimalField<Decimal128>>  { using Type = DecimalField<Decimal128>; };
+template <> struct NearestFieldTypeImpl<unsigned long long> { using Type = UInt64; };
-template <> struct NearestFieldType<Float32> { using Type = Float64; };
+
-template <> struct NearestFieldType<Float64> { using Type = Float64; };
+template <> struct NearestFieldTypeImpl<Int128> { using Type = Int128; };
-template <> struct NearestFieldType<const char*>  { using Type = String; };
+template <> struct NearestFieldTypeImpl<Decimal32> { using Type = DecimalField<Decimal32>; };
-template <> struct NearestFieldType<String>  { using Type = String; };
+template <> struct NearestFieldTypeImpl<Decimal64> { using Type = DecimalField<Decimal64>; };
-template <> struct NearestFieldType<Array>   { using Type = Array; };
+template <> struct NearestFieldTypeImpl<Decimal128> { using Type = DecimalField<Decimal128>; };
-template <> struct NearestFieldType<Tuple>   { using Type = Tuple; };
+template <> struct NearestFieldTypeImpl<DecimalField<Decimal32>> { using Type = DecimalField<Decimal32>; };
-template <> struct NearestFieldType<bool>    { using Type = UInt64; };
+template <> struct NearestFieldTypeImpl<DecimalField<Decimal64>> { using Type = DecimalField<Decimal64>; };
-template <> struct NearestFieldType<Null>    { using Type = Null; };
+template <> struct NearestFieldTypeImpl<DecimalField<Decimal128>> { using Type = DecimalField<Decimal128>; };
+template <> struct NearestFieldTypeImpl<Float32> { using Type = Float64; };
+template <> struct NearestFieldTypeImpl<Float64> { using Type = Float64; };
+template <> struct NearestFieldTypeImpl<const char *> { using Type = String; };
+template <> struct NearestFieldTypeImpl<String> { using Type = String; };
+template <> struct NearestFieldTypeImpl<Array> { using Type = Array; };
+template <> struct NearestFieldTypeImpl<Tuple> { using Type = Tuple; };
+template <> struct NearestFieldTypeImpl<bool> { using Type = UInt64; };
+template <> struct NearestFieldTypeImpl<Null> { using Type = Null; };
+
+template <typename T>
+using NearestFieldType = typename NearestFieldTypeImpl<T>::Type;
 
 template <typename T>
 decltype(auto) nearestFieldType(T && x)
 {
-    using U = typename NearestFieldType<std::decay_t<T>>::Type;
+    using U = NearestFieldType<std::decay_t<T>>;
     if constexpr (std::is_same_v<std::decay_t<T>, U>)
         return std::forward<T>(x);
     else

dbms/src/Core/Names.h

@@ -2,6 +2,7 @@
 
 #include <vector>
 #include <string>
+#include <set>
 #include <unordered_set>
 #include <unordered_map>
 
@@ -11,6 +12,7 @@ namespace DB
 
 using Names = std::vector<std::string>;
 using NameSet = std::unordered_set<std::string>;
+using NameOrderedSet = std::set<std::string>;
 using NameToNameMap = std::unordered_map<std::string, std::string>;
 using NameToNameSetMap = std::unordered_map<std::string, NameSet>;
 

dbms/src/Core/tests/CMakeLists.txt

@@ -1,18 +1,15 @@
-add_executable (exception exception.cpp)
-target_link_libraries (exception clickhouse_common_io)
-
 add_executable (string_pool string_pool.cpp)
-target_link_libraries (string_pool clickhouse_common_io)
+target_link_libraries (string_pool PRIVATE clickhouse_common_io)
 target_include_directories (string_pool SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
 
 add_executable (field field.cpp)
-target_link_libraries (field dbms)
+target_link_libraries (field PRIVATE dbms)
 
 add_executable (move_field move_field.cpp)
-target_link_libraries (move_field clickhouse_common_io)
+target_link_libraries (move_field PRIVATE clickhouse_common_io)
 
 add_executable (rvo_test rvo_test.cpp)
-target_link_libraries (rvo_test Threads::Threads)
+target_link_libraries (rvo_test PRIVATE Threads::Threads)
 
 add_executable (string_ref_hash string_ref_hash.cpp)
-target_link_libraries (string_ref_hash clickhouse_common_io)
+target_link_libraries (string_ref_hash PRIVATE clickhouse_common_io)

dbms/src/Core/tests/exception.cpp

@@ -1,25 +0,0 @@
-#include <iostream>
-
-#include <Poco/Net/NetException.h>
-
-#include <Common/Exception.h>
-
-
-int main(int, char **)
-{
-    try
-    {
-        //throw Poco::Net::ConnectionRefusedException();
-        throw DB::Exception(Poco::Net::ConnectionRefusedException());
-    }
-    catch (const DB::Exception & e)
-    {
-        std::cerr << e.displayText() << std::endl;
-    }
-    catch (const Poco::Exception & e)
-    {
-        std::cerr << e.displayText() << std::endl;
-    }
-
-    return 0;
-}

dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp

@@ -179,8 +179,9 @@ void MergingAggregatedMemoryEfficientBlockInputStream::start()
             auto thread_group = CurrentThread::getGroup();
             reading_pool->schedule([&child, thread_group]
             {
-                CurrentThread::attachToIfDetached(thread_group);
                 setThreadName("MergeAggReadThr");
+                if (thread_group)
+                    CurrentThread::attachToIfDetached(thread_group);
                 CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread};
                 child->readPrefix();
             });
@@ -485,8 +486,9 @@ MergingAggregatedMemoryEfficientBlockInputStream::BlocksToMerge MergingAggregate
                 auto thread_group = CurrentThread::getGroup();
                 reading_pool->schedule([&input, &read_from_input, thread_group]
                 {
-                    CurrentThread::attachToIfDetached(thread_group);
                     setThreadName("MergeAggReadThr");
+                    if (thread_group)
+                        CurrentThread::attachToIfDetached(thread_group);
                     CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread};
                     read_from_input(input);
                 });

dbms/src/DataStreams/ParallelInputsProcessor.h

@@ -107,9 +107,27 @@ public:
         active_threads = max_threads;
         threads.reserve(max_threads);
         auto thread_group = CurrentThread::getGroup();
+
+        try
+        {
             for (size_t i = 0; i < max_threads; ++i)
                 threads.emplace_back([=] () { thread(thread_group, i); } );
         }
+        catch (...)
+        {
+            cancel(false);
+            wait();
+            if (active_threads)
+            {
+                active_threads = 0;
+                /// handler.onFinish() is supposed to be called from one of the threads when the number of
+                /// finished threads reaches max_threads. But since we weren't able to launch all threads,
+                /// we have to call onFinish() manually here.
+                handler.onFinish();
+            }
+            throw;
+        }
+    }
 
     /// Ask all sources to stop earlier than they run out.
     void cancel(bool kill)

dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp

@@ -91,6 +91,7 @@ void PushingToViewsBlockOutputStream::write(const Block & block)
             pool.schedule([=]
             {
                 setThreadName("PushingToViewsBlockOutputStream");
+                if (thread_group)
                     CurrentThread::attachToIfDetached(thread_group);
                 process(block, view_num);
             });

dbms/src/DataStreams/tests/CMakeLists.txt

@@ -1,16 +1,16 @@
 set(SRCS)
 
 add_executable (expression_stream expression_stream.cpp ${SRCS})
-target_link_libraries (expression_stream dbms clickhouse_storages_system)
+target_link_libraries (expression_stream PRIVATE dbms clickhouse_storages_system clickhouse_parsers)
 
 add_executable (filter_stream filter_stream.cpp ${SRCS})
-target_link_libraries (filter_stream dbms clickhouse_storages_system)
+target_link_libraries (filter_stream PRIVATE dbms clickhouse_storages_system clickhouse_parsers clickhouse_common_io)
 
 add_executable (union_stream2 union_stream2.cpp ${SRCS})
-target_link_libraries (union_stream2 dbms)
+target_link_libraries (union_stream2 PRIVATE dbms)
 
 add_executable (collapsing_sorted_stream collapsing_sorted_stream.cpp ${SRCS})
-target_link_libraries (collapsing_sorted_stream dbms)
+target_link_libraries (collapsing_sorted_stream PRIVATE dbms)
 
 add_executable (finish_sorting_stream finish_sorting_stream.cpp ${SRCS})
-target_link_libraries (finish_sorting_stream dbms)
+target_link_libraries (finish_sorting_stream PRIVATE dbms)