Merge branch 'master' into decimal

.gitignore

@@ -178,7 +178,6 @@ utils/zookeeper-create-entry-to-download-part/zookeeper-create-entry-to-download
 utils/zookeeper-dump-tree/zookeeper-dump-tree
 utils/zookeeper-remove-by-list/zookeeper-remove-by-list
 dbms/src/Storages/tests/remove_symlink_directory
-dbms/tests/queries/1_stateful
 debian/control
 debian/copyright
 debian/tmp/

.gitmodules

@@ -15,7 +15,7 @@
 	url = https://github.com/google/cctz.git
 [submodule "contrib/zlib-ng"]
 	path = contrib/zlib-ng
-	url = https://github.com/Dead2/zlib-ng.git
+	url = https://github.com/ClickHouse-Extras/zlib-ng.git
 [submodule "contrib/googletest"]
 	path = contrib/googletest
 	url = https://github.com/google/googletest.git

CHANGELOG.md

@@ -36,6 +36,10 @@
 * Fixed a bug in the `anyHeavy`  aggregate function ([a2101df2](https://github.com/yandex/ClickHouse/commit/a2101df25a6a0fba99aa71f8793d762af2b801ee))
 * Fixed server crash when using the `countArray()` aggregate function.
 
+### Backward incompatible changes:
+
+* Parameters for `Kafka` engine was changed from `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_schema, kafka_num_consumers])` to `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_row_delimiter, kafka_schema, kafka_num_consumers])`. If your tables use `kafka_schema` or `kafka_num_consumers` parameters, you have to manually edit the metadata files `path/metadata/database/table.sql` and add `kafka_row_delimiter` parameter with `''` value.
+
 ## ClickHouse release 18.1.0, 2018-07-23
 
 ### New features:

CHANGELOG_RU.md

@@ -43,6 +43,10 @@
 * Исправлена ошибка в агрегатной функции `anyHeavy` ([a2101df2](https://github.com/yandex/ClickHouse/commit/a2101df25a6a0fba99aa71f8793d762af2b801ee))
 * Исправлено падение сервера при использовании функции `countArray()`.
 
+### Обратно несовместимые изменения:
+
+* Список параметров для таблиц `Kafka` был изменён с `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_schema, kafka_num_consumers])` на `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_row_delimiter, kafka_schema, kafka_num_consumers])`. Если вы использовали параметры `kafka_schema` или `kafka_num_consumers`, вам необходимо вручную отредактировать файлы с метаданными `path/metadata/database/table.sql`, добавив параметр `kafka_row_delimiter` со значением `''` в соответствующее место.
+
 
 ## ClickHouse release 18.1.0, 2018-07-23
 

CMakeLists.txt

@@ -34,10 +34,9 @@ endif ()
 string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
 message (STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE} )
 
-# ASan - build type with address sanitizer
+set (CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel" CACHE STRING "" FORCE)
-# UBSan - build type with undefined behaviour sanitizer
+
-# TSan is not supported due to false positive errors in libstdc++ and necessity to rebuild libstdc++ with TSan
+include (cmake/sanitize.cmake)
-set (CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel;ASan;UBSan" CACHE STRING "" FORCE)
 
 include (cmake/arch.cmake)
 
@@ -62,7 +61,7 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
 endif ()
 
 option (TEST_COVERAGE "Enables flags for test coverage" OFF)
-option (ENABLE_TESTS "Enables tests" ${NOT_MSVC})
+option (ENABLE_TESTS "Enables tests" ON)
 
 option (USE_STATIC_LIBRARIES "Set to FALSE to use shared libraries" ON)
 option (MAKE_STATIC_LIBRARIES "Set to FALSE to make shared libraries" ${USE_STATIC_LIBRARIES})
@@ -175,7 +174,7 @@ if (OS_LINUX AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
         endif ()
 
         if (LIBCXX_PATH)
-#            include_directories (BEFORE SYSTEM "${LIBCXX_PATH}/include" "${LIBCXX_PATH}/include/c++/v1")
+#            include_directories (SYSTEM BEFORE "${LIBCXX_PATH}/include" "${LIBCXX_PATH}/include/c++/v1")
             link_directories ("${LIBCXX_PATH}/lib")
         endif ()
     endif ()
@@ -190,8 +189,6 @@ if (NOT MAKE_STATIC_LIBRARIES)
     set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 endif ()
 
-include (cmake/sanitize.cmake)
-
 # Using "include-what-you-use" tool.
 option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF)
 if (USE_INCLUDE_WHAT_YOU_USE)

ci/jobs/quick-build/run.sh

@@ -21,7 +21,7 @@ BUILD_TARGETS=clickhouse
 BUILD_TYPE=Debug
 ENABLE_EMBEDDED_COMPILER=0
 
-CMAKE_FLAGS="-D CMAKE_C_FLAGS_ADD=-g0 -D CMAKE_CXX_FLAGS_ADD=-g0 -D ENABLE_TCMALLOC=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_UNWIND=0 -D ENABLE_ICU=0 -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_POCO_ODBC=0 -D ENABLE_ODBC=0 -D ENABLE_MYSQL=0"
+CMAKE_FLAGS="-D CMAKE_C_FLAGS_ADD=-g0 -D CMAKE_CXX_FLAGS_ADD=-g0 -D ENABLE_JEMALLOC=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_UNWIND=0 -D ENABLE_ICU=0 -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_POCO_ODBC=0 -D ENABLE_ODBC=0 -D ENABLE_MYSQL=0"
 
 [[ $(uname) == "FreeBSD" ]] && COMPILER_PACKAGE_VERSION=devel && export COMPILER_PATH=/usr/local/bin
 

cmake/find_capnp.cmake

@@ -1,4 +1,4 @@
-option (ENABLE_CAPNP "Enable Cap'n Proto" ${NOT_MSVC})
+option (ENABLE_CAPNP "Enable Cap'n Proto" ON)
 
 if (ENABLE_CAPNP)
     # cmake 3.5.1 bug:

cmake/find_llvm.cmake

@@ -43,6 +43,12 @@ if (ENABLE_EMBEDDED_COMPILER)
         else()
             set (USE_EMBEDDED_COMPILER 0)
         endif()
+
+        if (LLVM_FOUND AND OS_LINUX AND USE_LIBCXX)
+            message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY is not set but the LLVM library from OS packages in Linux is incompatible with libc++ ABI. LLVM Will be disabled.")
+            set (LLVM_FOUND 0)
+            set (USE_EMBEDDED_COMPILER 0)
+        endif ()
     else()
         set (LLVM_FOUND 1)
         set (USE_EMBEDDED_COMPILER 1)

cmake/find_rdkafka.cmake

@@ -1,4 +1,4 @@
-option (ENABLE_RDKAFKA "Enable kafka" ${NOT_MSVC})
+option (ENABLE_RDKAFKA "Enable kafka" ON)
 
 if (ENABLE_RDKAFKA)
 

cmake/sanitize.cmake

@@ -1,27 +1,37 @@
+option (SANITIZE "Enable sanitizer: address, memory, thread, undefined" "")
+
 set (SAN_FLAGS "${SAN_FLAGS} -g -fno-omit-frame-pointer -DSANITIZER")
-if (SAN_DEBUG)
-    set (SAN_FLAGS "${SAN_FLAGS} -O0")
-else ()
-    set (SAN_FLAGS "${SAN_FLAGS} -O3")
-endif ()
 
-set (CMAKE_CXX_FLAGS_ASAN                "${CMAKE_CXX_FLAGS_ASAN}  ${SAN_FLAGS} -fsanitize=address")
+if (SANITIZE)
-set (CMAKE_C_FLAGS_ASAN                  "${CMAKE_C_FLAGS_ASAN}    ${SAN_FLAGS} -fsanitize=address")
+    if (SANITIZE STREQUAL "address")
-set (CMAKE_EXE_LINKER_FLAGS_ASAN         "${CMAKE_EXE_LINKER_FLAGS_ASAN}        -fsanitize=address")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=address")
-set (CMAKE_CXX_FLAGS_UBSAN               "${CMAKE_CXX_FLAGS_UBSAN} ${SAN_FLAGS} -fsanitize=undefined")
+        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=address")
-set (CMAKE_C_FLAGS_UBSAN                 "${CMAKE_C_FLAGS_UBSAN}   ${SAN_FLAGS} -fsanitize=undefined")
+        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
-set (CMAKE_EXE_LINKER_FLAGS_UBSAN        "${CMAKE_EXE_LINKER_FLAGS_UBSAN}       -fsanitize=undefined")
+        if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-set (CMAKE_CXX_FLAGS_MSAN                "${CMAKE_CXX_FLAGS_MSAN}  ${SAN_FLAGS} -fsanitize=memory")
+            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libasan")
-set (CMAKE_C_FLAGS_MSAN                  "${CMAKE_C_FLAGS_MSAN}    ${SAN_FLAGS} -fsanitize=memory")
+        endif ()
-set (CMAKE_EXE_LINKER_FLAGS_MSAN         "${CMAKE_EXE_LINKER_FLAGS_MSAN}        -fsanitize=memory")
+    elseif (SANITIZE STREQUAL "memory")
-set (CMAKE_CXX_FLAGS_TSAN                "${CMAKE_CXX_FLAGS_TSAN}  ${SAN_FLAGS} -fsanitize=thread")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=memory")
-set (CMAKE_C_FLAGS_TSAN                  "${CMAKE_C_FLAGS_TSAN}    ${SAN_FLAGS} -fsanitize=thread")
+        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=memory")
-set (CMAKE_EXE_LINKER_FLAGS_TSAN         "${CMAKE_EXE_LINKER_FLAGS_TSAN}        -fsanitize=thread")
+        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory")
-
+        if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-# clang use static linking by default
+            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libmsan")
-if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+        endif ()
-    set (CMAKE_EXE_LINKER_FLAGS_ASAN     "${CMAKE_EXE_LINKER_FLAGS_ASAN} -static-libasan")
+    elseif (SANITIZE STREQUAL "thread")
-    set (CMAKE_EXE_LINKER_FLAGS_UBSAN    "${CMAKE_EXE_LINKER_FLAGS_UBSAN} -static-libubsan")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=thread")
-    set (CMAKE_EXE_LINKER_FLAGS_MSAN     "${CMAKE_EXE_LINKER_FLAGS_MSAN} -static-libmsan")
+        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=thread")
-    set (CMAKE_EXE_LINKER_FLAGS_TSAN     "${CMAKE_EXE_LINKER_FLAGS_TSAN} -static-libtsan")
+        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread")
-endif ()
+        if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libtsan")
+        endif ()
+    elseif (SANITIZE STREQUAL "undefined")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined")
+        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined")
+        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
+        if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libubsan")
+        endif ()
+    else ()
+        message (FATAL_ERROR "Unknown sanitizer type: ${SANITIZE}")
+    endif ()
+endif()

contrib/CMakeLists.txt

@@ -1,8 +1,8 @@
 # Third-party libraries may have substandard code.
 
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format")
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation")
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -std=c++1z")
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-implicit-fallthrough -std=c++1z")
 elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format")
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-format -std=c++1z")
@@ -98,8 +98,8 @@ if (USE_INTERNAL_SSL_LIBRARY)
     set (USE_SHARED ${USE_STATIC_LIBRARIES})
     set (LIBRESSL_SKIP_INSTALL 1)
     add_subdirectory (ssl)
-    target_include_directories(${OPENSSL_CRYPTO_LIBRARY} PUBLIC ${OPENSSL_INCLUDE_DIR})
+    target_include_directories(${OPENSSL_CRYPTO_LIBRARY} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
-    target_include_directories(${OPENSSL_SSL_LIBRARY} PUBLIC ${OPENSSL_INCLUDE_DIR})
+    target_include_directories(${OPENSSL_SSL_LIBRARY} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
 endif ()
 
 if (ENABLE_MYSQL AND USE_INTERNAL_MYSQL_LIBRARY)
@@ -109,26 +109,9 @@ if (ENABLE_MYSQL AND USE_INTERNAL_MYSQL_LIBRARY)
 endif ()
 
 if (USE_INTERNAL_RDKAFKA_LIBRARY)
-    set (RDKAFKA_BUILD_EXAMPLES OFF CACHE INTERNAL "")
+    add_subdirectory (librdkafka-cmake)
-    set (RDKAFKA_BUILD_TESTS OFF CACHE INTERNAL "")
-    set (RDKAFKA_BUILD_STATIC ${MAKE_STATIC_LIBRARIES} CACHE INTERNAL "")
-    mark_as_advanced (ZLIB_INCLUDE_DIR)
-
-    if (USE_INTERNAL_SSL_LIBRARY)
-        if (MAKE_STATIC_LIBRARIES)
-            add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY})
-            set (WITH_BUNDLED_SSL 1 CACHE INTERNAL "")
-        else ()
-            set (WITH_SSL 0 CACHE INTERNAL "")
-        endif ()
-    endif ()
-
-    add_subdirectory (librdkafka)
-
-    if (USE_INTERNAL_SSL_LIBRARY AND MAKE_STATIC_LIBRARIES)
-        target_include_directories(rdkafka PRIVATE BEFORE ${OPENSSL_INCLUDE_DIR})
-    endif ()
     target_include_directories(rdkafka PRIVATE BEFORE ${ZLIB_INCLUDE_DIR})
+    target_include_directories(rdkafka PRIVATE BEFORE ${OPENSSL_INCLUDE_DIR})
 endif ()
 
 if (ENABLE_ODBC AND USE_INTERNAL_ODBC_LIBRARY)
@@ -162,7 +145,7 @@ if (USE_INTERNAL_POCO_LIBRARY)
 
     if (OPENSSL_FOUND AND TARGET Crypto AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL))
         # Bug in poco https://github.com/pocoproject/poco/pull/2100 found on macos
-        target_include_directories(Crypto PUBLIC ${OPENSSL_INCLUDE_DIR})
+        target_include_directories(Crypto SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
     endif ()
 endif ()
 

contrib/boost-cmake/CMakeLists.txt

@@ -42,9 +42,9 @@ ${LIBRARY_DIR}/libs/filesystem/src/windows_file_codecvt.cpp)
 add_library(boost_system_internal
 ${LIBRARY_DIR}/libs/system/src/error_code.cpp)
 
-target_include_directories (boost_program_options_internal BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
+target_include_directories (boost_program_options_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
-target_include_directories (boost_filesystem_internal BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
+target_include_directories (boost_filesystem_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
-target_include_directories (boost_system_internal BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
+target_include_directories (boost_system_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
 
 target_compile_definitions (boost_program_options_internal PUBLIC BOOST_SYSTEM_NO_DEPRECATED)
 target_compile_definitions (boost_filesystem_internal PUBLIC BOOST_SYSTEM_NO_DEPRECATED)

contrib/libcpuid/CMakeLists.txt

@@ -17,4 +17,4 @@ include/libcpuid/recog_amd.h
 include/libcpuid/recog_intel.h
 )
 
-target_include_directories (cpuid PUBLIC include)
+target_include_directories (cpuid SYSTEM PUBLIC include)

contrib/librdkafka-cmake/CMakeLists.txt

@@ -0,0 +1,60 @@
+set(RDKAFKA_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/librdkafka/src)
+
+set(SRCS
+${RDKAFKA_SOURCE_DIR}/crc32c.c
+${RDKAFKA_SOURCE_DIR}/rdaddr.c
+${RDKAFKA_SOURCE_DIR}/rdavl.c
+${RDKAFKA_SOURCE_DIR}/rdbuf.c
+${RDKAFKA_SOURCE_DIR}/rdcrc32.c
+${RDKAFKA_SOURCE_DIR}/rdkafka.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_assignor.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_broker.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_buf.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_cgrp.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_event.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_feature.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_lz4.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_metadata.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_metadata_cache.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_msg.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_msgset_reader.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_msgset_writer.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_offset.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_op.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_partition.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_pattern.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_queue.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_range_assignor.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_request.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_roundrobin_assignor.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_sasl.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_plain.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_subscription.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_timer.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_topic.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_transport.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_interceptor.c
+${RDKAFKA_SOURCE_DIR}/rdkafka_header.c
+${RDKAFKA_SOURCE_DIR}/rdlist.c
+${RDKAFKA_SOURCE_DIR}/rdlog.c
+${RDKAFKA_SOURCE_DIR}/rdmurmur2.c
+${RDKAFKA_SOURCE_DIR}/rdports.c
+${RDKAFKA_SOURCE_DIR}/rdrand.c
+${RDKAFKA_SOURCE_DIR}/rdregex.c
+${RDKAFKA_SOURCE_DIR}/rdstring.c
+${RDKAFKA_SOURCE_DIR}/rdunittest.c
+${RDKAFKA_SOURCE_DIR}/rdvarint.c
+${RDKAFKA_SOURCE_DIR}/snappy.c
+${RDKAFKA_SOURCE_DIR}/tinycthread.c
+${RDKAFKA_SOURCE_DIR}/xxhash.c
+${RDKAFKA_SOURCE_DIR}/lz4.c
+${RDKAFKA_SOURCE_DIR}/lz4frame.c
+${RDKAFKA_SOURCE_DIR}/lz4hc.c
+${RDKAFKA_SOURCE_DIR}/rdgz.c
+)
+
+add_library(rdkafka STATIC ${SRCS})
+target_include_directories(rdkafka PRIVATE include)
+target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR})
+target_link_libraries(rdkafka PUBLIC ${ZLIB_LIBRARIES} ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})

contrib/librdkafka-cmake/config.h

@@ -0,0 +1,74 @@
+// Automatically generated by ./configure 
+#ifndef _CONFIG_H_
+#define _CONFIG_H_
+#define ARCH "x86_64"
+#define CPU "generic"
+#define WITHOUT_OPTIMIZATION 0
+#define ENABLE_DEVEL 0
+#define ENABLE_VALGRIND 0
+#define ENABLE_REFCNT_DEBUG 0
+#define ENABLE_SHAREDPTR_DEBUG 0
+#define ENABLE_LZ4_EXT 1
+#define ENABLE_SSL 1
+//#define ENABLE_SASL 1
+#define MKL_APP_NAME "librdkafka"
+#define MKL_APP_DESC_ONELINE "The Apache Kafka C/C++ library"
+// distro
+//#define SOLIB_EXT ".so"
+// gcc
+//#define WITH_GCC 1
+// gxx
+//#define WITH_GXX 1
+// pkgconfig
+//#define WITH_PKGCONFIG 1
+// install
+//#define WITH_INSTALL 1
+// PIC
+//#define HAVE_PIC 1
+// gnulib
+//#define WITH_GNULD 1
+// __atomic_32
+#define HAVE_ATOMICS_32 1
+// __atomic_32
+#define HAVE_ATOMICS_32_ATOMIC 1
+// atomic_32
+#define ATOMIC_OP32(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)
+// __atomic_64
+#define HAVE_ATOMICS_64 1
+// __atomic_64
+#define HAVE_ATOMICS_64_ATOMIC 1
+// atomic_64
+#define ATOMIC_OP64(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)
+// atomic_64
+#define ATOMIC_OP(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)
+// parseversion
+#define RDKAFKA_VERSION_STR "0.11.4"
+// parseversion
+#define MKL_APP_VERSION "0.11.4"
+// libdl
+//#define WITH_LIBDL 1
+// WITH_PLUGINS
+//#define WITH_PLUGINS 1
+// zlib
+#define WITH_ZLIB 1
+// WITH_SNAPPY
+#define WITH_SNAPPY 1
+// WITH_SOCKEM
+#define WITH_SOCKEM 1
+// libssl
+#define WITH_SSL 1
+// WITH_SASL_SCRAM
+//#define WITH_SASL_SCRAM 1
+// crc32chw
+#define WITH_CRC32C_HW 1
+// regex
+#define HAVE_REGEX 1
+// strndup
+#define HAVE_STRNDUP 1
+// strerror_r
+#define HAVE_STRERROR_R 1
+// pthread_setname_gnu
+#define HAVE_PTHREAD_SETNAME_GNU 1
+// python
+//#define HAVE_PYTHON 1
+#endif /* _CONFIG_H_ */

contrib/librdkafka-cmake/include/README

@@ -0,0 +1 @@
+This directory is needed because rdkafka files have #include "../config.h"

contrib/mariadb-connector-c-cmake/linux_x86_64/include/mariadb_version.h

@@ -15,7 +15,7 @@
 #define MARIADB_VERSION_ID		100306
 #define MYSQL_VERSION_ID		100306
 #define MARIADB_PORT	        	3306
-#define MARIADB_UNIX_ADDR               "/tmp/mysql.sock"
+#define MARIADB_UNIX_ADDR               "/var/run/mysqld/mysqld.sock"
 #define MYSQL_CONFIG_NAME		"my"
 
 #define MARIADB_PACKAGE_VERSION "3.0.6"

contrib/murmurhash/CMakeLists.txt

@@ -1,5 +1,7 @@
 add_library(murmurhash
     src/murmurhash2.cpp
-    include/murmurhash2.h)
+    src/murmurhash3.cpp
+    include/murmurhash2.h
+    include/murmurhash3.h)
 
 target_include_directories (murmurhash PUBLIC include)

contrib/murmurhash/include/murmurhash3.h

@@ -0,0 +1,37 @@
+//-----------------------------------------------------------------------------
+// MurmurHash3 was written by Austin Appleby, and is placed in the public
+// domain. The author hereby disclaims copyright to this source code.
+
+#ifndef _MURMURHASH3_H_
+#define _MURMURHASH3_H_
+
+//-----------------------------------------------------------------------------
+// Platform-specific functions and macros
+
+// Microsoft Visual Studio
+
+#if defined(_MSC_VER) && (_MSC_VER < 1600)
+
+typedef unsigned char uint8_t;
+typedef unsigned int uint32_t;
+typedef unsigned __int64 uint64_t;
+
+// Other compilers
+
+#else	// defined(_MSC_VER)
+
+#include <stdint.h>
+
+#endif // !defined(_MSC_VER)
+
+//-----------------------------------------------------------------------------
+
+void MurmurHash3_x86_32  ( const void * key, int len, uint32_t seed, void * out );
+
+void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out );
+
+void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out );
+
+//-----------------------------------------------------------------------------
+
+#endif // _MURMURHASH3_H_

contrib/murmurhash/src/murmurhash3.cpp

@@ -0,0 +1,331 @@
+// MurmurHash3 was written by Austin Appleby, and is placed in the public
+// domain. The author hereby disclaims copyright to this source code.
+
+// Note - The x86 and x64 versions do _not_ produce the same results, as the
+// algorithms are optimized for their respective platforms. You can still
+// compile and run any of them on any platform, but your performance with the
+// non-native version will be less than optimal.
+
+#include "murmurhash3.h"
+
+//-----------------------------------------------------------------------------
+// Platform-specific functions and macros
+
+// Microsoft Visual Studio
+
+#if defined(_MSC_VER)
+
+#define FORCE_INLINE	__forceinline
+
+#include <stdlib.h>
+
+#define ROTL32(x,y)	_rotl(x,y)
+#define ROTL64(x,y)	_rotl64(x,y)
+
+#define BIG_CONSTANT(x) (x)
+
+// Other compilers
+
+#else	// defined(_MSC_VER)
+
+#define	FORCE_INLINE inline __attribute__((always_inline))
+
+inline uint32_t rotl32 ( uint32_t x, int8_t r )
+{
+  return (x << r) | (x >> (32 - r));
+}
+
+inline uint64_t rotl64 ( uint64_t x, int8_t r )
+{
+  return (x << r) | (x >> (64 - r));
+}
+
+#define	ROTL32(x,y)	rotl32(x,y)
+#define ROTL64(x,y)	rotl64(x,y)
+
+#define BIG_CONSTANT(x) (x##LLU)
+
+#endif // !defined(_MSC_VER)
+
+//-----------------------------------------------------------------------------
+// Block read - if your platform needs to do endian-swapping or can only
+// handle aligned reads, do the conversion here
+
+FORCE_INLINE uint32_t getblock32 ( const uint32_t * p, int i )
+{
+  return p[i];
+}
+
+FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, int i )
+{
+  return p[i];
+}
+
+//-----------------------------------------------------------------------------
+// Finalization mix - force all bits of a hash block to avalanche
+
+FORCE_INLINE uint32_t fmix32 ( uint32_t h )
+{
+  h ^= h >> 16;
+  h *= 0x85ebca6b;
+  h ^= h >> 13;
+  h *= 0xc2b2ae35;
+  h ^= h >> 16;
+
+  return h;
+}
+
+//----------
+
+FORCE_INLINE uint64_t fmix64 ( uint64_t k )
+{
+  k ^= k >> 33;
+  k *= BIG_CONSTANT(0xff51afd7ed558ccd);
+  k ^= k >> 33;
+  k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53);
+  k ^= k >> 33;
+
+  return k;
+}
+
+//-----------------------------------------------------------------------------
+
+void MurmurHash3_x86_32 ( const void * key, int len,
+                          uint32_t seed, void * out )
+{
+  const uint8_t * data = (const uint8_t*)key;
+  const int nblocks = len / 4;
+
+  uint32_t h1 = seed;
+
+  const uint32_t c1 = 0xcc9e2d51;
+  const uint32_t c2 = 0x1b873593;
+
+  //----------
+  // body
+
+  const uint32_t * blocks = (const uint32_t *)(data + nblocks*4);
+
+  for(int i = -nblocks; i; i++)
+  {
+    uint32_t k1 = getblock32(blocks,i);
+
+    k1 *= c1;
+    k1 = ROTL32(k1,15);
+    k1 *= c2;
+    
+    h1 ^= k1;
+    h1 = ROTL32(h1,13); 
+    h1 = h1*5+0xe6546b64;
+  }
+
+  //----------
+  // tail
+
+  const uint8_t * tail = (const uint8_t*)(data + nblocks*4);
+
+  uint32_t k1 = 0;
+
+  switch(len & 3)
+  {
+  case 3: k1 ^= tail[2] << 16;
+  case 2: k1 ^= tail[1] << 8;
+  case 1: k1 ^= tail[0];
+          k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1;
+  };
+
+  //----------
+  // finalization
+
+  h1 ^= len;
+
+  h1 = fmix32(h1);
+
+  *(uint32_t*)out = h1;
+} 
+
+//-----------------------------------------------------------------------------
+
+void MurmurHash3_x86_128 ( const void * key, const int len,
+                           uint32_t seed, void * out )
+{
+  const uint8_t * data = (const uint8_t*)key;
+  const int nblocks = len / 16;
+
+  uint32_t h1 = seed;
+  uint32_t h2 = seed;
+  uint32_t h3 = seed;
+  uint32_t h4 = seed;
+
+  const uint32_t c1 = 0x239b961b; 
+  const uint32_t c2 = 0xab0e9789;
+  const uint32_t c3 = 0x38b34ae5; 
+  const uint32_t c4 = 0xa1e38b93;
+
+  //----------
+  // body
+
+  const uint32_t * blocks = (const uint32_t *)(data + nblocks*16);
+
+  for(int i = -nblocks; i; i++)
+  {
+    uint32_t k1 = getblock32(blocks,i*4+0);
+    uint32_t k2 = getblock32(blocks,i*4+1);
+    uint32_t k3 = getblock32(blocks,i*4+2);
+    uint32_t k4 = getblock32(blocks,i*4+3);
+
+    k1 *= c1; k1  = ROTL32(k1,15); k1 *= c2; h1 ^= k1;
+
+    h1 = ROTL32(h1,19); h1 += h2; h1 = h1*5+0x561ccd1b;
+
+    k2 *= c2; k2  = ROTL32(k2,16); k2 *= c3; h2 ^= k2;
+
+    h2 = ROTL32(h2,17); h2 += h3; h2 = h2*5+0x0bcaa747;
+
+    k3 *= c3; k3  = ROTL32(k3,17); k3 *= c4; h3 ^= k3;
+
+    h3 = ROTL32(h3,15); h3 += h4; h3 = h3*5+0x96cd1c35;
+
+    k4 *= c4; k4  = ROTL32(k4,18); k4 *= c1; h4 ^= k4;
+
+    h4 = ROTL32(h4,13); h4 += h1; h4 = h4*5+0x32ac3b17;
+  }
+
+  //----------
+  // tail
+
+  const uint8_t * tail = (const uint8_t*)(data + nblocks*16);
+
+  uint32_t k1 = 0;
+  uint32_t k2 = 0;
+  uint32_t k3 = 0;
+  uint32_t k4 = 0;
+
+  switch(len & 15)
+  {
+  case 15: k4 ^= tail[14] << 16;
+  case 14: k4 ^= tail[13] << 8;
+  case 13: k4 ^= tail[12] << 0;
+           k4 *= c4; k4  = ROTL32(k4,18); k4 *= c1; h4 ^= k4;
+
+  case 12: k3 ^= tail[11] << 24;
+  case 11: k3 ^= tail[10] << 16;
+  case 10: k3 ^= tail[ 9] << 8;
+  case  9: k3 ^= tail[ 8] << 0;
+           k3 *= c3; k3  = ROTL32(k3,17); k3 *= c4; h3 ^= k3;
+
+  case  8: k2 ^= tail[ 7] << 24;
+  case  7: k2 ^= tail[ 6] << 16;
+  case  6: k2 ^= tail[ 5] << 8;
+  case  5: k2 ^= tail[ 4] << 0;
+           k2 *= c2; k2  = ROTL32(k2,16); k2 *= c3; h2 ^= k2;
+
+  case  4: k1 ^= tail[ 3] << 24;
+  case  3: k1 ^= tail[ 2] << 16;
+  case  2: k1 ^= tail[ 1] << 8;
+  case  1: k1 ^= tail[ 0] << 0;
+           k1 *= c1; k1  = ROTL32(k1,15); k1 *= c2; h1 ^= k1;
+  };
+
+  //----------
+  // finalization
+
+  h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len;
+
+  h1 += h2; h1 += h3; h1 += h4;
+  h2 += h1; h3 += h1; h4 += h1;
+
+  h1 = fmix32(h1);
+  h2 = fmix32(h2);
+  h3 = fmix32(h3);
+  h4 = fmix32(h4);
+
+  h1 += h2; h1 += h3; h1 += h4;
+  h2 += h1; h3 += h1; h4 += h1;
+
+  ((uint32_t*)out)[0] = h1;
+  ((uint32_t*)out)[1] = h2;
+  ((uint32_t*)out)[2] = h3;
+  ((uint32_t*)out)[3] = h4;
+}
+
+//-----------------------------------------------------------------------------
+
+void MurmurHash3_x64_128 ( const void * key, const int len,
+                           const uint32_t seed, void * out )
+{
+  const uint8_t * data = (const uint8_t*)key;
+  const int nblocks = len / 16;
+
+  uint64_t h1 = seed;
+  uint64_t h2 = seed;
+
+  const uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5);
+  const uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f);
+
+  //----------
+  // body
+
+  const uint64_t * blocks = (const uint64_t *)(data);
+
+  for(int i = 0; i < nblocks; i++)
+  {
+    uint64_t k1 = getblock64(blocks,i*2+0);
+    uint64_t k2 = getblock64(blocks,i*2+1);
+
+    k1 *= c1; k1  = ROTL64(k1,31); k1 *= c2; h1 ^= k1;
+
+    h1 = ROTL64(h1,27); h1 += h2; h1 = h1*5+0x52dce729;
+
+    k2 *= c2; k2  = ROTL64(k2,33); k2 *= c1; h2 ^= k2;
+
+    h2 = ROTL64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5;
+  }
+
+  //----------
+  // tail
+
+  const uint8_t * tail = (const uint8_t*)(data + nblocks*16);
+
+  uint64_t k1 = 0;
+  uint64_t k2 = 0;
+
+  switch(len & 15)
+  {
+  case 15: k2 ^= ((uint64_t)tail[14]) << 48;
+  case 14: k2 ^= ((uint64_t)tail[13]) << 40;
+  case 13: k2 ^= ((uint64_t)tail[12]) << 32;
+  case 12: k2 ^= ((uint64_t)tail[11]) << 24;
+  case 11: k2 ^= ((uint64_t)tail[10]) << 16;
+  case 10: k2 ^= ((uint64_t)tail[ 9]) << 8;
+  case  9: k2 ^= ((uint64_t)tail[ 8]) << 0;
+           k2 *= c2; k2  = ROTL64(k2,33); k2 *= c1; h2 ^= k2;
+
+  case  8: k1 ^= ((uint64_t)tail[ 7]) << 56;
+  case  7: k1 ^= ((uint64_t)tail[ 6]) << 48;
+  case  6: k1 ^= ((uint64_t)tail[ 5]) << 40;
+  case  5: k1 ^= ((uint64_t)tail[ 4]) << 32;
+  case  4: k1 ^= ((uint64_t)tail[ 3]) << 24;
+  case  3: k1 ^= ((uint64_t)tail[ 2]) << 16;
+  case  2: k1 ^= ((uint64_t)tail[ 1]) << 8;
+  case  1: k1 ^= ((uint64_t)tail[ 0]) << 0;
+           k1 *= c1; k1  = ROTL64(k1,31); k1 *= c2; h1 ^= k1;
+  };
+
+  //----------
+  // finalization
+
+  h1 ^= len; h2 ^= len;
+
+  h1 += h2;
+  h2 += h1;
+
+  h1 = fmix64(h1);
+  h2 = fmix64(h2);
+
+  h1 += h2;
+  h2 += h1;
+
+  ((uint64_t*)out)[0] = h1;
+  ((uint64_t*)out)[1] = h2;
+}

contrib/poco

@@ -1 +1 @@
-Subproject commit 4ab45bc3bb0d2c476ea5385ec2d398c6bfc9f089
+Subproject commit 3df947389e6d9654919002797bdd86ed190b3963

contrib/re2_st/CMakeLists.txt

@@ -12,7 +12,8 @@ endforeach ()
 
 add_library (re2_st ${RE2_ST_SOURCES})
 target_compile_definitions (re2_st PRIVATE NDEBUG NO_THREADS re2=re2_st)
-target_include_directories (re2_st PRIVATE . PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${RE2_SOURCE_DIR})
+target_include_directories (re2_st PRIVATE .)
+target_include_directories (re2_st SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${RE2_SOURCE_DIR})
 
 file (MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/re2_st)
 foreach (FILENAME filtered_re2.h re2.h set.h stringpiece.h)

contrib/ssl

@@ -1 +1 @@
-Subproject commit 4f9a7b8745184410dc0b31ba548ce21ac64edd9c
+Subproject commit de02224a42c69e3d8c9112c82018816f821878d0

contrib/zlib-ng

@@ -1 +1 @@
-Subproject commit e07a52dbaa35d003f5659b221b29d220c091667b
+Subproject commit 1b0ed32725e2c26d754e6fbc2e5e7490f8e8e711

dbms/CMakeLists.txt

@@ -156,7 +156,6 @@ target_link_libraries (dbms
     ${MYSQLXX_LIBRARY}
     ${RE2_LIBRARY}
     ${RE2_ST_LIBRARY}
-    ${OPENSSL_CRYPTO_LIBRARY}
     ${BTRIE_LIBRARIES}
 )
 
@@ -219,6 +218,8 @@ if (USE_RDKAFKA)
     endif ()
 endif ()
 
+target_link_libraries(dbms ${OPENSSL_CRYPTO_LIBRARY})
+
 target_link_libraries (dbms
     Threads::Threads
 )

dbms/cmake/version.cmake

@@ -1,11 +1,11 @@
 # This strings autochanged from release_lib.sh:
-set(VERSION_REVISION 54404 CACHE STRING "")
+set(VERSION_REVISION 54405 CACHE STRING "")
 set(VERSION_MAJOR 18 CACHE STRING "")
-set(VERSION_MINOR 9 CACHE STRING "")
+set(VERSION_MINOR 10 CACHE STRING "")
-set(VERSION_PATCH 0 CACHE STRING "")
+set(VERSION_PATCH 2 CACHE STRING "")
-set(VERSION_GITHASH c83721a02db002eef7ff864f82d53ca89d47f9e6 CACHE STRING "")
+set(VERSION_GITHASH 39bee180bd7c15dbb35244cc78387628345c1efe CACHE STRING "")
-set(VERSION_DESCRIBE v18.9.0-testing CACHE STRING "")
+set(VERSION_DESCRIBE v18.10.2-testing CACHE STRING "")
-set(VERSION_STRING 18.9.0 CACHE STRING "")
+set(VERSION_STRING 18.10.2 CACHE STRING "")
 # end of autochange
 
 set(VERSION_EXTRA "" CACHE STRING "")

dbms/programs/clang/CMakeLists.txt

@@ -13,10 +13,10 @@ if (CLICKHOUSE_SPLIT_BINARY)
     endif ()
 endif ()
 
-set(TMP_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/headers")
+set(TMP_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/${INTERNAL_COMPILER_HEADERS_RELATIVE}")
 # Make and install empty dir for debian package if compiler disabled
 add_custom_target(make-headers-directory ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${TMP_HEADERS_DIR})
-install(DIRECTORY ${TMP_HEADERS_DIR} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse COMPONENT clickhouse)
+install(DIRECTORY ${TMP_HEADERS_DIR} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse/${INTERNAL_COMPILER_HEADERS_DIR} COMPONENT clickhouse)
 # TODO: fix on macos copy_headers.sh: sed --posix
 if (USE_EMBEDDED_COMPILER AND NOT APPLE)
     add_custom_target(copy-headers ALL env CLANG=${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-clang BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} ${ClickHouse_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${TMP_HEADERS_DIR} DEPENDS clickhouse-clang WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES ${ClickHouse_SOURCE_DIR}/copy_headers.sh)

dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt

@@ -43,4 +43,7 @@ LLVMSupport
 #PollyPPCG
 
 PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads
+${MALLOC_LIBRARIES}
+${GLIBC_COMPATIBILITY_LIBRARIES}
+${MEMCPY_LIBRARIES}
 )

dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt

@@ -43,4 +43,7 @@ ${REQUIRED_LLVM_LIBRARIES}
 #PollyPPCG
 
 PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads
+${MALLOC_LIBRARIES}
+${GLIBC_COMPATIBILITY_LIBRARIES}
+${MEMCPY_LIBRARIES}
 )

dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt

@@ -39,4 +39,7 @@ lldCore
 ${REQUIRED_LLVM_LIBRARIES}
 
 PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads
+${MALLOC_LIBRARIES}
+${GLIBC_COMPATIBILITY_LIBRARIES}
+${MEMCPY_LIBRARIES}
 )

dbms/programs/client/Client.cpp

@@ -204,6 +204,7 @@ private:
     bool is_interactive = true;          /// Use either readline interface or batch mode.
     bool need_render_progress = true;    /// Render query execution progress.
     bool echo_queries = false;           /// Print queries before execution in batch mode.
+    bool ignore_error = false;           /// In case of errors, don't print error message, continue to next query. Only applicable for non-interactive mode.
     bool print_time_to_stderr = false;   /// Output execution time to stderr in batch mode.
     bool stdin_is_not_tty = false;       /// stdin is not a terminal.
 
@@ -474,6 +475,7 @@ private:
         {
             need_render_progress = config().getBool("progress", false);
             echo_queries = config().getBool("echo", false);
+            ignore_error = config().getBool("ignore-error", false);
         }
 
         connect();
@@ -765,7 +767,6 @@ private:
 
     bool process(const String & text)
     {
-        const auto ignore_error = config().getBool("ignore-error", false);
         const bool test_mode = config().has("testmode");
         if (config().has("multiquery"))
         {
@@ -821,9 +822,10 @@ private:
                 }
                 catch (...)
                 {
-                    actual_client_error = getCurrentExceptionCode();
+                    last_exception = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
-                    if (!actual_client_error || actual_client_error != expected_client_error)
+                    actual_client_error = last_exception->code();
-                        std::cerr << "Error on processing query: " << query << std::endl << getCurrentExceptionMessage(true);
+                    if (!ignore_error && (!actual_client_error || actual_client_error != expected_client_error))
+                        std::cerr << "Error on processing query: " << query << std::endl << last_exception->message();
                     got_exception = true;
                 }
 
@@ -1544,6 +1546,7 @@ public:
             ("stacktrace", "print stack traces of exceptions")
             ("progress", "print progress even in non-interactive mode")
             ("version,V", "print version information and exit")
+            ("version-clean", "print version in machine-readable format and exit")
             ("echo", "in batch mode, print query before execution")
             ("max_client_network_bandwidth", boost::program_options::value<int>(), "the maximum speed of data exchange over the network for the client in bytes per second.")
             ("compression", boost::program_options::value<bool>(), "enable or disable compression")
@@ -1573,6 +1576,12 @@ public:
             exit(0);
         }
 
+        if (options.count("version-clean"))
+        {
+            std::cout << VERSION_STRING;
+            exit(0);
+        }
+
         /// Output of help message.
         if (options.count("help")
             || (options.count("host") && options["host"].as<std::string>() == "elp"))    /// If user writes -help instead of --help.

dbms/programs/server/Server.cpp

@@ -301,6 +301,9 @@ int Server::main(const std::vector<std::string> & /*args*/)
     if (config().has("max_table_size_to_drop"))
         global_context->setMaxTableSizeToDrop(config().getUInt64("max_table_size_to_drop"));
 
+    if (config().has("max_partition_size_to_drop"))
+        global_context->setMaxPartitionSizeToDrop(config().getUInt64("max_partition_size_to_drop"));
+
     /// Size of cache for uncompressed blocks. Zero means disabled.
     size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size", 0);
     if (uncompressed_cache_size)

dbms/programs/server/config.xml

@@ -322,10 +322,12 @@
     <!-- Protection from accidental DROP.
          If size of a MergeTree table is greater than max_table_size_to_drop (in bytes) than table could not be dropped with any DROP query.
          If you want do delete one table and don't want to restart clickhouse-server, you could create special file <clickhouse-path>/flags/force_drop_table and make DROP once.
-         By default max_table_size_to_drop is 50GB, max_table_size_to_drop=0 allows to DROP any tables.
+         By default max_table_size_to_drop is 50GB; max_table_size_to_drop=0 allows to DROP any tables.
+         The same for max_partition_size_to_drop.
          Uncomment to disable protection.
     -->
     <!-- <max_table_size_to_drop>0</max_table_size_to_drop> -->
+    <!-- <max_partition_size_to_drop>0</max_partition_size_to_drop> -->
 
     <!-- Example of parameters for GraphiteMergeTree table engine -->
     <graphite_rollup_example>

dbms/src/Client/ConnectionPoolWithFailover.cpp

@@ -226,6 +226,6 @@ ConnectionPoolWithFailover::tryGetEntry(
         }
     }
     return result;
-};
+}
 
 }

dbms/src/Columns/Collator.cpp

@@ -3,10 +3,7 @@
 #include <Common/config.h>
 
 #if USE_ICU
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wold-style-cast"
     #include <unicode/ucol.h>
-    #pragma GCC diagnostic pop
 #else
     #ifdef __clang__
         #pragma clang diagnostic push

dbms/src/Common/COWPtr.h

@@ -1,17 +1,7 @@
 #pragma once
 
-#ifdef __clang__
-    #pragma clang diagnostic push
-    #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
-#endif
-
 #include <boost/smart_ptr/intrusive_ptr.hpp>
 #include <boost/smart_ptr/intrusive_ref_counter.hpp>
-
-#ifdef __clang__
-    #pragma clang diagnostic pop
-#endif
-
 #include <initializer_list>
 
 

dbms/src/Common/ConcurrentBoundedQueue.h

@@ -37,7 +37,7 @@ namespace detail
     {
         MoveOrCopyIfThrow<T>()(std::forward<T>(src), dst);
     }
-};
+}
 
 /** A very simple thread-safe queue of limited size.
   * If you try to pop an item from an empty queue, the thread is blocked until the queue becomes nonempty.

dbms/src/Common/Config/ConfigProcessor.cpp

@@ -406,9 +406,6 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string &
     /// Add path_to_config/conf.d dir
     merge_dir_path.setBaseName("conf");
     merge_dirs.insert(merge_dir_path.toString());
-    /// Add path_to_config/config.d dir
-    merge_dir_path.setBaseName("config");
-    merge_dirs.insert(merge_dir_path.toString());
 
     for (const std::string & merge_dir_name : merge_dirs)
     {

dbms/src/Common/ErrorCodes.cpp

@@ -380,6 +380,7 @@ namespace ErrorCodes
     extern const int INVALID_JOIN_ON_EXPRESSION = 403;
     extern const int BAD_ODBC_CONNECTION_STRING = 404;
     extern const int DECIMAL_OVERFLOW = 405;
+    extern const int PARTITION_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT = 406;
 
     extern const int KEEPER_EXCEPTION = 999;
     extern const int POCO_EXCEPTION = 1000;

dbms/src/Common/Exception.cpp

@@ -130,11 +130,11 @@ int getCurrentExceptionCode()
     {
         return e.code();
     }
-    catch (const Poco::Exception & e)
+    catch (const Poco::Exception &)
     {
         return ErrorCodes::POCO_EXCEPTION;
     }
-    catch (const std::exception & e)
+    catch (const std::exception &)
     {
         return ErrorCodes::STD_EXCEPTION;
     }

dbms/src/Common/FileChecker.cpp

@@ -95,13 +95,14 @@ void FileChecker::save() const
         /// So complex JSON structure - for compatibility with the old format.
         writeCString("{\"yandex\":{", out);
 
+        auto settings = FormatSettings();
         for (auto it = map.begin(); it != map.end(); ++it)
         {
             if (it != map.begin())
                 writeString(",", out);
 
             /// `escapeForFileName` is not really needed. But it is left for compatibility with the old code.
-            writeJSONString(escapeForFileName(it->first), out);
+            writeJSONString(escapeForFileName(it->first), out, settings);
             writeString(":{\"size\":\"", out);
             writeIntText(it->second, out);
             writeString("\"}", out);

dbms/src/Common/HashTable/HashTable.h

@@ -74,7 +74,7 @@ bool check(const T x) { return x == 0; }
 template <typename T>
 void set(T & x) { x = 0; }
 
-};
+}
 
 
 /** Compile-time interface for cell of the hash table.

dbms/src/Common/IFactoryWithAliases.h

@@ -94,8 +94,8 @@ public:
     {
         if (auto it = aliases.find(name); it != aliases.end())
             return it->second;
-        else if (auto it = case_insensitive_aliases.find(Poco::toLower(name)); it != case_insensitive_aliases.end())
+        else if (auto jt = case_insensitive_aliases.find(Poco::toLower(name)); jt != case_insensitive_aliases.end())
-            return it->second;
+            return jt->second;
 
         throw Exception(getFactoryName() + ": name '" + name + "' is not alias", ErrorCodes::LOGICAL_ERROR);
     }

dbms/src/Common/PODArray.h

@@ -5,24 +5,9 @@
 #include <algorithm>
 #include <memory>
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wold-style-cast"
-
-#ifdef __clang__
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
-#pragma clang diagnostic ignored "-Wreserved-id-macro"
-#endif
-
 #include <boost/noncopyable.hpp>
 #include <boost/iterator_adaptors.hpp>
 
-#ifdef __clang__
-#pragma clang diagnostic pop
-#endif
-
-#pragma GCC diagnostic pop
-
 #include <common/likely.h>
 #include <common/strong_typedef.h>
 

dbms/src/Common/SpaceSaving.h

@@ -340,4 +340,4 @@ private:
     size_t m_capacity;
 };
 
-};
+}

dbms/src/Common/ZooKeeper/KeeperException.h

@@ -50,4 +50,4 @@ private:
     static size_t getFailedOpIndex(int32_t code, const Responses & responses);
 };
 
-};
+}

dbms/src/Common/ZooKeeper/ZooKeeperHolder.h

@@ -83,4 +83,4 @@ void ZooKeeperHolder::init(Args&&... args)
 
 using ZooKeeperHolderPtr = std::shared_ptr<ZooKeeperHolder>;
 
-};
+}

dbms/src/Common/ZooKeeper/ZooKeeperImpl.h

@@ -646,4 +646,4 @@ private:
     CurrentMetrics::Increment active_session_metric_increment{CurrentMetrics::ZooKeeperSession};
 };
 
-};
+}

dbms/src/Common/formatIPv6.cpp

@@ -77,7 +77,10 @@ void formatIPv6(const unsigned char * src, char *& dst, UInt8 zeroed_tail_bytes_
         if (words[i] == 0)
         {
             if (cur.base == -1)
-                cur.base = i, cur.len = 1;
+            {
+                cur.base = i;
+                cur.len = 1;
+            }
             else
                 cur.len++;
         }

dbms/src/Common/localBackup.cpp

@@ -108,7 +108,7 @@ void localBackup(const Poco::Path & source_path, const Poco::Path & destination_
 
             continue;
         }
-        catch (const Poco::FileNotFoundException & e)
+        catch (const Poco::FileNotFoundException &)
         {
             ++try_no;
             if (try_no == max_tries)

dbms/src/Common/tests/CMakeLists.txt

@@ -60,7 +60,7 @@ add_executable (space_saving space_saving.cpp)
 target_link_libraries (space_saving clickhouse_common_io)
 
 add_executable (integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp)
-target_include_directories (integer_hash_tables_and_hashes BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
+target_include_directories (integer_hash_tables_and_hashes SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
 target_link_libraries (integer_hash_tables_and_hashes clickhouse_common_io)
 
 add_executable (allocator allocator.cpp)

dbms/src/Core/Field.h

@@ -25,7 +25,7 @@ namespace ErrorCodes
 class Field;
 using Array = std::vector<Field>;
 using TupleBackend = std::vector<Field>;
-STRONG_TYPEDEF(TupleBackend, Tuple); /// Array and Tuple are different types with equal representation inside Field.
+STRONG_TYPEDEF(TupleBackend, Tuple) /// Array and Tuple are different types with equal representation inside Field.
 
 
 class DecField

dbms/src/Core/UUID.h

@@ -6,6 +6,6 @@
 namespace DB
 {
 
-STRONG_TYPEDEF(UInt128, UUID);
+STRONG_TYPEDEF(UInt128, UUID)
 
 }

dbms/src/DataStreams/AggregatingSortedBlockInputStream.cpp

@@ -63,7 +63,7 @@ Block AggregatingSortedBlockInputStream::readImpl()
     for (size_t i = 0, size = columns_to_aggregate.size(); i < size; ++i)
         columns_to_aggregate[i] = typeid_cast<ColumnAggregateFunction *>(merged_columns[column_numbers_to_aggregate[i]].get());
 
-    merge(merged_columns, queue);
+    merge(merged_columns, queue_without_collation);
     return header.cloneWithColumns(std::move(merged_columns));
 }
 

dbms/src/DataStreams/CollapsingSortedBlockInputStream.cpp

@@ -40,60 +40,45 @@ void CollapsingSortedBlockInputStream::reportIncorrectData()
 }
 
 
-void CollapsingSortedBlockInputStream::insertRows(MutableColumns & merged_columns, size_t & merged_rows, bool last_in_stream)
+void CollapsingSortedBlockInputStream::insertRows(MutableColumns & merged_columns, size_t & merged_rows)
 {
     if (count_positive == 0 && count_negative == 0)
+    {
+        /// No input rows have been read.
         return;
+    }
 
     if (count_positive == count_negative && !last_is_positive)
     {
-        /// If all the rows in the input streams was collapsed, we still want to give at least one block in the result.
+        /// Input rows exactly cancel out.
-        if (last_in_stream && merged_rows == 0 && !blocks_written)
+        return;
-        {
-            LOG_INFO(log, "All rows collapsed");
-            ++merged_rows;
-            for (size_t i = 0; i < num_columns; ++i)
-                merged_columns[i]->insertFrom(*(*last_positive.columns)[i], last_positive.row_num);
-            ++merged_rows;
-            for (size_t i = 0; i < num_columns; ++i)
-                merged_columns[i]->insertFrom(*(*last_negative.columns)[i], last_negative.row_num);
-
-            if (out_row_sources_buf)
-            {
-                /// true flag value means "skip row"
-                current_row_sources[last_positive_pos].setSkipFlag(false);
-                current_row_sources[last_negative_pos].setSkipFlag(false);
-            }
-        }
     }
-    else
+
+    if (count_positive <= count_negative)
     {
-        if (count_positive <= count_negative)
+        ++merged_rows;
-        {
+        for (size_t i = 0; i < num_columns; ++i)
-            ++merged_rows;
+            merged_columns[i]->insertFrom(*(*first_negative.columns)[i], first_negative.row_num);
-            for (size_t i = 0; i < num_columns; ++i)
-                merged_columns[i]->insertFrom(*(*first_negative.columns)[i], first_negative.row_num);
 
-            if (out_row_sources_buf)
+        if (out_row_sources_buf)
-                current_row_sources[first_negative_pos].setSkipFlag(false);
+            current_row_sources[first_negative_pos].setSkipFlag(false);
-        }
+    }
 
-        if (count_positive >= count_negative)
+    if (count_positive >= count_negative)
-        {
+    {
-            ++merged_rows;
+        ++merged_rows;
-            for (size_t i = 0; i < num_columns; ++i)
+        for (size_t i = 0; i < num_columns; ++i)
-                merged_columns[i]->insertFrom(*(*last_positive.columns)[i], last_positive.row_num);
+            merged_columns[i]->insertFrom(*(*last_positive.columns)[i], last_positive.row_num);
 
-            if (out_row_sources_buf)
+        if (out_row_sources_buf)
-                current_row_sources[last_positive_pos].setSkipFlag(false);
+            current_row_sources[last_positive_pos].setSkipFlag(false);
-        }
+    }
 
-        if (!(count_positive == count_negative || count_positive + 1 == count_negative || count_positive == count_negative + 1))
+    if (!(count_positive == count_negative || count_positive + 1 == count_negative || count_positive == count_negative + 1))
-        {
+    {
-            if (count_incorrect_data < MAX_ERROR_MESSAGES)
+        if (count_incorrect_data < MAX_ERROR_MESSAGES)
-                reportIncorrectData();
+            reportIncorrectData();
-            ++count_incorrect_data;
+        ++count_incorrect_data;
-        }
     }
 
     if (out_row_sources_buf)
@@ -117,7 +102,7 @@ Block CollapsingSortedBlockInputStream::readImpl()
     if (merged_columns.empty())
         return {};
 
-    merge(merged_columns, queue);
+    merge(merged_columns, queue_without_collation);
     return header.cloneWithColumns(std::move(merged_columns));
 }
 
@@ -211,7 +196,7 @@ void CollapsingSortedBlockInputStream::merge(MutableColumns & merged_columns, st
     }
 
     /// Write data for last primary key.
-    insertRows(merged_columns, merged_rows, true);
+    insertRows(merged_columns, merged_rows);
 
     finished = true;
 }

dbms/src/DataStreams/CollapsingSortedBlockInputStream.h

@@ -74,7 +74,7 @@ private:
     void merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue);
 
     /// Output to result rows for the current primary key.
-    void insertRows(MutableColumns & merged_columns, size_t & merged_rows, bool last_in_stream = false);
+    void insertRows(MutableColumns & merged_columns, size_t & merged_rows);
 
     void reportIncorrectData();
 };

dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.cpp

@@ -102,7 +102,7 @@ Block GraphiteRollupSortedBlockInputStream::readImpl()
     if (merged_columns.empty())
         return Block();
 
-    merge(merged_columns, queue);
+    merge(merged_columns, queue_without_collation);
     return header.cloneWithColumns(std::move(merged_columns));
 }
 

dbms/src/DataStreams/IProfilingBlockInputStream.cpp

@@ -188,7 +188,7 @@ static bool handleOverflowMode(OverflowMode mode, const String & message, int co
         default:
             throw Exception("Logical error: unknown overflow mode", ErrorCodes::LOGICAL_ERROR);
     }
-};
+}
 
 
 bool IProfilingBlockInputStream::checkTimeLimit()

dbms/src/DataStreams/MergeSortingBlockInputStream.cpp

@@ -183,7 +183,7 @@ MergeSortingBlocksBlockInputStream::MergeSortingBlocksBlockInputStream(
     if (!has_collation)
     {
         for (size_t i = 0; i < cursors.size(); ++i)
-            queue.push(SortCursor(&cursors[i]));
+            queue_without_collation.push(SortCursor(&cursors[i]));
     }
     else
     {
@@ -206,7 +206,7 @@ Block MergeSortingBlocksBlockInputStream::readImpl()
     }
 
     return !has_collation
-        ? mergeImpl<SortCursor>(queue)
+        ? mergeImpl<SortCursor>(queue_without_collation)
         : mergeImpl<SortCursorWithCollation>(queue_with_collation);
 }
 

dbms/src/DataStreams/MergeSortingBlockInputStream.h

@@ -55,7 +55,7 @@ private:
 
     bool has_collation = false;
 
-    std::priority_queue<SortCursor> queue;
+    std::priority_queue<SortCursor> queue_without_collation;
     std::priority_queue<SortCursorWithCollation> queue_with_collation;
 
     /** Two different cursors are supported - with and without Collation.

dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp

@@ -320,7 +320,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(MemoryTracker
               * - or, if no next blocks, set 'exhausted' flag.
               */
             {
-                std::lock_guard<std::mutex> lock(parallel_merge_data->get_next_blocks_mutex);
+                std::lock_guard<std::mutex> lock_next_blocks(parallel_merge_data->get_next_blocks_mutex);
 
                 if (parallel_merge_data->exhausted || parallel_merge_data->finish)
                     break;
@@ -330,7 +330,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(MemoryTracker
                 if (!blocks_to_merge || blocks_to_merge->empty())
                 {
                     {
-                        std::unique_lock<std::mutex> lock(parallel_merge_data->merged_blocks_mutex);
+                        std::unique_lock<std::mutex> lock_merged_blocks(parallel_merge_data->merged_blocks_mutex);
                         parallel_merge_data->exhausted = true;
                     }
 
@@ -344,9 +344,9 @@ void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(MemoryTracker
                     : blocks_to_merge->front().info.bucket_num;
 
                 {
-                    std::unique_lock<std::mutex> lock(parallel_merge_data->merged_blocks_mutex);
+                    std::unique_lock<std::mutex> lock_merged_blocks(parallel_merge_data->merged_blocks_mutex);
 
-                    parallel_merge_data->have_space.wait(lock, [this]
+                    parallel_merge_data->have_space.wait(lock_merged_blocks, [this]
                     {
                         return parallel_merge_data->merged_blocks.size() < merging_threads
                             || parallel_merge_data->finish;

dbms/src/DataStreams/MergingSortedBlockInputStream.cpp

@@ -58,7 +58,7 @@ void MergingSortedBlockInputStream::init(MutableColumns & merged_columns)
         if (has_collation)
             initQueue(queue_with_collation);
         else
-            initQueue(queue);
+            initQueue(queue_without_collation);
     }
 
     /// Let's check that all source blocks have the same structure.
@@ -105,7 +105,7 @@ Block MergingSortedBlockInputStream::readImpl()
     if (has_collation)
         merge(merged_columns, queue_with_collation);
     else
-        merge(merged_columns, queue);
+        merge(merged_columns, queue_without_collation);
 
     return header.cloneWithColumns(std::move(merged_columns));
 }
@@ -200,7 +200,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::
 
     //            std::cerr << "copied columns\n";
 
-                size_t merged_rows = merged_columns.at(0)->size();
+                merged_rows = merged_columns.at(0)->size();
 
                 if (limit && total_merged_rows + merged_rows > limit)
                 {

dbms/src/DataStreams/MergingSortedBlockInputStream.h

@@ -2,17 +2,8 @@
 
 #include <queue>
 
-#ifdef __clang__
-    #pragma clang diagnostic push
-    #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
-#endif
-
 #include <boost/smart_ptr/intrusive_ptr.hpp>
 
-#ifdef __clang__
-    #pragma clang diagnostic pop
-#endif
-
 #include <common/logger_useful.h>
 
 #include <Core/Row.h>
@@ -161,7 +152,7 @@ protected:
     CursorImpls cursors;
 
     using Queue = std::priority_queue<SortCursor>;
-    Queue queue;
+    Queue queue_without_collation;
 
     using QueueWithCollation = std::priority_queue<SortCursorWithCollation>;
     QueueWithCollation queue_with_collation;

dbms/src/DataStreams/RemoteBlockInputStream.cpp

@@ -58,21 +58,21 @@ RemoteBlockInputStream::RemoteBlockInputStream(
 
     create_multiplexed_connections = [this, pool, throttler]()
     {
-        const Settings & settings = context.getSettingsRef();
+        const Settings & current_settings = context.getSettingsRef();
 
         std::vector<IConnectionPool::Entry> connections;
         if (main_table)
         {
-            auto try_results = pool->getManyChecked(&settings, pool_mode, *main_table);
+            auto try_results = pool->getManyChecked(&current_settings, pool_mode, *main_table);
             connections.reserve(try_results.size());
             for (auto & try_result : try_results)
                 connections.emplace_back(std::move(try_result.entry));
         }
         else
-            connections = pool->getMany(&settings, pool_mode);
+            connections = pool->getMany(&current_settings, pool_mode);
 
         return std::make_unique<MultiplexedConnections>(
-                std::move(connections), settings, throttler, append_extra_info);
+                std::move(connections), current_settings, throttler, append_extra_info);
     };
 }
 

dbms/src/DataStreams/ReplacingSortedBlockInputStream.cpp

@@ -44,7 +44,7 @@ Block ReplacingSortedBlockInputStream::readImpl()
     if (merged_columns.empty())
         return Block();
 
-    merge(merged_columns, queue);
+    merge(merged_columns, queue_without_collation);
     return header.cloneWithColumns(std::move(merged_columns));
 }
 

dbms/src/DataStreams/SummingSortedBlockInputStream.cpp

@@ -195,7 +195,7 @@ SummingSortedBlockInputStream::SummingSortedBlockInputStream(
 }
 
 
-void SummingSortedBlockInputStream::insertCurrentRowIfNeeded(MutableColumns & merged_columns, bool force_insertion)
+void SummingSortedBlockInputStream::insertCurrentRowIfNeeded(MutableColumns & merged_columns)
 {
     for (auto & desc : columns_to_aggregate)
     {
@@ -237,9 +237,9 @@ void SummingSortedBlockInputStream::insertCurrentRowIfNeeded(MutableColumns & me
             desc.merged_column->insertDefault();
     }
 
-    /// If it is "zero" row and it is not the last row of the result block, then
+    /// If it is "zero" row, then rollback the insertion
-    ///  rollback the insertion (at this moment we need rollback only cols from columns_to_aggregate)
+    /// (at this moment we need rollback only cols from columns_to_aggregate)
-    if (!force_insertion && current_row_is_zero)
+    if (current_row_is_zero)
     {
         for (auto & desc : columns_to_aggregate)
             desc.merged_column->popBack(1);
@@ -252,7 +252,6 @@ void SummingSortedBlockInputStream::insertCurrentRowIfNeeded(MutableColumns & me
 
     /// Update per-block and per-group flags
     ++merged_rows;
-    output_is_non_empty = true;
 }
 
 
@@ -287,7 +286,7 @@ Block SummingSortedBlockInputStream::readImpl()
             desc.merged_column = header.safeGetByPosition(desc.column_numbers[0]).column->cloneEmpty();
     }
 
-    merge(merged_columns, queue);
+    merge(merged_columns, queue_without_collation);
     Block res = header.cloneWithColumns(std::move(merged_columns));
 
     /// Place aggregation results into block.
@@ -333,7 +332,7 @@ void SummingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::
         {
             if (!current_key.empty())
                 /// Write the data for the previous group.
-                insertCurrentRowIfNeeded(merged_columns, false);
+                insertCurrentRowIfNeeded(merged_columns);
 
             if (merged_rows >= max_block_size)
             {
@@ -393,7 +392,7 @@ void SummingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::
 
     /// We will write the data for the last group, if it is non-zero.
     /// If it is zero, and without it the output stream will be empty, we will write it anyway.
-    insertCurrentRowIfNeeded(merged_columns, !output_is_non_empty);
+    insertCurrentRowIfNeeded(merged_columns);
     finished = true;
 }
 

dbms/src/DataStreams/SummingSortedBlockInputStream.h

@@ -134,7 +134,6 @@ private:
     Row current_row;
     bool current_row_is_zero = true;    /// Are all summed columns zero (or empty)? It is updated incrementally.
 
-    bool output_is_non_empty = false;   /// Have we given out at least one row as a result.
     size_t merged_rows = 0;             /// Number of rows merged into current result block
 
     /** We support two different cursors - with Collation and without.
@@ -143,8 +142,7 @@ private:
     void merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue);
 
     /// Insert the summed row for the current group into the result and updates some of per-block flags if the row is not "zero".
-    /// If force_insertion=true, then the row will be inserted even if it is "zero"
+    void insertCurrentRowIfNeeded(MutableColumns & merged_columns);
-    void insertCurrentRowIfNeeded(MutableColumns & merged_columns, bool force_insertion);
 
     /// Returns true if merge result is not empty
     bool mergeMap(const MapDescription & map, Row & row, SortCursor & cursor);

dbms/src/DataStreams/VersionedCollapsingSortedBlockInputStream.cpp

@@ -15,11 +15,11 @@ namespace ErrorCodes
 
 VersionedCollapsingSortedBlockInputStream::VersionedCollapsingSortedBlockInputStream(
     const BlockInputStreams & inputs_, const SortDescription & description_,
-    const String & sign_column_, size_t max_block_size_, bool can_collapse_all_rows_,
+    const String & sign_column_, size_t max_block_size_,
     WriteBuffer * out_row_sources_buf_)
     : MergingSortedBlockInputStream(inputs_, description_, max_block_size_, 0, out_row_sources_buf_)
     , max_rows_in_queue(std::min(std::max<size_t>(3, max_block_size_), MAX_ROWS_IN_MULTIVERSION_QUEUE) - 2)
-    , current_keys(max_rows_in_queue + 1), can_collapse_all_rows(can_collapse_all_rows_)
+    , current_keys(max_rows_in_queue + 1)
 {
     sign_column_number = header.getPositionByName(sign_column_);
 }
@@ -76,7 +76,7 @@ Block VersionedCollapsingSortedBlockInputStream::readImpl()
     if (merged_columns.empty())
         return {};
 
-    merge(merged_columns, queue);
+    merge(merged_columns, queue_without_collation);
     return header.cloneWithColumns(std::move(merged_columns));
 }
 
@@ -130,10 +130,7 @@ void VersionedCollapsingSortedBlockInputStream::merge(MutableColumns & merged_co
             {
                 update_queue(current);
 
-                /// If all the rows was collapsed, we still want to give at least one block in the result.
+                if (sign == sign_in_queue)
-                /// If queue is empty then don't collapse two last rows.
-                if (sign == sign_in_queue || (!can_collapse_all_rows && blocks_written == 0
-                                              && merged_rows == 0 && queue.empty() && current_keys.size() == 1))
                     current_keys.pushBack(next_key);
                 else
                 {

dbms/src/DataStreams/VersionedCollapsingSortedBlockInputStream.h

@@ -176,7 +176,7 @@ public:
     /// max_rows_in_queue should be about max_block_size_ if we won't store a lot of extra blocks (RowRef holds SharedBlockPtr).
     VersionedCollapsingSortedBlockInputStream(
         const BlockInputStreams & inputs_, const SortDescription & description_,
-        const String & sign_column_, size_t max_block_size_, bool can_collapse_all_rows_,
+        const String & sign_column_, size_t max_block_size_,
         WriteBuffer * out_row_sources_buf_ = nullptr);
 
     String getName() const override { return "VersionedCollapsingSorted"; }
@@ -203,8 +203,6 @@ private:
     /// Sources of rows for VERTICAL merge algorithm. Size equals to (size + number of gaps) in current_keys.
     std::queue<RowSourcePart> current_row_sources;
 
-    const bool can_collapse_all_rows;
-
     void merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue);
 
     /// Output to result row for the current primary key.

dbms/src/DataTypes/DataTypeAggregateFunction.cpp

@@ -212,9 +212,9 @@ void DataTypeAggregateFunction::deserializeTextQuoted(IColumn & column, ReadBuff
 }
 
 
-void DataTypeAggregateFunction::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
+void DataTypeAggregateFunction::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    writeJSONString(serializeToString(function, column, row_num), ostr);
+    writeJSONString(serializeToString(function, column, row_num), ostr, settings);
 }
 
 

dbms/src/DataTypes/DataTypeEnum.cpp

@@ -97,7 +97,7 @@ DataTypeEnum<Type>::DataTypeEnum(const Values & values_) : values{values_}
     });
 
     fillMaps();
-    name = generateName(values);
+    type_name = generateName(values);
 }
 
 template <typename Type>
@@ -145,9 +145,9 @@ template <typename Type>
 void DataTypeEnum<Type>::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     /// NOTE It would be nice to do without creating a temporary object - at least extract std::string out.
-    std::string name;
+    std::string field_name;
-    readEscapedString(name, istr);
+    readEscapedString(field_name, istr);
-    static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(name)));
+    static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(field_name)));
 }
 
 template <typename Type>
@@ -159,15 +159,15 @@ void DataTypeEnum<Type>::serializeTextQuoted(const IColumn & column, size_t row_
 template <typename Type>
 void DataTypeEnum<Type>::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
-    std::string name;
+    std::string field_name;
-    readQuotedStringWithSQLStyle(name, istr);
+    readQuotedStringWithSQLStyle(field_name, istr);
-    static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(name)));
+    static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(field_name)));
 }
 
 template <typename Type>
-void DataTypeEnum<Type>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
+void DataTypeEnum<Type>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    writeJSONString(getNameForValue(static_cast<const ColumnType &>(column).getData()[row_num]), ostr);
+    writeJSONString(getNameForValue(static_cast<const ColumnType &>(column).getData()[row_num]), ostr, settings);
 }
 
 template <typename Type>
@@ -179,9 +179,9 @@ void DataTypeEnum<Type>::serializeTextXML(const IColumn & column, size_t row_num
 template <typename Type>
 void DataTypeEnum<Type>::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
-    std::string name;
+    std::string field_name;
-    readJSONString(name, istr);
+    readJSONString(field_name, istr);
-    static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(name)));
+    static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(field_name)));
 }
 
 template <typename Type>
@@ -193,9 +193,9 @@ void DataTypeEnum<Type>::serializeTextCSV(const IColumn & column, size_t row_num
 template <typename Type>
 void DataTypeEnum<Type>::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
-    std::string name;
+    std::string field_name;
-    readCSVString(name, istr, settings.csv);
+    readCSVString(field_name, istr, settings.csv);
-    static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(name)));
+    static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(field_name)));
 }
 
 template <typename Type>
@@ -237,7 +237,7 @@ void DataTypeEnum<Type>::insertDefaultInto(IColumn & column) const
 template <typename Type>
 bool DataTypeEnum<Type>::equals(const IDataType & rhs) const
 {
-    return typeid(rhs) == typeid(*this) && name == static_cast<const DataTypeEnum<Type> &>(rhs).name;
+    return typeid(rhs) == typeid(*this) && type_name == static_cast<const DataTypeEnum<Type> &>(rhs).type_name;
 }
 
 
@@ -346,14 +346,14 @@ static DataTypePtr create(const ASTPtr & arguments)
             throw Exception("Elements of Enum data type must be of form: 'name' = number, where name is string literal and number is an integer",
                 ErrorCodes::UNEXPECTED_AST_STRUCTURE);
 
-        const String & name = name_literal->value.get<String>();
+        const String & field_name = name_literal->value.get<String>();
         const auto value = value_literal->value.get<typename NearestFieldType<FieldType>::Type>();
 
         if (value > std::numeric_limits<FieldType>::max() || value < std::numeric_limits<FieldType>::min())
-            throw Exception{"Value " + toString(value) + " for element '" + name + "' exceeds range of " + EnumName<FieldType>::value,
+            throw Exception{"Value " + toString(value) + " for element '" + field_name + "' exceeds range of " + EnumName<FieldType>::value,
                 ErrorCodes::ARGUMENT_OUT_OF_BOUND};
 
-        values.emplace_back(name, value);
+        values.emplace_back(field_name, value);
     }
 
     return std::make_shared<DataTypeEnum>(values);

dbms/src/DataTypes/DataTypeEnum.h

@@ -53,7 +53,7 @@ private:
     Values values;
     NameToValueMap name_to_value_map;
     ValueToNameMap value_to_name_map;
-    std::string name;
+    std::string type_name;
 
     static std::string generateName(const Values & values);
     void fillMaps();
@@ -62,7 +62,7 @@ public:
     explicit DataTypeEnum(const Values & values_);
 
     const Values & getValues() const { return values; }
-    std::string getName() const override { return name; }
+    std::string getName() const override { return type_name; }
     const char * getFamilyName() const override;
 
     const StringRef & getNameForValue(const FieldType & value) const
@@ -74,11 +74,11 @@ public:
         return it->second;
     }
 
-    FieldType getValue(StringRef name) const
+    FieldType getValue(StringRef field_name) const
     {
-        const auto it = name_to_value_map.find(name);
+        const auto it = name_to_value_map.find(field_name);
         if (it == std::end(name_to_value_map))
-            throw Exception{"Unknown element '" + name.toString() + "' for type " + getName(), ErrorCodes::LOGICAL_ERROR};
+            throw Exception{"Unknown element '" + field_name.toString() + "' for type " + getName(), ErrorCodes::LOGICAL_ERROR};
 
         return it->second;
     }

dbms/src/DataTypes/DataTypeFixedString.cpp

@@ -168,10 +168,10 @@ void DataTypeFixedString::deserializeTextQuoted(IColumn & column, ReadBuffer & i
 }
 
 
-void DataTypeFixedString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
+void DataTypeFixedString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     const char * pos = reinterpret_cast<const char *>(&static_cast<const ColumnFixedString &>(column).getChars()[n * row_num]);
-    writeJSONString(pos, pos + n, ostr);
+    writeJSONString(pos, pos + n, ostr, settings);
 }
 
 

dbms/src/DataTypes/DataTypeString.cpp

@@ -262,9 +262,9 @@ void DataTypeString::deserializeTextQuoted(IColumn & column, ReadBuffer & istr,
 }
 
 
-void DataTypeString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
+void DataTypeString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    writeJSONString(static_cast<const ColumnString &>(column).getDataAt(row_num), ostr);
+    writeJSONString(static_cast<const ColumnString &>(column).getDataAt(row_num), ostr, settings);
 }
 
 

dbms/src/DataTypes/NestedUtils.h

@@ -22,6 +22,6 @@ namespace Nested
 
     /// Collect Array columns in a form of `column_name.element_name` to single Array(Tuple(...)) column.
     NamesAndTypesList collect(const NamesAndTypesList & names_and_types);
-};
+}
 
 }

dbms/src/Databases/DatabaseDictionary.cpp

@@ -39,15 +39,15 @@ Tables DatabaseDictionary::loadTables()
     Tables tables;
     for (const auto & pair : dictionaries)
     {
-        const std::string & name = pair.first;
+        const std::string & dict_name = pair.first;
-        if (deleted_tables.count(name))
+        if (deleted_tables.count(dict_name))
             continue;
         auto dict_ptr = std::static_pointer_cast<IDictionaryBase>(pair.second.loadable);
         if (dict_ptr)
         {
             const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
             auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
-            tables[name] = StorageDictionary::create(name, ColumnsDescription{columns}, dictionary_structure, name);
+            tables[dict_name] = StorageDictionary::create(dict_name, ColumnsDescription{columns}, dictionary_structure, dict_name);
         }
     }
 

dbms/src/Dictionaries/CacheDictionary.cpp

@@ -101,12 +101,12 @@ void CacheDictionary::isInImpl(
 {
     /// Transform all children to parents until ancestor id or null_value will be reached.
 
-    size_t size = out.size();
+    size_t out_size = out.size();
-    memset(out.data(), 0xFF, size);        /// 0xFF means "not calculated"
+    memset(out.data(), 0xFF, out_size);        /// 0xFF means "not calculated"
 
     const auto null_value = std::get<UInt64>(hierarchical_attribute->null_values);
 
-    PaddedPODArray<Key> children(size);
+    PaddedPODArray<Key> children(out_size);
     PaddedPODArray<Key> parents(child_ids.begin(), child_ids.end());
 
     while (true)
@@ -115,7 +115,7 @@ void CacheDictionary::isInImpl(
         size_t parents_idx = 0;
         size_t new_children_idx = 0;
 
-        while (out_idx < size)
+        while (out_idx < out_size)
         {
             /// Already calculated
             if (out[out_idx] != 0xFF)
@@ -203,7 +203,7 @@ void CacheDictionary::isInConstantVector(
     }
 
     /// Assuming short hierarchy, so linear search is Ok.
-    for (size_t i = 0, size = out.size(); i < size; ++i)
+    for (size_t i = 0, out_size = out.size(); i < out_size; ++i)
         out[i] = std::find(ancestors.begin(), ancestors.end(), ancestor_ids[i]) != ancestors.end();
 }
 
@@ -936,12 +936,12 @@ void CacheDictionary::setAttributeValue(Attribute & attribute, const Key idx, co
             if (string_ref.data && string_ref.data != null_value_ref.data())
                 string_arena->free(const_cast<char *>(string_ref.data), string_ref.size);
 
-            const auto size = string.size();
+            const auto str_size = string.size();
-            if (size != 0)
+            if (str_size != 0)
             {
-                auto string_ptr = string_arena->alloc(size + 1);
+                auto string_ptr = string_arena->alloc(str_size + 1);
-                std::copy(string.data(), string.data() + size + 1, string_ptr);
+                std::copy(string.data(), string.data() + str_size + 1, string_ptr);
-                string_ref = StringRef{string_ptr, size};
+                string_ref = StringRef{string_ptr, str_size};
             }
             else
                 string_ref = {};

dbms/src/Dictionaries/CatBoostModel.cpp

@@ -307,13 +307,13 @@ private:
 
     /// buffer[column_size * cat_features_count] -> char * => cat_features[column_size][cat_features_count] -> char *
     void fillCatFeaturesBuffer(const char *** cat_features, const char ** buffer,
-                               size_t column_size, size_t cat_features_count) const
+                               size_t column_size, size_t cat_features_count_current) const
     {
         for (size_t i = 0; i < column_size; ++i)
         {
             *cat_features = buffer;
             ++cat_features;
-            buffer += cat_features_count;
+            buffer += cat_features_count_current;
         }
     }
 
@@ -321,7 +321,7 @@ private:
     ///  * CalcModelPredictionFlat if no cat features
     ///  * CalcModelPrediction if all cat features are strings
     ///  * CalcModelPredictionWithHashedCatFeatures if has int cat features.
-    ColumnPtr evalImpl(const ColumnRawPtrs & columns, size_t float_features_count, size_t cat_features_count,
+    ColumnPtr evalImpl(const ColumnRawPtrs & columns, size_t float_features_count_current, size_t cat_features_count_current,
                        bool cat_features_are_strings) const
     {
         std::string error_msg = "Error occurred while applying CatBoost model: ";
@@ -334,12 +334,12 @@ private:
         PODArray<const float *> float_features(column_size);
         auto float_features_buf = float_features.data();
         /// Store all float data into single column. float_features is a list of pointers to it.
-        auto float_features_col = placeNumericColumns<float>(columns, 0, float_features_count, float_features_buf);
+        auto float_features_col = placeNumericColumns<float>(columns, 0, float_features_count_current, float_features_buf);
 
-        if (cat_features_count == 0)
+        if (cat_features_count_current == 0)
         {
             if (!api->CalcModelPredictionFlat(handle->get(), column_size,
-                                              float_features_buf, float_features_count,
+                                              float_features_buf, float_features_count_current,
                                               result_buf, column_size))
             {
 
@@ -352,18 +352,18 @@ private:
         if (cat_features_are_strings)
         {
             /// cat_features_holder stores pointers to ColumnString data or fixed_strings_data.
-            PODArray<const char *> cat_features_holder(cat_features_count * column_size);
+            PODArray<const char *> cat_features_holder(cat_features_count_current * column_size);
             PODArray<const char **> cat_features(column_size);
             auto cat_features_buf = cat_features.data();
 
-            fillCatFeaturesBuffer(cat_features_buf, cat_features_holder.data(), column_size, cat_features_count);
+            fillCatFeaturesBuffer(cat_features_buf, cat_features_holder.data(), column_size, cat_features_count_current);
             /// Fixed strings are stored without termination zero, so have to copy data into fixed_strings_data.
-            auto fixed_strings_data = placeStringColumns(columns, float_features_count,
+            auto fixed_strings_data = placeStringColumns(columns, float_features_count_current,
-                                                         cat_features_count, cat_features_holder.data());
+                                                         cat_features_count_current, cat_features_holder.data());
 
             if (!api->CalcModelPrediction(handle->get(), column_size,
-                                          float_features_buf, float_features_count,
+                                          float_features_buf, float_features_count_current,
-                                          cat_features_buf, cat_features_count,
+                                          cat_features_buf, cat_features_count_current,
                                           result_buf, column_size))
             {
                 throw Exception(error_msg + api->GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL);
@@ -373,13 +373,13 @@ private:
         {
             PODArray<const int *> cat_features(column_size);
             auto cat_features_buf = cat_features.data();
-            auto cat_features_col = placeNumericColumns<int>(columns, float_features_count,
+            auto cat_features_col = placeNumericColumns<int>(columns, float_features_count_current,
-                                                             cat_features_count, cat_features_buf);
+                                                             cat_features_count_current, cat_features_buf);
-            calcHashes(columns, float_features_count, cat_features_count, cat_features_buf);
+            calcHashes(columns, float_features_count_current, cat_features_count_current, cat_features_buf);
             if (!api->CalcModelPredictionWithHashedCatFeatures(
                     handle->get(), column_size,
-                    float_features_buf, float_features_count,
+                    float_features_buf, float_features_count_current,
-                    cat_features_buf, cat_features_count,
+                    cat_features_buf, cat_features_count_current,
                     result_buf, column_size))
             {
                 throw Exception(error_msg + api->GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL);
@@ -453,7 +453,7 @@ CatBoostModel::CatBoostModel(std::string name_, std::string model_path_, std::st
 {
     try
     {
-        init(lib_path);
+        init();
     }
     catch (...)
     {
@@ -463,7 +463,7 @@ CatBoostModel::CatBoostModel(std::string name_, std::string model_path_, std::st
     creation_time = std::chrono::system_clock::now();
 }
 
-void CatBoostModel::init(const std::string & lib_path)
+void CatBoostModel::init()
 {
     api_provider = getCatBoostWrapperHolder(lib_path);
     api = &api_provider->getAPI();

dbms/src/Dictionaries/CatBoostModel.h

@@ -80,7 +80,7 @@ private:
     std::chrono::time_point<std::chrono::system_clock> creation_time;
     std::exception_ptr creation_exception;
 
-    void init(const std::string & lib_path);
+    void init();
 };
 
 }

dbms/src/Dictionaries/ComplexKeyCacheDictionary_setAttributeValue.cpp

@@ -28,12 +28,12 @@ void ComplexKeyCacheDictionary::setAttributeValue(Attribute & attribute, const s
             if (string_ref.data && string_ref.data != null_value_ref.data())
                 string_arena->free(const_cast<char *>(string_ref.data), string_ref.size);
 
-            const auto size = string.size();
+            const auto str_size = string.size();
-            if (size != 0)
+            if (str_size != 0)
             {
-                auto string_ptr = string_arena->alloc(size + 1);
+                auto string_ptr = string_arena->alloc(str_size + 1);
-                std::copy(string.data(), string.data() + size + 1, string_ptr);
+                std::copy(string.data(), string.data() + str_size + 1, string_ptr);
-                string_ref = StringRef{string_ptr, size};
+                string_ref = StringRef{string_ptr, str_size};
             }
             else
                 string_ref = {};

dbms/src/Dictionaries/DictionaryBlockInputStream.h

@@ -223,47 +223,47 @@ Block DictionaryBlockInputStream<DictionaryType, Key>::getBlock(size_t start, si
 template <typename DictionaryType, typename Key>
 template <typename Type, typename Container>
 void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
-    DictionaryGetter<Type> getter, const PaddedPODArray<Key> & ids,
+    DictionaryGetter<Type> getter, const PaddedPODArray<Key> & ids_to_fill,
     const Columns & /*keys*/, const DataTypes & /*data_types*/,
-    Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const
+    Container & container, const DictionaryAttribute & attribute, const DictionaryType & dict) const
 {
-    (dictionary.*getter)(attribute.name, ids, container);
+    (dict.*getter)(attribute.name, ids_to_fill, container);
 }
 
 template <typename DictionaryType, typename Key>
 template <typename Container>
 void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
-    DictionaryStringGetter getter, const PaddedPODArray<Key> & ids,
+    DictionaryStringGetter getter, const PaddedPODArray<Key> & ids_to_fill,
     const Columns & /*keys*/, const DataTypes & /*data_types*/,
-    Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const
+    Container & container, const DictionaryAttribute & attribute, const DictionaryType & dict) const
 {
-    (dictionary.*getter)(attribute.name, ids, container);
+    (dict.*getter)(attribute.name, ids_to_fill, container);
 }
 
 template <typename DictionaryType, typename Key>
 template <typename Type, typename Container>
 void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
-    GetterByKey<Type> getter, const PaddedPODArray<Key> & /*ids*/,
+    GetterByKey<Type> getter, const PaddedPODArray<Key> & /*ids_to_fill*/,
     const Columns & keys, const DataTypes & data_types,
-    Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const
+    Container & container, const DictionaryAttribute & attribute, const DictionaryType & dict) const
 {
-    (dictionary.*getter)(attribute.name, keys, data_types, container);
+    (dict.*getter)(attribute.name, keys, data_types, container);
 }
 
 template <typename DictionaryType, typename Key>
 template <typename Container>
 void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
-    StringGetterByKey getter, const PaddedPODArray<Key> & /*ids*/,
+    StringGetterByKey getter, const PaddedPODArray<Key> & /*ids_to_fill*/,
     const Columns & keys, const DataTypes & data_types,
-    Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const
+    Container & container, const DictionaryAttribute & attribute, const DictionaryType & dict) const
 {
-    (dictionary.*getter)(attribute.name, keys, data_types, container);
+    (dict.*getter)(attribute.name, keys, data_types, container);
 }
 
 template <typename DictionaryType, typename Key>
 template <template <typename> class Getter, typename StringGetter>
 Block DictionaryBlockInputStream<DictionaryType, Key>::fillBlock(
-    const PaddedPODArray<Key> & ids, const Columns & keys, const DataTypes & types, ColumnsWithTypeAndName && view) const
+    const PaddedPODArray<Key> & ids_to_fill, const Columns & keys, const DataTypes & types, ColumnsWithTypeAndName && view) const
 {
     std::unordered_set<std::string> names(column_names.begin(), column_names.end());
 
@@ -283,7 +283,7 @@ Block DictionaryBlockInputStream<DictionaryType, Key>::fillBlock(
     const DictionaryStructure & structure = dictionary->getStructure();
 
     if (structure.id && names.find(structure.id->name) != names.end())
-        block_columns.emplace_back(getColumnFromIds(ids), std::make_shared<DataTypeUInt64>(), structure.id->name);
+        block_columns.emplace_back(getColumnFromIds(ids_to_fill), std::make_shared<DataTypeUInt64>(), structure.id->name);
 
     for (const auto idx : ext::range(0, structure.attributes.size()))
     {
@@ -293,7 +293,7 @@ Block DictionaryBlockInputStream<DictionaryType, Key>::fillBlock(
             ColumnPtr column;
 #define GET_COLUMN_FORM_ATTRIBUTE(TYPE) \
                 column = getColumnFromAttribute<TYPE, Getter<TYPE>>( \
-                &DictionaryType::get##TYPE, ids, keys, data_types, attribute, *dictionary)
+                &DictionaryType::get##TYPE, ids_to_fill, keys, data_types, attribute, *dictionary)
             switch (attribute.underlying_type)
             {
             case AttributeUnderlyingType::UInt8:
@@ -346,37 +346,37 @@ Block DictionaryBlockInputStream<DictionaryType, Key>::fillBlock(
 template <typename DictionaryType, typename Key>
 template <typename AttributeType, typename Getter>
 ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromAttribute(
-    Getter getter, const PaddedPODArray<Key> & ids,
+    Getter getter, const PaddedPODArray<Key> & ids_to_fill,
     const Columns & keys, const DataTypes & data_types,
-    const DictionaryAttribute & attribute, const DictionaryType & dictionary) const
+    const DictionaryAttribute & attribute, const DictionaryType & dict) const
 {
-    auto size = ids.size();
+    auto size = ids_to_fill.size();
     if (!keys.empty())
         size = keys.front()->size();
     auto column_vector = ColumnVector<AttributeType>::create(size);
-    callGetter(getter, ids, keys, data_types, column_vector->getData(), attribute, dictionary);
+    callGetter(getter, ids_to_fill, keys, data_types, column_vector->getData(), attribute, dict);
     return std::move(column_vector);
 }
 
 template <typename DictionaryType, typename Key>
 template <typename Getter>
 ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromStringAttribute(
-    Getter getter, const PaddedPODArray<Key> & ids,
+    Getter getter, const PaddedPODArray<Key> & ids_to_fill,
     const Columns & keys, const DataTypes & data_types,
-    const DictionaryAttribute& attribute, const DictionaryType& dictionary) const
+    const DictionaryAttribute& attribute, const DictionaryType & dict) const
 {
     auto column_string = ColumnString::create();
     auto ptr = column_string.get();
-    callGetter(getter, ids, keys, data_types, ptr, attribute, dictionary);
+    callGetter(getter, ids_to_fill, keys, data_types, ptr, attribute, dict);
     return std::move(column_string);
 }
 
 template <typename DictionaryType, typename Key>
-ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromIds(const PaddedPODArray<Key> & ids) const
+ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromIds(const PaddedPODArray<Key> & ids_to_fill) const
 {
     auto column_vector = ColumnVector<UInt64>::create();
-    column_vector->getData().reserve(ids.size());
+    column_vector->getData().reserve(ids_to_fill.size());
-    for (UInt64 id : ids)
+    for (UInt64 id : ids_to_fill)
         column_vector->insert(id);
     return std::move(column_vector);
 }

dbms/src/Dictionaries/DictionaryBlockInputStreamBase.h

@@ -7,7 +7,7 @@ namespace DB
 class DictionaryBlockInputStreamBase : public IProfilingBlockInputStream
 {
 protected:
-    Block block;
+    //Block block;
 
     DictionaryBlockInputStreamBase(size_t rows_count, size_t max_block_size);
 

dbms/src/Dictionaries/DictionarySourceFactory.cpp

@@ -20,10 +20,7 @@
     #include <Dictionaries/MongoDBDictionarySource.h>
 #endif
 #if USE_POCO_SQLODBC || USE_POCO_DATAODBC
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-parameter"
     #include <Poco/Data/ODBC/Connector.h>
-#pragma GCC diagnostic pop
     #include <Dictionaries/ODBCDictionarySource.h>
 #endif
 #if USE_MYSQL

dbms/src/Dictionaries/DictionaryStructure.cpp

@@ -240,22 +240,22 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
     const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
     const bool hierarchy_allowed, const bool allow_null_values)
 {
-    Poco::Util::AbstractConfiguration::Keys keys;
+    Poco::Util::AbstractConfiguration::Keys config_elems;
-    config.keys(config_prefix, keys);
+    config.keys(config_prefix, config_elems);
     auto has_hierarchy = false;
 
-    std::vector<DictionaryAttribute> attributes;
+    std::vector<DictionaryAttribute> res_attributes;
 
     const FormatSettings format_settings;
 
-    for (const auto & key : keys)
+    for (const auto & config_elem : config_elems)
     {
-        if (!startsWith(key.data(), "attribute"))
+        if (!startsWith(config_elem.data(), "attribute"))
             continue;
 
-        const auto prefix = config_prefix + '.' + key + '.';
+        const auto prefix = config_prefix + '.' + config_elem + '.';
         Poco::Util::AbstractConfiguration::Keys attribute_keys;
-        config.keys(config_prefix + '.' + key, attribute_keys);
+        config.keys(config_prefix + '.' + config_elem, attribute_keys);
 
         checkAttributeKeys(attribute_keys);
 
@@ -300,12 +300,12 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
 
         has_hierarchy = has_hierarchy || hierarchical;
 
-        attributes.emplace_back(DictionaryAttribute{
+        res_attributes.emplace_back(DictionaryAttribute{
             name, underlying_type, type, expression, null_value, hierarchical, injective, is_object_id
         });
     }
 
-    return attributes;
+    return res_attributes;
 }
 
 }

dbms/src/Dictionaries/MongoDBBlockInputStream.cpp

@@ -5,13 +5,10 @@
 #include <string>
 #include <sstream>
 
-#pragma GCC diagnostic push
+#include <Poco/MongoDB/Connection.h>
-#pragma GCC diagnostic ignored "-Wunused-parameter"
+#include <Poco/MongoDB/Cursor.h>
-    #include <Poco/MongoDB/Connection.h>
+#include <Poco/MongoDB/Element.h>
-    #include <Poco/MongoDB/Cursor.h>
+#include <Poco/MongoDB/ObjectId.h>
-    #include <Poco/MongoDB/Element.h>
-    #include <Poco/MongoDB/ObjectId.h>
-#pragma GCC diagnostic pop
 
 #include <Dictionaries/DictionaryStructure.h>
 #include <Dictionaries/MongoDBBlockInputStream.h>

dbms/src/Dictionaries/MongoDBDictionarySource.cpp

@@ -2,14 +2,11 @@
 #if USE_POCO_MONGODB
 #include <Poco/Util/AbstractConfiguration.h>
 
-#pragma GCC diagnostic push
+#include <Poco/MongoDB/Connection.h>
-#pragma GCC diagnostic ignored "-Wunused-parameter"
+#include <Poco/MongoDB/Database.h>
-    #include <Poco/MongoDB/Connection.h>
+#include <Poco/MongoDB/Cursor.h>
-    #include <Poco/MongoDB/Database.h>
+#include <Poco/MongoDB/Array.h>
-    #include <Poco/MongoDB/Cursor.h>
+#include <Poco/MongoDB/ObjectId.h>
-    #include <Poco/MongoDB/Array.h>
-    #include <Poco/MongoDB/ObjectId.h>
-#pragma GCC diagnostic pop
 
 #include <Poco/Version.h>
 

dbms/src/Dictionaries/MySQLDictionarySource.cpp

@@ -167,10 +167,10 @@ std::string MySQLDictionarySource::quoteForLike(const std::string s)
 
 LocalDateTime MySQLDictionarySource::getLastModification() const
 {
-    LocalDateTime update_time{std::time(nullptr)};
+    LocalDateTime modification_time{std::time(nullptr)};
 
     if (dont_check_update_time)
-        return update_time;
+        return modification_time;
 
     try
     {
@@ -190,8 +190,8 @@ LocalDateTime MySQLDictionarySource::getLastModification() const
 
             if (!update_time_value.isNull())
             {
-                update_time = update_time_value.getDateTime();
+                modification_time = update_time_value.getDateTime();
-                LOG_TRACE(log, "Got update time: " << update_time);
+                LOG_TRACE(log, "Got modification time: " << modification_time);
             }
 
             /// fetch remaining rows to avoid "commands out of sync" error
@@ -211,15 +211,15 @@ LocalDateTime MySQLDictionarySource::getLastModification() const
     }
 
     /// we suppose failure to get modification time is not an error, therefore return current time
-    return update_time;
+    return modification_time;
 }
 
 std::string MySQLDictionarySource::doInvalidateQuery(const std::string & request) const
 {
-    Block sample_block;
+    Block invalidate_sample_block;
     ColumnPtr column(ColumnString::create());
-    sample_block.insert(ColumnWithTypeAndName(column, std::make_shared<DataTypeString>(), "Sample Block"));
+    invalidate_sample_block.insert(ColumnWithTypeAndName(column, std::make_shared<DataTypeString>(), "Sample Block"));
-    MySQLBlockInputStream block_input_stream(pool.Get(), request, sample_block, 1);
+    MySQLBlockInputStream block_input_stream(pool.Get(), request, invalidate_sample_block, 1);
     return readInvalidateQuery(block_input_stream);
 }
 

dbms/src/Dictionaries/ODBCBlockInputStream.h

@@ -4,12 +4,9 @@
 #include <DataStreams/IProfilingBlockInputStream.h>
 #include <Dictionaries/ExternalResultDescription.h>
 
-#pragma GCC diagnostic push
+#include <Poco/Data/Session.h>
-#pragma GCC diagnostic ignored "-Wunused-parameter"
+#include <Poco/Data/Statement.h>
-    #include <Poco/Data/Session.h>
+#include <Poco/Data/RecordSet.h>
-    #include <Poco/Data/Statement.h>
-    #include <Poco/Data/RecordSet.h>
-#pragma GCC diagnostic pop
 
 #include <string>
 

dbms/src/Dictionaries/ODBCDictionarySource.cpp

@@ -145,10 +145,10 @@ bool ODBCDictionarySource::isModified() const
 
 std::string ODBCDictionarySource::doInvalidateQuery(const std::string & request) const
 {
-    Block sample_block;
+    Block invalidate_sample_block;
     ColumnPtr column(ColumnString::create());
-    sample_block.insert(ColumnWithTypeAndName(column, std::make_shared<DataTypeString>(), "Sample Block"));
+    invalidate_sample_block.insert(ColumnWithTypeAndName(column, std::make_shared<DataTypeString>(), "Sample Block"));
-    ODBCBlockInputStream block_input_stream(pool->get(), request, sample_block, 1);
+    ODBCBlockInputStream block_input_stream(pool->get(), request, invalidate_sample_block, 1);
     return readInvalidateQuery(block_input_stream);
 }
 

dbms/src/Dictionaries/ODBCDictionarySource.h

@@ -1,13 +1,12 @@
 #pragma once
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-parameter"
 #include <Poco/Data/SessionPool.h>
-#pragma GCC diagnostic pop
+
 #include <Dictionaries/IDictionarySource.h>
 #include <Dictionaries/ExternalQueryBuilder.h>
 #include <Dictionaries/DictionaryStructure.h>
 
+
 namespace Poco
 {
     namespace Util

dbms/src/Dictionaries/validateODBCConnectionString.cpp

@@ -189,22 +189,22 @@ std::string validateODBCConnectionString(const std::string & connection_string)
     {
         reconstructed_connection_string += '{';
 
-        const char * pos = value.data();
+        const char * value_pos = value.data();
-        const char * end = pos + value.size();
+        const char * value_end = value_pos + value.size();
         while (true)
         {
-            const char * next_pos = find_first_symbols<'}'>(pos, end);
+            const char * next_pos = find_first_symbols<'}'>(value_pos, value_end);
 
-            if (next_pos == end)
+            if (next_pos == value_end)
             {
-                reconstructed_connection_string.append(pos, next_pos - pos);
+                reconstructed_connection_string.append(value_pos, next_pos - value_pos);
                 break;
             }
             else
             {
-                reconstructed_connection_string.append(pos, next_pos - pos);
+                reconstructed_connection_string.append(value_pos, next_pos - value_pos);
                 reconstructed_connection_string.append("}}");
-                pos = next_pos + 1;
+                value_pos = next_pos + 1;
             }
         }
 

dbms/src/Formats/FormatFactory.cpp

@@ -60,6 +60,7 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, WriteBuffer &
     FormatSettings format_settings;
     format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers;
     format_settings.json.quote_denormals = settings.output_format_json_quote_denormals;
+    format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes;
     format_settings.csv.delimiter = settings.format_csv_delimiter;
     format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes;
     format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes;
@@ -119,6 +120,7 @@ void registerOutputFormatJSON(FormatFactory & factory);
 void registerOutputFormatJSONCompact(FormatFactory & factory);
 void registerOutputFormatXML(FormatFactory & factory);
 void registerOutputFormatODBCDriver(FormatFactory & factory);
+void registerOutputFormatODBCDriver2(FormatFactory & factory);
 void registerOutputFormatNull(FormatFactory & factory);
 
 /// Input only formats.
@@ -152,6 +154,7 @@ FormatFactory::FormatFactory()
     registerOutputFormatJSONCompact(*this);
     registerOutputFormatXML(*this);
     registerOutputFormatODBCDriver(*this);
+    registerOutputFormatODBCDriver2(*this);
     registerOutputFormatNull(*this);
 }
 

dbms/src/Formats/FormatSettings.h

@@ -17,6 +17,7 @@ struct FormatSettings
     {
         bool quote_64bit_integers = true;
         bool quote_denormals = true;
+        bool escape_forward_slashes = true;
     };
 
     JSON json;

dbms/src/Formats/JSONEachRowRowOutputStream.cpp

@@ -18,7 +18,7 @@ JSONEachRowRowOutputStream::JSONEachRowRowOutputStream(WriteBuffer & ostr_, cons
     for (size_t i = 0; i < columns; ++i)
     {
         WriteBufferFromString out(fields[i]);
-        writeJSONString(sample.getByPosition(i).name, out);
+        writeJSONString(sample.getByPosition(i).name, out, settings);
     }
 }
 

dbms/src/Formats/JSONRowOutputStream.cpp

@@ -21,7 +21,7 @@ JSONRowOutputStream::JSONRowOutputStream(WriteBuffer & ostr_, const Block & samp
             need_validate_utf8 = true;
 
         WriteBufferFromOwnString out;
-        writeJSONString(fields[i].name, out);
+        writeJSONString(fields[i].name, out, settings);
 
         fields[i].name = out.str();
     }
@@ -50,7 +50,7 @@ void JSONRowOutputStream::writePrefix()
         writeString(fields[i].name, *ostr);
         writeCString(",\n", *ostr);
         writeCString("\t\t\t\"type\": ", *ostr);
-        writeJSONString(fields[i].type->getName(), *ostr);
+        writeJSONString(fields[i].type->getName(), *ostr, settings);
         writeChar('\n', *ostr);
 
         writeCString("\t\t}", *ostr);
@@ -149,7 +149,7 @@ void JSONRowOutputStream::writeTotals()
                 writeCString(",\n", *ostr);
 
             writeCString("\t\t", *ostr);
-            writeJSONString(column.name, *ostr);
+            writeJSONString(column.name, *ostr, settings);
             writeCString(": ", *ostr);
             column.type->serializeTextJSON(*column.column.get(), 0, *ostr, settings);
         }
@@ -176,7 +176,7 @@ static void writeExtremesElement(const char * title, const Block & extremes, siz
             writeCString(",\n", ostr);
 
         writeCString("\t\t\t", ostr);
-        writeJSONString(column.name, ostr);
+        writeJSONString(column.name, ostr, settings);
         writeCString(": ", ostr);
         column.type->serializeTextJSON(*column.column.get(), row_num, ostr, settings);
     }

dbms/src/Formats/ODBCDriver2BlockOutputStream.cpp

@@ -0,0 +1,96 @@
+#include <Core/Block.h>
+#include <Formats/FormatFactory.h>
+#include <Formats/ODBCDriver2BlockOutputStream.h>
+#include <IO/WriteBuffer.h>
+#include <IO/WriteHelpers.h>
+
+
+#include <Core/iostream_debug_helpers.h>
+
+
+namespace DB
+{
+ODBCDriver2BlockOutputStream::ODBCDriver2BlockOutputStream(
+    WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings)
+    : out(out_), header(header_), format_settings(format_settings)
+{
+}
+
+void ODBCDriver2BlockOutputStream::flush()
+{
+    out.next();
+}
+
+void writeODBCString(WriteBuffer & out, const std::string & str)
+{
+    writeIntBinary(Int32(str.size()), out);
+    out.write(str.data(), str.size());
+}
+
+void ODBCDriver2BlockOutputStream::write(const Block & block)
+{
+    const size_t rows = block.rows();
+    const size_t columns = block.columns();
+    String text_value;
+
+    for (size_t i = 0; i < rows; ++i)
+    {
+        for (size_t j = 0; j < columns; ++j)
+        {
+            text_value.resize(0);
+            const ColumnWithTypeAndName & col = block.getByPosition(j);
+
+            if (col.column->isNullAt(i))
+            {
+                writeIntBinary(Int32(-1), out);
+            }
+            else
+            {
+                {
+                    WriteBufferFromString text_out(text_value);
+                    col.type->serializeText(*col.column, i, text_out, format_settings);
+                }
+                writeODBCString(out, text_value);
+            }
+        }
+    }
+}
+
+void ODBCDriver2BlockOutputStream::writePrefix()
+{
+    const size_t columns = header.columns();
+
+    /// Number of header rows.
+    writeIntBinary(Int32(2), out);
+
+    /// Names of columns.
+    /// Number of columns + 1 for first name column.
+    writeIntBinary(Int32(columns + 1), out);
+    writeODBCString(out, "name");
+    for (size_t i = 0; i < columns; ++i)
+    {
+        const ColumnWithTypeAndName & col = header.getByPosition(i);
+        writeODBCString(out, col.name);
+    }
+
+    /// Types of columns.
+    writeIntBinary(Int32(columns + 1), out);
+    writeODBCString(out, "type");
+    for (size_t i = 0; i < columns; ++i)
+    {
+        const ColumnWithTypeAndName & col = header.getByPosition(i);
+        writeODBCString(out, col.type->getName());
+    }
+}
+
+
+void registerOutputFormatODBCDriver2(FormatFactory & factory)
+{
+    factory.registerOutputFormat(
+        "ODBCDriver2", [](WriteBuffer & buf, const Block & sample, const Context &, const FormatSettings & format_settings)
+        {
+            return std::make_shared<ODBCDriver2BlockOutputStream>(buf, sample, format_settings);
+        });
+}
+
+}

dbms/src/Formats/ODBCDriver2BlockOutputStream.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <string>
+#include <Core/Block.h>
+#include <DataStreams/IBlockOutputStream.h>
+#include <Formats/FormatSettings.h>
+
+
+namespace DB
+{
+class WriteBuffer;
+
+
+/** A data format designed to simplify the implementation of the ODBC driver.
+  * ODBC driver is designed to be build for different platforms without dependencies from the main code,
+  *  so the format is made that way so that it can be as easy as possible to parse it.
+  * A header is displayed with the required information.
+  * The data is then output in the order of the rows. Each value is displayed as follows: length in Int32 format (-1 for NULL), then data in text form.
+  */
+class ODBCDriver2BlockOutputStream : public IBlockOutputStream
+{
+public:
+    ODBCDriver2BlockOutputStream(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings);
+
+    Block getHeader() const override
+    {
+        return header;
+    }
+    void write(const Block & block) override;
+    void writePrefix() override;
+
+    void flush() override;
+    std::string getContentType() const override
+    {
+        return "application/octet-stream";
+    }
+
+private:
+    WriteBuffer & out;
+    const Block header;
+    const FormatSettings format_settings;
+};
+
+}

dbms/src/Functions/FunctionsHashing.cpp

@@ -20,7 +20,10 @@ void registerFunctionsHashing(FunctionFactory & factory)
     factory.registerFunction<FunctionIntHash32>();
     factory.registerFunction<FunctionIntHash64>();
     factory.registerFunction<FunctionURLHash>();
-    factory.registerFunction<FunctionMurmurHash2>();
+    factory.registerFunction<FunctionMurmurHash2_32>();
+    factory.registerFunction<FunctionMurmurHash2_64>();
+    factory.registerFunction<FunctionMurmurHash3_32>();
+    factory.registerFunction<FunctionMurmurHash3_64>();
+    factory.registerFunction<FunctionMurmurHash3_128>();
 }
-
 }

dbms/src/Functions/FunctionsHashing.h

@@ -6,6 +6,7 @@
 #include <farmhash.h>
 #include <metrohash.h>
 #include <murmurhash2.h>
+#include <murmurhash3.h>
 
 #include <Poco/ByteOrder.h>
 
@@ -64,6 +65,8 @@ namespace ErrorCodes
 
 struct HalfMD5Impl
 {
+    using ReturnType = UInt64;
+
     static UInt64 apply(const char * begin, size_t size)
     {
         union
@@ -139,6 +142,8 @@ struct SHA256Impl
 
 struct SipHash64Impl
 {
+    using ReturnType = UInt64;
+
     static UInt64 apply(const char * begin, size_t size)
     {
         return sipHash64(begin, size);
@@ -179,63 +184,6 @@ struct IntHash64Impl
 };
 
 
-template <typename Impl, typename Name>
-class FunctionStringHash64 : public IFunction
-{
-public:
-    static constexpr auto name = Name::name;
-    static FunctionPtr create(const Context &) { return std::make_shared<FunctionStringHash64>(); }
-
-    String getName() const override
-    {
-        return name;
-    }
-
-    size_t getNumberOfArguments() const override { return 1; }
-
-    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
-    {
-        if (!arguments[0]->isString())
-            throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
-
-        return std::make_shared<DataTypeUInt64>();
-    }
-
-    bool useDefaultImplementationForConstants() const override { return true; }
-
-    void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override
-    {
-        if (const ColumnString * col_from = checkAndGetColumn<ColumnString>(block.getByPosition(arguments[0]).column.get()))
-        {
-            auto col_to = ColumnUInt64::create();
-
-            const typename ColumnString::Chars_t & data = col_from->getChars();
-            const typename ColumnString::Offsets & offsets = col_from->getOffsets();
-            typename ColumnUInt64::Container & vec_to = col_to->getData();
-            size_t size = offsets.size();
-            vec_to.resize(size);
-
-            ColumnString::Offset current_offset = 0;
-            for (size_t i = 0; i < size; ++i)
-            {
-                vec_to[i] = Impl::apply(
-                    reinterpret_cast<const char *>(&data[current_offset]),
-                    offsets[i] - current_offset - 1);
-
-                current_offset = offsets[i];
-            }
-
-            block.getByPosition(result).column = std::move(col_to);
-        }
-        else
-            throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
-                    + " of first argument of function " + Name::name,
-                ErrorCodes::ILLEGAL_COLUMN);
-    }
-};
-
-
 template <typename Impl>
 class FunctionStringHashFixedString : public IFunction
 {
@@ -622,12 +570,12 @@ public:
 };
 
 
-template <typename Impl>
+template <typename Impl, typename Name>
-class FunctionStringHash32 : public IFunction
+class FunctionStringHash : public IFunction
 {
 public:
-    static constexpr auto name = Impl::name;
+    static constexpr auto name = Name::name;
-    static FunctionPtr create(const Context &) { return std::make_shared<FunctionStringHash32>(); }
+    static FunctionPtr create(const Context &) { return std::make_shared<FunctionStringHash>(); }
 
     String getName() const override { return name; }
 
@@ -635,14 +583,15 @@ public:
 
     size_t getNumberOfArguments() const override { return 1; }
 
-    DataTypePtr getReturnTypeImpl(const DataTypes & /* arguments */) const override { return std::make_shared<DataTypeUInt32>(); }
+    DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments */) const override
+    { return std::make_shared<DataTypeNumber<ToType>>(); }
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
     void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
     {
-        auto col_to = ColumnUInt32::create(input_rows_count);
+        auto col_to = ColumnVector<ToType>::create(input_rows_count);
-        ColumnUInt32::Container & vec_to = col_to->getData();
+        typename ColumnVector<ToType>::Container & vec_to = col_to->getData();
 
         const ColumnWithTypeAndName & col = block.getByPosition(arguments[0]);
         const IDataType * from_type = col.type.get();
@@ -671,8 +620,10 @@ public:
         block.getByPosition(result).column = std::move(col_to);
     }
 private:
+    using ToType = typename Impl::ReturnType;
+
     template <typename FromType>
-    void executeIntType(const IColumn * column, ColumnUInt32::Container & vec_to)
+    void executeIntType(const IColumn * column, typename ColumnVector<ToType>::Container & vec_to)
     {
         if (const ColumnVector<FromType> * col_from = checkAndGetColumn<ColumnVector<FromType>>(column))
         {
@@ -680,7 +631,7 @@ private:
             size_t size = vec_from.size();
             for (size_t i = 0; i < size; ++i)
             {
-                vec_to[i] = Impl::Hash32(reinterpret_cast<const char *>(&vec_from[i]), sizeof(FromType));
+                vec_to[i] = Impl::apply(reinterpret_cast<const char *>(&vec_from[i]), sizeof(FromType));
             }
         }
         else
@@ -689,7 +640,7 @@ private:
                 ErrorCodes::ILLEGAL_COLUMN);
     }
 
-    void executeString(const IColumn * column, ColumnUInt32::Container & vec_to)
+    void executeString(const IColumn * column, typename ColumnVector<ToType>::Container & vec_to)
     {
         if (const ColumnString * col_from = checkAndGetColumn<ColumnString>(column))
         {
@@ -700,7 +651,7 @@ private:
             ColumnString::Offset current_offset = 0;
             for (size_t i = 0; i < size; ++i)
             {
-                vec_to[i] = Impl::Hash32(
+                vec_to[i] = Impl::apply(
                     reinterpret_cast<const char *>(&data[current_offset]),
                     offsets[i] - current_offset - 1);
 
@@ -713,7 +664,7 @@ private:
             size_t n = col_from->getN();
             size_t size = data.size() / n;
             for (size_t i = 0; i < size; ++i)
-                vec_to[i] = Impl::Hash32(reinterpret_cast<const char *>(&data[i * n]), n);
+                vec_to[i] = Impl::apply(reinterpret_cast<const char *>(&data[i * n]), n);
         }
         else
             throw Exception("Illegal column " + column->getName()
@@ -730,15 +681,69 @@ private:
   * in ClickHouse as is. For example, it is needed to reproduce the behaviour
   * for NGINX a/b testing module: https://nginx.ru/en/docs/http/ngx_http_split_clients_module.html
   */
-struct MurmurHash2Impl
+struct MurmurHash2Impl32
 {
-    static constexpr auto name = "murmurHash2_32";
+    using ReturnType = UInt32;
-    static UInt32 Hash32(const char * data, const size_t size)
+
+    static UInt32 apply(const char * data, const size_t size)
     {
         return MurmurHash2(data, size, 0);
     }
 };
 
+struct MurmurHash2Impl64
+{
+    using ReturnType = UInt64;
+
+    static UInt64 apply(const char * data, const size_t size)
+    {
+        return MurmurHash64A(data, size, 0);
+    }
+};
+
+struct MurmurHash3Impl32
+{
+    using ReturnType = UInt32;
+
+    static UInt32 apply(const char * data, const size_t size)
+    {
+        union
+        {
+            UInt32 h;
+            char bytes[sizeof(h)];
+        };
+        MurmurHash3_x86_32(data, size, 0, bytes);
+        return h;
+    }
+};
+
+struct MurmurHash3Impl64
+{
+    using ReturnType = UInt64;
+
+    static UInt64 apply(const char * data, const size_t size)
+    {
+        union
+        {
+            UInt64 h[2];
+            char bytes[16];
+        };
+        MurmurHash3_x64_128(data, size, 0, bytes);
+        return h[0] ^ h[1];
+    }
+};
+
+struct MurmurHash3Impl128
+{
+    static constexpr auto name = "murmurHash3_128";
+    enum { length = 16 };
+
+    static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
+    {
+        MurmurHash3_x64_128(begin, size, 0, out_char_data);
+    }
+};
+
 
 struct URLHashImpl
 {
@@ -933,6 +938,12 @@ struct NameHalfMD5   { static constexpr auto name = "halfMD5"; };
 struct NameSipHash64 { static constexpr auto name = "sipHash64"; };
 struct NameIntHash32 { static constexpr auto name = "intHash32"; };
 struct NameIntHash64 { static constexpr auto name = "intHash64"; };
+struct NameMurmurHash2_32 { static constexpr auto name = "murmurHash2_32"; };
+struct NameMurmurHash2_64 { static constexpr auto name = "murmurHash2_64"; };
+struct NameMurmurHash3_32 { static constexpr auto name = "murmurHash3_32"; };
+struct NameMurmurHash3_64 { static constexpr auto name = "murmurHash3_64"; };
+struct NameMurmurHash3_128 { static constexpr auto name = "murmurHash3_128"; };
+
 
 struct ImplCityHash64
 {
@@ -973,8 +984,8 @@ struct ImplMetroHash64
     }
 };
 
-using FunctionHalfMD5 = FunctionStringHash64<HalfMD5Impl, NameHalfMD5>;
+using FunctionHalfMD5 = FunctionStringHash<HalfMD5Impl, NameHalfMD5>;
-using FunctionSipHash64 = FunctionStringHash64<SipHash64Impl, NameSipHash64>;
+using FunctionSipHash64 = FunctionStringHash<SipHash64Impl, NameSipHash64>;
 using FunctionIntHash32 = FunctionIntHash<IntHash32Impl, NameIntHash32>;
 using FunctionIntHash64 = FunctionIntHash<IntHash64Impl, NameIntHash64>;
 using FunctionMD5 = FunctionStringHashFixedString<MD5Impl>;
@@ -985,5 +996,9 @@ using FunctionSipHash128 = FunctionStringHashFixedString<SipHash128Impl>;
 using FunctionCityHash64 = FunctionNeighbourhoodHash64<ImplCityHash64>;
 using FunctionFarmHash64 = FunctionNeighbourhoodHash64<ImplFarmHash64>;
 using FunctionMetroHash64 = FunctionNeighbourhoodHash64<ImplMetroHash64>;
-using FunctionMurmurHash2 = FunctionStringHash32<MurmurHash2Impl>;
+using FunctionMurmurHash2_32 = FunctionStringHash<MurmurHash2Impl32, NameMurmurHash2_32>;
+using FunctionMurmurHash2_64 = FunctionStringHash<MurmurHash2Impl64, NameMurmurHash2_64>;
+using FunctionMurmurHash3_32 = FunctionStringHash<MurmurHash3Impl32, NameMurmurHash3_32>;
+using FunctionMurmurHash3_64 = FunctionStringHash<MurmurHash3Impl64, NameMurmurHash3_64>;
+using FunctionMurmurHash3_128 = FunctionStringHashFixedString<MurmurHash3Impl128>;
 }

dbms/src/Functions/FunctionsMath.h

@@ -16,18 +16,18 @@
   */
 
 #if USE_VECTORCLASS
-       #ifdef __clang__
+    #ifdef __clang__
-               #pragma clang diagnostic push
+            #pragma clang diagnostic push
-               #pragma clang diagnostic ignored "-Wshift-negative-value"
+            #pragma clang diagnostic ignored "-Wshift-negative-value"
-       #endif
+    #endif
 
-       #include <vectorf128.h> // Y_IGNORE
+    #include <vectorf128.h> // Y_IGNORE
-       #include <vectormath_exp.h> // Y_IGNORE
+    #include <vectormath_exp.h> // Y_IGNORE
-       #include <vectormath_trig.h> // Y_IGNORE
+    #include <vectormath_trig.h> // Y_IGNORE
 
-       #ifdef __clang__
+    #ifdef __clang__
-               #pragma clang diagnostic pop
+            #pragma clang diagnostic pop
-       #endif
+    #endif
 #endif