mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-25 17:12:03 +00:00
Merge branch 'master' into decimal
This commit is contained in:
commit
f538bd2113
1
.gitignore
vendored
1
.gitignore
vendored
@ -178,7 +178,6 @@ utils/zookeeper-create-entry-to-download-part/zookeeper-create-entry-to-download
|
||||
utils/zookeeper-dump-tree/zookeeper-dump-tree
|
||||
utils/zookeeper-remove-by-list/zookeeper-remove-by-list
|
||||
dbms/src/Storages/tests/remove_symlink_directory
|
||||
dbms/tests/queries/1_stateful
|
||||
debian/control
|
||||
debian/copyright
|
||||
debian/tmp/
|
||||
|
2
.gitmodules
vendored
2
.gitmodules
vendored
@ -15,7 +15,7 @@
|
||||
url = https://github.com/google/cctz.git
|
||||
[submodule "contrib/zlib-ng"]
|
||||
path = contrib/zlib-ng
|
||||
url = https://github.com/Dead2/zlib-ng.git
|
||||
url = https://github.com/ClickHouse-Extras/zlib-ng.git
|
||||
[submodule "contrib/googletest"]
|
||||
path = contrib/googletest
|
||||
url = https://github.com/google/googletest.git
|
||||
|
@ -36,6 +36,10 @@
|
||||
* Fixed a bug in the `anyHeavy` aggregate function ([a2101df2](https://github.com/yandex/ClickHouse/commit/a2101df25a6a0fba99aa71f8793d762af2b801ee))
|
||||
* Fixed server crash when using the `countArray()` aggregate function.
|
||||
|
||||
### Backward incompatible changes:
|
||||
|
||||
* Parameters for `Kafka` engine was changed from `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_schema, kafka_num_consumers])` to `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_row_delimiter, kafka_schema, kafka_num_consumers])`. If your tables use `kafka_schema` or `kafka_num_consumers` parameters, you have to manually edit the metadata files `path/metadata/database/table.sql` and add `kafka_row_delimiter` parameter with `''` value.
|
||||
|
||||
## ClickHouse release 18.1.0, 2018-07-23
|
||||
|
||||
### New features:
|
||||
|
@ -43,6 +43,10 @@
|
||||
* Исправлена ошибка в агрегатной функции `anyHeavy` ([a2101df2](https://github.com/yandex/ClickHouse/commit/a2101df25a6a0fba99aa71f8793d762af2b801ee))
|
||||
* Исправлено падение сервера при использовании функции `countArray()`.
|
||||
|
||||
### Обратно несовместимые изменения:
|
||||
|
||||
* Список параметров для таблиц `Kafka` был изменён с `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_schema, kafka_num_consumers])` на `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_row_delimiter, kafka_schema, kafka_num_consumers])`. Если вы использовали параметры `kafka_schema` или `kafka_num_consumers`, вам необходимо вручную отредактировать файлы с метаданными `path/metadata/database/table.sql`, добавив параметр `kafka_row_delimiter` со значением `''` в соответствующее место.
|
||||
|
||||
|
||||
## ClickHouse release 18.1.0, 2018-07-23
|
||||
|
||||
|
@ -34,10 +34,9 @@ endif ()
|
||||
string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
|
||||
message (STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE} )
|
||||
|
||||
# ASan - build type with address sanitizer
|
||||
# UBSan - build type with undefined behaviour sanitizer
|
||||
# TSan is not supported due to false positive errors in libstdc++ and necessity to rebuild libstdc++ with TSan
|
||||
set (CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel;ASan;UBSan" CACHE STRING "" FORCE)
|
||||
set (CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel" CACHE STRING "" FORCE)
|
||||
|
||||
include (cmake/sanitize.cmake)
|
||||
|
||||
include (cmake/arch.cmake)
|
||||
|
||||
@ -62,7 +61,7 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
|
||||
endif ()
|
||||
|
||||
option (TEST_COVERAGE "Enables flags for test coverage" OFF)
|
||||
option (ENABLE_TESTS "Enables tests" ${NOT_MSVC})
|
||||
option (ENABLE_TESTS "Enables tests" ON)
|
||||
|
||||
option (USE_STATIC_LIBRARIES "Set to FALSE to use shared libraries" ON)
|
||||
option (MAKE_STATIC_LIBRARIES "Set to FALSE to make shared libraries" ${USE_STATIC_LIBRARIES})
|
||||
@ -175,7 +174,7 @@ if (OS_LINUX AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
|
||||
endif ()
|
||||
|
||||
if (LIBCXX_PATH)
|
||||
# include_directories (BEFORE SYSTEM "${LIBCXX_PATH}/include" "${LIBCXX_PATH}/include/c++/v1")
|
||||
# include_directories (SYSTEM BEFORE "${LIBCXX_PATH}/include" "${LIBCXX_PATH}/include/c++/v1")
|
||||
link_directories ("${LIBCXX_PATH}/lib")
|
||||
endif ()
|
||||
endif ()
|
||||
@ -190,8 +189,6 @@ if (NOT MAKE_STATIC_LIBRARIES)
|
||||
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
|
||||
endif ()
|
||||
|
||||
include (cmake/sanitize.cmake)
|
||||
|
||||
# Using "include-what-you-use" tool.
|
||||
option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF)
|
||||
if (USE_INCLUDE_WHAT_YOU_USE)
|
||||
|
@ -21,7 +21,7 @@ BUILD_TARGETS=clickhouse
|
||||
BUILD_TYPE=Debug
|
||||
ENABLE_EMBEDDED_COMPILER=0
|
||||
|
||||
CMAKE_FLAGS="-D CMAKE_C_FLAGS_ADD=-g0 -D CMAKE_CXX_FLAGS_ADD=-g0 -D ENABLE_TCMALLOC=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_UNWIND=0 -D ENABLE_ICU=0 -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_POCO_ODBC=0 -D ENABLE_ODBC=0 -D ENABLE_MYSQL=0"
|
||||
CMAKE_FLAGS="-D CMAKE_C_FLAGS_ADD=-g0 -D CMAKE_CXX_FLAGS_ADD=-g0 -D ENABLE_JEMALLOC=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_UNWIND=0 -D ENABLE_ICU=0 -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_POCO_ODBC=0 -D ENABLE_ODBC=0 -D ENABLE_MYSQL=0"
|
||||
|
||||
[[ $(uname) == "FreeBSD" ]] && COMPILER_PACKAGE_VERSION=devel && export COMPILER_PATH=/usr/local/bin
|
||||
|
||||
|
@ -1,4 +1,4 @@
|
||||
option (ENABLE_CAPNP "Enable Cap'n Proto" ${NOT_MSVC})
|
||||
option (ENABLE_CAPNP "Enable Cap'n Proto" ON)
|
||||
|
||||
if (ENABLE_CAPNP)
|
||||
# cmake 3.5.1 bug:
|
||||
|
@ -43,6 +43,12 @@ if (ENABLE_EMBEDDED_COMPILER)
|
||||
else()
|
||||
set (USE_EMBEDDED_COMPILER 0)
|
||||
endif()
|
||||
|
||||
if (LLVM_FOUND AND OS_LINUX AND USE_LIBCXX)
|
||||
message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY is not set but the LLVM library from OS packages in Linux is incompatible with libc++ ABI. LLVM Will be disabled.")
|
||||
set (LLVM_FOUND 0)
|
||||
set (USE_EMBEDDED_COMPILER 0)
|
||||
endif ()
|
||||
else()
|
||||
set (LLVM_FOUND 1)
|
||||
set (USE_EMBEDDED_COMPILER 1)
|
||||
|
@ -1,4 +1,4 @@
|
||||
option (ENABLE_RDKAFKA "Enable kafka" ${NOT_MSVC})
|
||||
option (ENABLE_RDKAFKA "Enable kafka" ON)
|
||||
|
||||
if (ENABLE_RDKAFKA)
|
||||
|
||||
|
@ -1,27 +1,37 @@
|
||||
option (SANITIZE "Enable sanitizer: address, memory, thread, undefined" "")
|
||||
|
||||
set (SAN_FLAGS "${SAN_FLAGS} -g -fno-omit-frame-pointer -DSANITIZER")
|
||||
if (SAN_DEBUG)
|
||||
set (SAN_FLAGS "${SAN_FLAGS} -O0")
|
||||
else ()
|
||||
set (SAN_FLAGS "${SAN_FLAGS} -O3")
|
||||
endif ()
|
||||
|
||||
set (CMAKE_CXX_FLAGS_ASAN "${CMAKE_CXX_FLAGS_ASAN} ${SAN_FLAGS} -fsanitize=address")
|
||||
set (CMAKE_C_FLAGS_ASAN "${CMAKE_C_FLAGS_ASAN} ${SAN_FLAGS} -fsanitize=address")
|
||||
set (CMAKE_EXE_LINKER_FLAGS_ASAN "${CMAKE_EXE_LINKER_FLAGS_ASAN} -fsanitize=address")
|
||||
set (CMAKE_CXX_FLAGS_UBSAN "${CMAKE_CXX_FLAGS_UBSAN} ${SAN_FLAGS} -fsanitize=undefined")
|
||||
set (CMAKE_C_FLAGS_UBSAN "${CMAKE_C_FLAGS_UBSAN} ${SAN_FLAGS} -fsanitize=undefined")
|
||||
set (CMAKE_EXE_LINKER_FLAGS_UBSAN "${CMAKE_EXE_LINKER_FLAGS_UBSAN} -fsanitize=undefined")
|
||||
set (CMAKE_CXX_FLAGS_MSAN "${CMAKE_CXX_FLAGS_MSAN} ${SAN_FLAGS} -fsanitize=memory")
|
||||
set (CMAKE_C_FLAGS_MSAN "${CMAKE_C_FLAGS_MSAN} ${SAN_FLAGS} -fsanitize=memory")
|
||||
set (CMAKE_EXE_LINKER_FLAGS_MSAN "${CMAKE_EXE_LINKER_FLAGS_MSAN} -fsanitize=memory")
|
||||
set (CMAKE_CXX_FLAGS_TSAN "${CMAKE_CXX_FLAGS_TSAN} ${SAN_FLAGS} -fsanitize=thread")
|
||||
set (CMAKE_C_FLAGS_TSAN "${CMAKE_C_FLAGS_TSAN} ${SAN_FLAGS} -fsanitize=thread")
|
||||
set (CMAKE_EXE_LINKER_FLAGS_TSAN "${CMAKE_EXE_LINKER_FLAGS_TSAN} -fsanitize=thread")
|
||||
|
||||
# clang use static linking by default
|
||||
if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
set (CMAKE_EXE_LINKER_FLAGS_ASAN "${CMAKE_EXE_LINKER_FLAGS_ASAN} -static-libasan")
|
||||
set (CMAKE_EXE_LINKER_FLAGS_UBSAN "${CMAKE_EXE_LINKER_FLAGS_UBSAN} -static-libubsan")
|
||||
set (CMAKE_EXE_LINKER_FLAGS_MSAN "${CMAKE_EXE_LINKER_FLAGS_MSAN} -static-libmsan")
|
||||
set (CMAKE_EXE_LINKER_FLAGS_TSAN "${CMAKE_EXE_LINKER_FLAGS_TSAN} -static-libtsan")
|
||||
endif ()
|
||||
if (SANITIZE)
|
||||
if (SANITIZE STREQUAL "address")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=address")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=address")
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
|
||||
if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libasan")
|
||||
endif ()
|
||||
elseif (SANITIZE STREQUAL "memory")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=memory")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=memory")
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory")
|
||||
if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libmsan")
|
||||
endif ()
|
||||
elseif (SANITIZE STREQUAL "thread")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=thread")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=thread")
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread")
|
||||
if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libtsan")
|
||||
endif ()
|
||||
elseif (SANITIZE STREQUAL "undefined")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined")
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
|
||||
if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libubsan")
|
||||
endif ()
|
||||
else ()
|
||||
message (FATAL_ERROR "Unknown sanitizer type: ${SANITIZE}")
|
||||
endif ()
|
||||
endif()
|
||||
|
31
contrib/CMakeLists.txt
vendored
31
contrib/CMakeLists.txt
vendored
@ -1,8 +1,8 @@
|
||||
# Third-party libraries may have substandard code.
|
||||
|
||||
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -std=c++1z")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-implicit-fallthrough -std=c++1z")
|
||||
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-format -std=c++1z")
|
||||
@ -98,8 +98,8 @@ if (USE_INTERNAL_SSL_LIBRARY)
|
||||
set (USE_SHARED ${USE_STATIC_LIBRARIES})
|
||||
set (LIBRESSL_SKIP_INSTALL 1)
|
||||
add_subdirectory (ssl)
|
||||
target_include_directories(${OPENSSL_CRYPTO_LIBRARY} PUBLIC ${OPENSSL_INCLUDE_DIR})
|
||||
target_include_directories(${OPENSSL_SSL_LIBRARY} PUBLIC ${OPENSSL_INCLUDE_DIR})
|
||||
target_include_directories(${OPENSSL_CRYPTO_LIBRARY} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
|
||||
target_include_directories(${OPENSSL_SSL_LIBRARY} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
|
||||
endif ()
|
||||
|
||||
if (ENABLE_MYSQL AND USE_INTERNAL_MYSQL_LIBRARY)
|
||||
@ -109,26 +109,9 @@ if (ENABLE_MYSQL AND USE_INTERNAL_MYSQL_LIBRARY)
|
||||
endif ()
|
||||
|
||||
if (USE_INTERNAL_RDKAFKA_LIBRARY)
|
||||
set (RDKAFKA_BUILD_EXAMPLES OFF CACHE INTERNAL "")
|
||||
set (RDKAFKA_BUILD_TESTS OFF CACHE INTERNAL "")
|
||||
set (RDKAFKA_BUILD_STATIC ${MAKE_STATIC_LIBRARIES} CACHE INTERNAL "")
|
||||
mark_as_advanced (ZLIB_INCLUDE_DIR)
|
||||
|
||||
if (USE_INTERNAL_SSL_LIBRARY)
|
||||
if (MAKE_STATIC_LIBRARIES)
|
||||
add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY})
|
||||
set (WITH_BUNDLED_SSL 1 CACHE INTERNAL "")
|
||||
else ()
|
||||
set (WITH_SSL 0 CACHE INTERNAL "")
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
add_subdirectory (librdkafka)
|
||||
|
||||
if (USE_INTERNAL_SSL_LIBRARY AND MAKE_STATIC_LIBRARIES)
|
||||
target_include_directories(rdkafka PRIVATE BEFORE ${OPENSSL_INCLUDE_DIR})
|
||||
endif ()
|
||||
add_subdirectory (librdkafka-cmake)
|
||||
target_include_directories(rdkafka PRIVATE BEFORE ${ZLIB_INCLUDE_DIR})
|
||||
target_include_directories(rdkafka PRIVATE BEFORE ${OPENSSL_INCLUDE_DIR})
|
||||
endif ()
|
||||
|
||||
if (ENABLE_ODBC AND USE_INTERNAL_ODBC_LIBRARY)
|
||||
@ -162,7 +145,7 @@ if (USE_INTERNAL_POCO_LIBRARY)
|
||||
|
||||
if (OPENSSL_FOUND AND TARGET Crypto AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL))
|
||||
# Bug in poco https://github.com/pocoproject/poco/pull/2100 found on macos
|
||||
target_include_directories(Crypto PUBLIC ${OPENSSL_INCLUDE_DIR})
|
||||
target_include_directories(Crypto SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
|
@ -42,9 +42,9 @@ ${LIBRARY_DIR}/libs/filesystem/src/windows_file_codecvt.cpp)
|
||||
add_library(boost_system_internal
|
||||
${LIBRARY_DIR}/libs/system/src/error_code.cpp)
|
||||
|
||||
target_include_directories (boost_program_options_internal BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
|
||||
target_include_directories (boost_filesystem_internal BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
|
||||
target_include_directories (boost_system_internal BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
|
||||
target_include_directories (boost_program_options_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
|
||||
target_include_directories (boost_filesystem_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
|
||||
target_include_directories (boost_system_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
|
||||
|
||||
target_compile_definitions (boost_program_options_internal PUBLIC BOOST_SYSTEM_NO_DEPRECATED)
|
||||
target_compile_definitions (boost_filesystem_internal PUBLIC BOOST_SYSTEM_NO_DEPRECATED)
|
||||
|
@ -17,4 +17,4 @@ include/libcpuid/recog_amd.h
|
||||
include/libcpuid/recog_intel.h
|
||||
)
|
||||
|
||||
target_include_directories (cpuid PUBLIC include)
|
||||
target_include_directories (cpuid SYSTEM PUBLIC include)
|
||||
|
60
contrib/librdkafka-cmake/CMakeLists.txt
Normal file
60
contrib/librdkafka-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,60 @@
|
||||
set(RDKAFKA_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/librdkafka/src)
|
||||
|
||||
set(SRCS
|
||||
${RDKAFKA_SOURCE_DIR}/crc32c.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdaddr.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdavl.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdbuf.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdcrc32.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_assignor.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_broker.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_buf.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_cgrp.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_event.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_feature.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_lz4.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_metadata.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_metadata_cache.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_msg.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_msgset_reader.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_msgset_writer.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_offset.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_op.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_partition.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_pattern.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_queue.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_range_assignor.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_request.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_roundrobin_assignor.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_sasl.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_plain.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_subscription.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_timer.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_topic.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_transport.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_interceptor.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdkafka_header.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdlist.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdlog.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdmurmur2.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdports.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdrand.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdregex.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdstring.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdunittest.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdvarint.c
|
||||
${RDKAFKA_SOURCE_DIR}/snappy.c
|
||||
${RDKAFKA_SOURCE_DIR}/tinycthread.c
|
||||
${RDKAFKA_SOURCE_DIR}/xxhash.c
|
||||
${RDKAFKA_SOURCE_DIR}/lz4.c
|
||||
${RDKAFKA_SOURCE_DIR}/lz4frame.c
|
||||
${RDKAFKA_SOURCE_DIR}/lz4hc.c
|
||||
${RDKAFKA_SOURCE_DIR}/rdgz.c
|
||||
)
|
||||
|
||||
add_library(rdkafka STATIC ${SRCS})
|
||||
target_include_directories(rdkafka PRIVATE include)
|
||||
target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR})
|
||||
target_link_libraries(rdkafka PUBLIC ${ZLIB_LIBRARIES} ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
|
74
contrib/librdkafka-cmake/config.h
Normal file
74
contrib/librdkafka-cmake/config.h
Normal file
@ -0,0 +1,74 @@
|
||||
// Automatically generated by ./configure
|
||||
#ifndef _CONFIG_H_
|
||||
#define _CONFIG_H_
|
||||
#define ARCH "x86_64"
|
||||
#define CPU "generic"
|
||||
#define WITHOUT_OPTIMIZATION 0
|
||||
#define ENABLE_DEVEL 0
|
||||
#define ENABLE_VALGRIND 0
|
||||
#define ENABLE_REFCNT_DEBUG 0
|
||||
#define ENABLE_SHAREDPTR_DEBUG 0
|
||||
#define ENABLE_LZ4_EXT 1
|
||||
#define ENABLE_SSL 1
|
||||
//#define ENABLE_SASL 1
|
||||
#define MKL_APP_NAME "librdkafka"
|
||||
#define MKL_APP_DESC_ONELINE "The Apache Kafka C/C++ library"
|
||||
// distro
|
||||
//#define SOLIB_EXT ".so"
|
||||
// gcc
|
||||
//#define WITH_GCC 1
|
||||
// gxx
|
||||
//#define WITH_GXX 1
|
||||
// pkgconfig
|
||||
//#define WITH_PKGCONFIG 1
|
||||
// install
|
||||
//#define WITH_INSTALL 1
|
||||
// PIC
|
||||
//#define HAVE_PIC 1
|
||||
// gnulib
|
||||
//#define WITH_GNULD 1
|
||||
// __atomic_32
|
||||
#define HAVE_ATOMICS_32 1
|
||||
// __atomic_32
|
||||
#define HAVE_ATOMICS_32_ATOMIC 1
|
||||
// atomic_32
|
||||
#define ATOMIC_OP32(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)
|
||||
// __atomic_64
|
||||
#define HAVE_ATOMICS_64 1
|
||||
// __atomic_64
|
||||
#define HAVE_ATOMICS_64_ATOMIC 1
|
||||
// atomic_64
|
||||
#define ATOMIC_OP64(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)
|
||||
// atomic_64
|
||||
#define ATOMIC_OP(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)
|
||||
// parseversion
|
||||
#define RDKAFKA_VERSION_STR "0.11.4"
|
||||
// parseversion
|
||||
#define MKL_APP_VERSION "0.11.4"
|
||||
// libdl
|
||||
//#define WITH_LIBDL 1
|
||||
// WITH_PLUGINS
|
||||
//#define WITH_PLUGINS 1
|
||||
// zlib
|
||||
#define WITH_ZLIB 1
|
||||
// WITH_SNAPPY
|
||||
#define WITH_SNAPPY 1
|
||||
// WITH_SOCKEM
|
||||
#define WITH_SOCKEM 1
|
||||
// libssl
|
||||
#define WITH_SSL 1
|
||||
// WITH_SASL_SCRAM
|
||||
//#define WITH_SASL_SCRAM 1
|
||||
// crc32chw
|
||||
#define WITH_CRC32C_HW 1
|
||||
// regex
|
||||
#define HAVE_REGEX 1
|
||||
// strndup
|
||||
#define HAVE_STRNDUP 1
|
||||
// strerror_r
|
||||
#define HAVE_STRERROR_R 1
|
||||
// pthread_setname_gnu
|
||||
#define HAVE_PTHREAD_SETNAME_GNU 1
|
||||
// python
|
||||
//#define HAVE_PYTHON 1
|
||||
#endif /* _CONFIG_H_ */
|
1
contrib/librdkafka-cmake/include/README
Normal file
1
contrib/librdkafka-cmake/include/README
Normal file
@ -0,0 +1 @@
|
||||
This directory is needed because rdkafka files have #include "../config.h"
|
@ -15,7 +15,7 @@
|
||||
#define MARIADB_VERSION_ID 100306
|
||||
#define MYSQL_VERSION_ID 100306
|
||||
#define MARIADB_PORT 3306
|
||||
#define MARIADB_UNIX_ADDR "/tmp/mysql.sock"
|
||||
#define MARIADB_UNIX_ADDR "/var/run/mysqld/mysqld.sock"
|
||||
#define MYSQL_CONFIG_NAME "my"
|
||||
|
||||
#define MARIADB_PACKAGE_VERSION "3.0.6"
|
||||
|
@ -1,5 +1,7 @@
|
||||
add_library(murmurhash
|
||||
src/murmurhash2.cpp
|
||||
include/murmurhash2.h)
|
||||
src/murmurhash3.cpp
|
||||
include/murmurhash2.h
|
||||
include/murmurhash3.h)
|
||||
|
||||
target_include_directories (murmurhash PUBLIC include)
|
||||
|
37
contrib/murmurhash/include/murmurhash3.h
Normal file
37
contrib/murmurhash/include/murmurhash3.h
Normal file
@ -0,0 +1,37 @@
|
||||
//-----------------------------------------------------------------------------
|
||||
// MurmurHash3 was written by Austin Appleby, and is placed in the public
|
||||
// domain. The author hereby disclaims copyright to this source code.
|
||||
|
||||
#ifndef _MURMURHASH3_H_
|
||||
#define _MURMURHASH3_H_
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Platform-specific functions and macros
|
||||
|
||||
// Microsoft Visual Studio
|
||||
|
||||
#if defined(_MSC_VER) && (_MSC_VER < 1600)
|
||||
|
||||
typedef unsigned char uint8_t;
|
||||
typedef unsigned int uint32_t;
|
||||
typedef unsigned __int64 uint64_t;
|
||||
|
||||
// Other compilers
|
||||
|
||||
#else // defined(_MSC_VER)
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#endif // !defined(_MSC_VER)
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
void MurmurHash3_x86_32 ( const void * key, int len, uint32_t seed, void * out );
|
||||
|
||||
void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out );
|
||||
|
||||
void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out );
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
#endif // _MURMURHASH3_H_
|
331
contrib/murmurhash/src/murmurhash3.cpp
Normal file
331
contrib/murmurhash/src/murmurhash3.cpp
Normal file
@ -0,0 +1,331 @@
|
||||
// MurmurHash3 was written by Austin Appleby, and is placed in the public
|
||||
// domain. The author hereby disclaims copyright to this source code.
|
||||
|
||||
// Note - The x86 and x64 versions do _not_ produce the same results, as the
|
||||
// algorithms are optimized for their respective platforms. You can still
|
||||
// compile and run any of them on any platform, but your performance with the
|
||||
// non-native version will be less than optimal.
|
||||
|
||||
#include "murmurhash3.h"
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Platform-specific functions and macros
|
||||
|
||||
// Microsoft Visual Studio
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
|
||||
#define FORCE_INLINE __forceinline
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
#define ROTL32(x,y) _rotl(x,y)
|
||||
#define ROTL64(x,y) _rotl64(x,y)
|
||||
|
||||
#define BIG_CONSTANT(x) (x)
|
||||
|
||||
// Other compilers
|
||||
|
||||
#else // defined(_MSC_VER)
|
||||
|
||||
#define FORCE_INLINE inline __attribute__((always_inline))
|
||||
|
||||
inline uint32_t rotl32 ( uint32_t x, int8_t r )
|
||||
{
|
||||
return (x << r) | (x >> (32 - r));
|
||||
}
|
||||
|
||||
inline uint64_t rotl64 ( uint64_t x, int8_t r )
|
||||
{
|
||||
return (x << r) | (x >> (64 - r));
|
||||
}
|
||||
|
||||
#define ROTL32(x,y) rotl32(x,y)
|
||||
#define ROTL64(x,y) rotl64(x,y)
|
||||
|
||||
#define BIG_CONSTANT(x) (x##LLU)
|
||||
|
||||
#endif // !defined(_MSC_VER)
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Block read - if your platform needs to do endian-swapping or can only
|
||||
// handle aligned reads, do the conversion here
|
||||
|
||||
FORCE_INLINE uint32_t getblock32 ( const uint32_t * p, int i )
|
||||
{
|
||||
return p[i];
|
||||
}
|
||||
|
||||
FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, int i )
|
||||
{
|
||||
return p[i];
|
||||
}
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Finalization mix - force all bits of a hash block to avalanche
|
||||
|
||||
FORCE_INLINE uint32_t fmix32 ( uint32_t h )
|
||||
{
|
||||
h ^= h >> 16;
|
||||
h *= 0x85ebca6b;
|
||||
h ^= h >> 13;
|
||||
h *= 0xc2b2ae35;
|
||||
h ^= h >> 16;
|
||||
|
||||
return h;
|
||||
}
|
||||
|
||||
//----------
|
||||
|
||||
FORCE_INLINE uint64_t fmix64 ( uint64_t k )
|
||||
{
|
||||
k ^= k >> 33;
|
||||
k *= BIG_CONSTANT(0xff51afd7ed558ccd);
|
||||
k ^= k >> 33;
|
||||
k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53);
|
||||
k ^= k >> 33;
|
||||
|
||||
return k;
|
||||
}
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
void MurmurHash3_x86_32 ( const void * key, int len,
|
||||
uint32_t seed, void * out )
|
||||
{
|
||||
const uint8_t * data = (const uint8_t*)key;
|
||||
const int nblocks = len / 4;
|
||||
|
||||
uint32_t h1 = seed;
|
||||
|
||||
const uint32_t c1 = 0xcc9e2d51;
|
||||
const uint32_t c2 = 0x1b873593;
|
||||
|
||||
//----------
|
||||
// body
|
||||
|
||||
const uint32_t * blocks = (const uint32_t *)(data + nblocks*4);
|
||||
|
||||
for(int i = -nblocks; i; i++)
|
||||
{
|
||||
uint32_t k1 = getblock32(blocks,i);
|
||||
|
||||
k1 *= c1;
|
||||
k1 = ROTL32(k1,15);
|
||||
k1 *= c2;
|
||||
|
||||
h1 ^= k1;
|
||||
h1 = ROTL32(h1,13);
|
||||
h1 = h1*5+0xe6546b64;
|
||||
}
|
||||
|
||||
//----------
|
||||
// tail
|
||||
|
||||
const uint8_t * tail = (const uint8_t*)(data + nblocks*4);
|
||||
|
||||
uint32_t k1 = 0;
|
||||
|
||||
switch(len & 3)
|
||||
{
|
||||
case 3: k1 ^= tail[2] << 16;
|
||||
case 2: k1 ^= tail[1] << 8;
|
||||
case 1: k1 ^= tail[0];
|
||||
k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1;
|
||||
};
|
||||
|
||||
//----------
|
||||
// finalization
|
||||
|
||||
h1 ^= len;
|
||||
|
||||
h1 = fmix32(h1);
|
||||
|
||||
*(uint32_t*)out = h1;
|
||||
}
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
void MurmurHash3_x86_128 ( const void * key, const int len,
|
||||
uint32_t seed, void * out )
|
||||
{
|
||||
const uint8_t * data = (const uint8_t*)key;
|
||||
const int nblocks = len / 16;
|
||||
|
||||
uint32_t h1 = seed;
|
||||
uint32_t h2 = seed;
|
||||
uint32_t h3 = seed;
|
||||
uint32_t h4 = seed;
|
||||
|
||||
const uint32_t c1 = 0x239b961b;
|
||||
const uint32_t c2 = 0xab0e9789;
|
||||
const uint32_t c3 = 0x38b34ae5;
|
||||
const uint32_t c4 = 0xa1e38b93;
|
||||
|
||||
//----------
|
||||
// body
|
||||
|
||||
const uint32_t * blocks = (const uint32_t *)(data + nblocks*16);
|
||||
|
||||
for(int i = -nblocks; i; i++)
|
||||
{
|
||||
uint32_t k1 = getblock32(blocks,i*4+0);
|
||||
uint32_t k2 = getblock32(blocks,i*4+1);
|
||||
uint32_t k3 = getblock32(blocks,i*4+2);
|
||||
uint32_t k4 = getblock32(blocks,i*4+3);
|
||||
|
||||
k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1;
|
||||
|
||||
h1 = ROTL32(h1,19); h1 += h2; h1 = h1*5+0x561ccd1b;
|
||||
|
||||
k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2;
|
||||
|
||||
h2 = ROTL32(h2,17); h2 += h3; h2 = h2*5+0x0bcaa747;
|
||||
|
||||
k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3;
|
||||
|
||||
h3 = ROTL32(h3,15); h3 += h4; h3 = h3*5+0x96cd1c35;
|
||||
|
||||
k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4;
|
||||
|
||||
h4 = ROTL32(h4,13); h4 += h1; h4 = h4*5+0x32ac3b17;
|
||||
}
|
||||
|
||||
//----------
|
||||
// tail
|
||||
|
||||
const uint8_t * tail = (const uint8_t*)(data + nblocks*16);
|
||||
|
||||
uint32_t k1 = 0;
|
||||
uint32_t k2 = 0;
|
||||
uint32_t k3 = 0;
|
||||
uint32_t k4 = 0;
|
||||
|
||||
switch(len & 15)
|
||||
{
|
||||
case 15: k4 ^= tail[14] << 16;
|
||||
case 14: k4 ^= tail[13] << 8;
|
||||
case 13: k4 ^= tail[12] << 0;
|
||||
k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4;
|
||||
|
||||
case 12: k3 ^= tail[11] << 24;
|
||||
case 11: k3 ^= tail[10] << 16;
|
||||
case 10: k3 ^= tail[ 9] << 8;
|
||||
case 9: k3 ^= tail[ 8] << 0;
|
||||
k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3;
|
||||
|
||||
case 8: k2 ^= tail[ 7] << 24;
|
||||
case 7: k2 ^= tail[ 6] << 16;
|
||||
case 6: k2 ^= tail[ 5] << 8;
|
||||
case 5: k2 ^= tail[ 4] << 0;
|
||||
k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2;
|
||||
|
||||
case 4: k1 ^= tail[ 3] << 24;
|
||||
case 3: k1 ^= tail[ 2] << 16;
|
||||
case 2: k1 ^= tail[ 1] << 8;
|
||||
case 1: k1 ^= tail[ 0] << 0;
|
||||
k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1;
|
||||
};
|
||||
|
||||
//----------
|
||||
// finalization
|
||||
|
||||
h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len;
|
||||
|
||||
h1 += h2; h1 += h3; h1 += h4;
|
||||
h2 += h1; h3 += h1; h4 += h1;
|
||||
|
||||
h1 = fmix32(h1);
|
||||
h2 = fmix32(h2);
|
||||
h3 = fmix32(h3);
|
||||
h4 = fmix32(h4);
|
||||
|
||||
h1 += h2; h1 += h3; h1 += h4;
|
||||
h2 += h1; h3 += h1; h4 += h1;
|
||||
|
||||
((uint32_t*)out)[0] = h1;
|
||||
((uint32_t*)out)[1] = h2;
|
||||
((uint32_t*)out)[2] = h3;
|
||||
((uint32_t*)out)[3] = h4;
|
||||
}
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
void MurmurHash3_x64_128 ( const void * key, const int len,
|
||||
const uint32_t seed, void * out )
|
||||
{
|
||||
const uint8_t * data = (const uint8_t*)key;
|
||||
const int nblocks = len / 16;
|
||||
|
||||
uint64_t h1 = seed;
|
||||
uint64_t h2 = seed;
|
||||
|
||||
const uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5);
|
||||
const uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f);
|
||||
|
||||
//----------
|
||||
// body
|
||||
|
||||
const uint64_t * blocks = (const uint64_t *)(data);
|
||||
|
||||
for(int i = 0; i < nblocks; i++)
|
||||
{
|
||||
uint64_t k1 = getblock64(blocks,i*2+0);
|
||||
uint64_t k2 = getblock64(blocks,i*2+1);
|
||||
|
||||
k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; h1 ^= k1;
|
||||
|
||||
h1 = ROTL64(h1,27); h1 += h2; h1 = h1*5+0x52dce729;
|
||||
|
||||
k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; h2 ^= k2;
|
||||
|
||||
h2 = ROTL64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5;
|
||||
}
|
||||
|
||||
//----------
|
||||
// tail
|
||||
|
||||
const uint8_t * tail = (const uint8_t*)(data + nblocks*16);
|
||||
|
||||
uint64_t k1 = 0;
|
||||
uint64_t k2 = 0;
|
||||
|
||||
switch(len & 15)
|
||||
{
|
||||
case 15: k2 ^= ((uint64_t)tail[14]) << 48;
|
||||
case 14: k2 ^= ((uint64_t)tail[13]) << 40;
|
||||
case 13: k2 ^= ((uint64_t)tail[12]) << 32;
|
||||
case 12: k2 ^= ((uint64_t)tail[11]) << 24;
|
||||
case 11: k2 ^= ((uint64_t)tail[10]) << 16;
|
||||
case 10: k2 ^= ((uint64_t)tail[ 9]) << 8;
|
||||
case 9: k2 ^= ((uint64_t)tail[ 8]) << 0;
|
||||
k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; h2 ^= k2;
|
||||
|
||||
case 8: k1 ^= ((uint64_t)tail[ 7]) << 56;
|
||||
case 7: k1 ^= ((uint64_t)tail[ 6]) << 48;
|
||||
case 6: k1 ^= ((uint64_t)tail[ 5]) << 40;
|
||||
case 5: k1 ^= ((uint64_t)tail[ 4]) << 32;
|
||||
case 4: k1 ^= ((uint64_t)tail[ 3]) << 24;
|
||||
case 3: k1 ^= ((uint64_t)tail[ 2]) << 16;
|
||||
case 2: k1 ^= ((uint64_t)tail[ 1]) << 8;
|
||||
case 1: k1 ^= ((uint64_t)tail[ 0]) << 0;
|
||||
k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; h1 ^= k1;
|
||||
};
|
||||
|
||||
//----------
|
||||
// finalization
|
||||
|
||||
h1 ^= len; h2 ^= len;
|
||||
|
||||
h1 += h2;
|
||||
h2 += h1;
|
||||
|
||||
h1 = fmix64(h1);
|
||||
h2 = fmix64(h2);
|
||||
|
||||
h1 += h2;
|
||||
h2 += h1;
|
||||
|
||||
((uint64_t*)out)[0] = h1;
|
||||
((uint64_t*)out)[1] = h2;
|
||||
}
|
2
contrib/poco
vendored
2
contrib/poco
vendored
@ -1 +1 @@
|
||||
Subproject commit 4ab45bc3bb0d2c476ea5385ec2d398c6bfc9f089
|
||||
Subproject commit 3df947389e6d9654919002797bdd86ed190b3963
|
@ -12,7 +12,8 @@ endforeach ()
|
||||
|
||||
add_library (re2_st ${RE2_ST_SOURCES})
|
||||
target_compile_definitions (re2_st PRIVATE NDEBUG NO_THREADS re2=re2_st)
|
||||
target_include_directories (re2_st PRIVATE . PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${RE2_SOURCE_DIR})
|
||||
target_include_directories (re2_st PRIVATE .)
|
||||
target_include_directories (re2_st SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${RE2_SOURCE_DIR})
|
||||
|
||||
file (MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/re2_st)
|
||||
foreach (FILENAME filtered_re2.h re2.h set.h stringpiece.h)
|
||||
|
2
contrib/ssl
vendored
2
contrib/ssl
vendored
@ -1 +1 @@
|
||||
Subproject commit 4f9a7b8745184410dc0b31ba548ce21ac64edd9c
|
||||
Subproject commit de02224a42c69e3d8c9112c82018816f821878d0
|
2
contrib/zlib-ng
vendored
2
contrib/zlib-ng
vendored
@ -1 +1 @@
|
||||
Subproject commit e07a52dbaa35d003f5659b221b29d220c091667b
|
||||
Subproject commit 1b0ed32725e2c26d754e6fbc2e5e7490f8e8e711
|
@ -156,7 +156,6 @@ target_link_libraries (dbms
|
||||
${MYSQLXX_LIBRARY}
|
||||
${RE2_LIBRARY}
|
||||
${RE2_ST_LIBRARY}
|
||||
${OPENSSL_CRYPTO_LIBRARY}
|
||||
${BTRIE_LIBRARIES}
|
||||
)
|
||||
|
||||
@ -219,6 +218,8 @@ if (USE_RDKAFKA)
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
target_link_libraries(dbms ${OPENSSL_CRYPTO_LIBRARY})
|
||||
|
||||
target_link_libraries (dbms
|
||||
Threads::Threads
|
||||
)
|
||||
|
@ -1,11 +1,11 @@
|
||||
# This strings autochanged from release_lib.sh:
|
||||
set(VERSION_REVISION 54404 CACHE STRING "")
|
||||
set(VERSION_REVISION 54405 CACHE STRING "")
|
||||
set(VERSION_MAJOR 18 CACHE STRING "")
|
||||
set(VERSION_MINOR 9 CACHE STRING "")
|
||||
set(VERSION_PATCH 0 CACHE STRING "")
|
||||
set(VERSION_GITHASH c83721a02db002eef7ff864f82d53ca89d47f9e6 CACHE STRING "")
|
||||
set(VERSION_DESCRIBE v18.9.0-testing CACHE STRING "")
|
||||
set(VERSION_STRING 18.9.0 CACHE STRING "")
|
||||
set(VERSION_MINOR 10 CACHE STRING "")
|
||||
set(VERSION_PATCH 2 CACHE STRING "")
|
||||
set(VERSION_GITHASH 39bee180bd7c15dbb35244cc78387628345c1efe CACHE STRING "")
|
||||
set(VERSION_DESCRIBE v18.10.2-testing CACHE STRING "")
|
||||
set(VERSION_STRING 18.10.2 CACHE STRING "")
|
||||
# end of autochange
|
||||
|
||||
set(VERSION_EXTRA "" CACHE STRING "")
|
||||
|
@ -13,10 +13,10 @@ if (CLICKHOUSE_SPLIT_BINARY)
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
set(TMP_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/headers")
|
||||
set(TMP_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/${INTERNAL_COMPILER_HEADERS_RELATIVE}")
|
||||
# Make and install empty dir for debian package if compiler disabled
|
||||
add_custom_target(make-headers-directory ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${TMP_HEADERS_DIR})
|
||||
install(DIRECTORY ${TMP_HEADERS_DIR} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse COMPONENT clickhouse)
|
||||
install(DIRECTORY ${TMP_HEADERS_DIR} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse/${INTERNAL_COMPILER_HEADERS_DIR} COMPONENT clickhouse)
|
||||
# TODO: fix on macos copy_headers.sh: sed --posix
|
||||
if (USE_EMBEDDED_COMPILER AND NOT APPLE)
|
||||
add_custom_target(copy-headers ALL env CLANG=${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-clang BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} ${ClickHouse_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${TMP_HEADERS_DIR} DEPENDS clickhouse-clang WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES ${ClickHouse_SOURCE_DIR}/copy_headers.sh)
|
||||
|
@ -43,4 +43,7 @@ LLVMSupport
|
||||
#PollyPPCG
|
||||
|
||||
PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads
|
||||
${MALLOC_LIBRARIES}
|
||||
${GLIBC_COMPATIBILITY_LIBRARIES}
|
||||
${MEMCPY_LIBRARIES}
|
||||
)
|
||||
|
@ -43,4 +43,7 @@ ${REQUIRED_LLVM_LIBRARIES}
|
||||
#PollyPPCG
|
||||
|
||||
PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads
|
||||
${MALLOC_LIBRARIES}
|
||||
${GLIBC_COMPATIBILITY_LIBRARIES}
|
||||
${MEMCPY_LIBRARIES}
|
||||
)
|
||||
|
@ -39,4 +39,7 @@ lldCore
|
||||
${REQUIRED_LLVM_LIBRARIES}
|
||||
|
||||
PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads
|
||||
${MALLOC_LIBRARIES}
|
||||
${GLIBC_COMPATIBILITY_LIBRARIES}
|
||||
${MEMCPY_LIBRARIES}
|
||||
)
|
||||
|
@ -204,6 +204,7 @@ private:
|
||||
bool is_interactive = true; /// Use either readline interface or batch mode.
|
||||
bool need_render_progress = true; /// Render query execution progress.
|
||||
bool echo_queries = false; /// Print queries before execution in batch mode.
|
||||
bool ignore_error = false; /// In case of errors, don't print error message, continue to next query. Only applicable for non-interactive mode.
|
||||
bool print_time_to_stderr = false; /// Output execution time to stderr in batch mode.
|
||||
bool stdin_is_not_tty = false; /// stdin is not a terminal.
|
||||
|
||||
@ -474,6 +475,7 @@ private:
|
||||
{
|
||||
need_render_progress = config().getBool("progress", false);
|
||||
echo_queries = config().getBool("echo", false);
|
||||
ignore_error = config().getBool("ignore-error", false);
|
||||
}
|
||||
|
||||
connect();
|
||||
@ -765,7 +767,6 @@ private:
|
||||
|
||||
bool process(const String & text)
|
||||
{
|
||||
const auto ignore_error = config().getBool("ignore-error", false);
|
||||
const bool test_mode = config().has("testmode");
|
||||
if (config().has("multiquery"))
|
||||
{
|
||||
@ -821,9 +822,10 @@ private:
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
actual_client_error = getCurrentExceptionCode();
|
||||
if (!actual_client_error || actual_client_error != expected_client_error)
|
||||
std::cerr << "Error on processing query: " << query << std::endl << getCurrentExceptionMessage(true);
|
||||
last_exception = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
|
||||
actual_client_error = last_exception->code();
|
||||
if (!ignore_error && (!actual_client_error || actual_client_error != expected_client_error))
|
||||
std::cerr << "Error on processing query: " << query << std::endl << last_exception->message();
|
||||
got_exception = true;
|
||||
}
|
||||
|
||||
@ -1544,6 +1546,7 @@ public:
|
||||
("stacktrace", "print stack traces of exceptions")
|
||||
("progress", "print progress even in non-interactive mode")
|
||||
("version,V", "print version information and exit")
|
||||
("version-clean", "print version in machine-readable format and exit")
|
||||
("echo", "in batch mode, print query before execution")
|
||||
("max_client_network_bandwidth", boost::program_options::value<int>(), "the maximum speed of data exchange over the network for the client in bytes per second.")
|
||||
("compression", boost::program_options::value<bool>(), "enable or disable compression")
|
||||
@ -1573,6 +1576,12 @@ public:
|
||||
exit(0);
|
||||
}
|
||||
|
||||
if (options.count("version-clean"))
|
||||
{
|
||||
std::cout << VERSION_STRING;
|
||||
exit(0);
|
||||
}
|
||||
|
||||
/// Output of help message.
|
||||
if (options.count("help")
|
||||
|| (options.count("host") && options["host"].as<std::string>() == "elp")) /// If user writes -help instead of --help.
|
||||
|
@ -301,6 +301,9 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
if (config().has("max_table_size_to_drop"))
|
||||
global_context->setMaxTableSizeToDrop(config().getUInt64("max_table_size_to_drop"));
|
||||
|
||||
if (config().has("max_partition_size_to_drop"))
|
||||
global_context->setMaxPartitionSizeToDrop(config().getUInt64("max_partition_size_to_drop"));
|
||||
|
||||
/// Size of cache for uncompressed blocks. Zero means disabled.
|
||||
size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size", 0);
|
||||
if (uncompressed_cache_size)
|
||||
|
@ -322,10 +322,12 @@
|
||||
<!-- Protection from accidental DROP.
|
||||
If size of a MergeTree table is greater than max_table_size_to_drop (in bytes) than table could not be dropped with any DROP query.
|
||||
If you want do delete one table and don't want to restart clickhouse-server, you could create special file <clickhouse-path>/flags/force_drop_table and make DROP once.
|
||||
By default max_table_size_to_drop is 50GB, max_table_size_to_drop=0 allows to DROP any tables.
|
||||
By default max_table_size_to_drop is 50GB; max_table_size_to_drop=0 allows to DROP any tables.
|
||||
The same for max_partition_size_to_drop.
|
||||
Uncomment to disable protection.
|
||||
-->
|
||||
<!-- <max_table_size_to_drop>0</max_table_size_to_drop> -->
|
||||
<!-- <max_partition_size_to_drop>0</max_partition_size_to_drop> -->
|
||||
|
||||
<!-- Example of parameters for GraphiteMergeTree table engine -->
|
||||
<graphite_rollup_example>
|
||||
|
@ -226,6 +226,6 @@ ConnectionPoolWithFailover::tryGetEntry(
|
||||
}
|
||||
}
|
||||
return result;
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -3,10 +3,7 @@
|
||||
#include <Common/config.h>
|
||||
|
||||
#if USE_ICU
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wold-style-cast"
|
||||
#include <unicode/ucol.h>
|
||||
#pragma GCC diagnostic pop
|
||||
#else
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic push
|
||||
|
@ -1,17 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic push
|
||||
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
|
||||
#endif
|
||||
|
||||
#include <boost/smart_ptr/intrusive_ptr.hpp>
|
||||
#include <boost/smart_ptr/intrusive_ref_counter.hpp>
|
||||
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic pop
|
||||
#endif
|
||||
|
||||
#include <initializer_list>
|
||||
|
||||
|
||||
|
@ -37,7 +37,7 @@ namespace detail
|
||||
{
|
||||
MoveOrCopyIfThrow<T>()(std::forward<T>(src), dst);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** A very simple thread-safe queue of limited size.
|
||||
* If you try to pop an item from an empty queue, the thread is blocked until the queue becomes nonempty.
|
||||
|
@ -406,9 +406,6 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string &
|
||||
/// Add path_to_config/conf.d dir
|
||||
merge_dir_path.setBaseName("conf");
|
||||
merge_dirs.insert(merge_dir_path.toString());
|
||||
/// Add path_to_config/config.d dir
|
||||
merge_dir_path.setBaseName("config");
|
||||
merge_dirs.insert(merge_dir_path.toString());
|
||||
|
||||
for (const std::string & merge_dir_name : merge_dirs)
|
||||
{
|
||||
|
@ -380,6 +380,7 @@ namespace ErrorCodes
|
||||
extern const int INVALID_JOIN_ON_EXPRESSION = 403;
|
||||
extern const int BAD_ODBC_CONNECTION_STRING = 404;
|
||||
extern const int DECIMAL_OVERFLOW = 405;
|
||||
extern const int PARTITION_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT = 406;
|
||||
|
||||
extern const int KEEPER_EXCEPTION = 999;
|
||||
extern const int POCO_EXCEPTION = 1000;
|
||||
|
@ -130,11 +130,11 @@ int getCurrentExceptionCode()
|
||||
{
|
||||
return e.code();
|
||||
}
|
||||
catch (const Poco::Exception & e)
|
||||
catch (const Poco::Exception &)
|
||||
{
|
||||
return ErrorCodes::POCO_EXCEPTION;
|
||||
}
|
||||
catch (const std::exception & e)
|
||||
catch (const std::exception &)
|
||||
{
|
||||
return ErrorCodes::STD_EXCEPTION;
|
||||
}
|
||||
|
@ -95,13 +95,14 @@ void FileChecker::save() const
|
||||
/// So complex JSON structure - for compatibility with the old format.
|
||||
writeCString("{\"yandex\":{", out);
|
||||
|
||||
auto settings = FormatSettings();
|
||||
for (auto it = map.begin(); it != map.end(); ++it)
|
||||
{
|
||||
if (it != map.begin())
|
||||
writeString(",", out);
|
||||
|
||||
/// `escapeForFileName` is not really needed. But it is left for compatibility with the old code.
|
||||
writeJSONString(escapeForFileName(it->first), out);
|
||||
writeJSONString(escapeForFileName(it->first), out, settings);
|
||||
writeString(":{\"size\":\"", out);
|
||||
writeIntText(it->second, out);
|
||||
writeString("\"}", out);
|
||||
|
@ -74,7 +74,7 @@ bool check(const T x) { return x == 0; }
|
||||
template <typename T>
|
||||
void set(T & x) { x = 0; }
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/** Compile-time interface for cell of the hash table.
|
||||
|
@ -94,8 +94,8 @@ public:
|
||||
{
|
||||
if (auto it = aliases.find(name); it != aliases.end())
|
||||
return it->second;
|
||||
else if (auto it = case_insensitive_aliases.find(Poco::toLower(name)); it != case_insensitive_aliases.end())
|
||||
return it->second;
|
||||
else if (auto jt = case_insensitive_aliases.find(Poco::toLower(name)); jt != case_insensitive_aliases.end())
|
||||
return jt->second;
|
||||
|
||||
throw Exception(getFactoryName() + ": name '" + name + "' is not alias", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
@ -5,24 +5,9 @@
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wold-style-cast"
|
||||
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic push
|
||||
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
|
||||
#pragma clang diagnostic ignored "-Wreserved-id-macro"
|
||||
#endif
|
||||
|
||||
#include <boost/noncopyable.hpp>
|
||||
#include <boost/iterator_adaptors.hpp>
|
||||
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic pop
|
||||
#endif
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
#include <common/likely.h>
|
||||
#include <common/strong_typedef.h>
|
||||
|
||||
|
@ -340,4 +340,4 @@ private:
|
||||
size_t m_capacity;
|
||||
};
|
||||
|
||||
};
|
||||
}
|
||||
|
@ -50,4 +50,4 @@ private:
|
||||
static size_t getFailedOpIndex(int32_t code, const Responses & responses);
|
||||
};
|
||||
|
||||
};
|
||||
}
|
||||
|
@ -83,4 +83,4 @@ void ZooKeeperHolder::init(Args&&... args)
|
||||
|
||||
using ZooKeeperHolderPtr = std::shared_ptr<ZooKeeperHolder>;
|
||||
|
||||
};
|
||||
}
|
||||
|
@ -646,4 +646,4 @@ private:
|
||||
CurrentMetrics::Increment active_session_metric_increment{CurrentMetrics::ZooKeeperSession};
|
||||
};
|
||||
|
||||
};
|
||||
}
|
||||
|
@ -77,7 +77,10 @@ void formatIPv6(const unsigned char * src, char *& dst, UInt8 zeroed_tail_bytes_
|
||||
if (words[i] == 0)
|
||||
{
|
||||
if (cur.base == -1)
|
||||
cur.base = i, cur.len = 1;
|
||||
{
|
||||
cur.base = i;
|
||||
cur.len = 1;
|
||||
}
|
||||
else
|
||||
cur.len++;
|
||||
}
|
||||
|
@ -108,7 +108,7 @@ void localBackup(const Poco::Path & source_path, const Poco::Path & destination_
|
||||
|
||||
continue;
|
||||
}
|
||||
catch (const Poco::FileNotFoundException & e)
|
||||
catch (const Poco::FileNotFoundException &)
|
||||
{
|
||||
++try_no;
|
||||
if (try_no == max_tries)
|
||||
|
@ -60,7 +60,7 @@ add_executable (space_saving space_saving.cpp)
|
||||
target_link_libraries (space_saving clickhouse_common_io)
|
||||
|
||||
add_executable (integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp)
|
||||
target_include_directories (integer_hash_tables_and_hashes BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
|
||||
target_include_directories (integer_hash_tables_and_hashes SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
|
||||
target_link_libraries (integer_hash_tables_and_hashes clickhouse_common_io)
|
||||
|
||||
add_executable (allocator allocator.cpp)
|
||||
|
@ -25,7 +25,7 @@ namespace ErrorCodes
|
||||
class Field;
|
||||
using Array = std::vector<Field>;
|
||||
using TupleBackend = std::vector<Field>;
|
||||
STRONG_TYPEDEF(TupleBackend, Tuple); /// Array and Tuple are different types with equal representation inside Field.
|
||||
STRONG_TYPEDEF(TupleBackend, Tuple) /// Array and Tuple are different types with equal representation inside Field.
|
||||
|
||||
|
||||
class DecField
|
||||
|
@ -6,6 +6,6 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
STRONG_TYPEDEF(UInt128, UUID);
|
||||
STRONG_TYPEDEF(UInt128, UUID)
|
||||
|
||||
}
|
||||
|
@ -63,7 +63,7 @@ Block AggregatingSortedBlockInputStream::readImpl()
|
||||
for (size_t i = 0, size = columns_to_aggregate.size(); i < size; ++i)
|
||||
columns_to_aggregate[i] = typeid_cast<ColumnAggregateFunction *>(merged_columns[column_numbers_to_aggregate[i]].get());
|
||||
|
||||
merge(merged_columns, queue);
|
||||
merge(merged_columns, queue_without_collation);
|
||||
return header.cloneWithColumns(std::move(merged_columns));
|
||||
}
|
||||
|
||||
|
@ -40,60 +40,45 @@ void CollapsingSortedBlockInputStream::reportIncorrectData()
|
||||
}
|
||||
|
||||
|
||||
void CollapsingSortedBlockInputStream::insertRows(MutableColumns & merged_columns, size_t & merged_rows, bool last_in_stream)
|
||||
void CollapsingSortedBlockInputStream::insertRows(MutableColumns & merged_columns, size_t & merged_rows)
|
||||
{
|
||||
if (count_positive == 0 && count_negative == 0)
|
||||
{
|
||||
/// No input rows have been read.
|
||||
return;
|
||||
}
|
||||
|
||||
if (count_positive == count_negative && !last_is_positive)
|
||||
{
|
||||
/// If all the rows in the input streams was collapsed, we still want to give at least one block in the result.
|
||||
if (last_in_stream && merged_rows == 0 && !blocks_written)
|
||||
{
|
||||
LOG_INFO(log, "All rows collapsed");
|
||||
++merged_rows;
|
||||
for (size_t i = 0; i < num_columns; ++i)
|
||||
merged_columns[i]->insertFrom(*(*last_positive.columns)[i], last_positive.row_num);
|
||||
++merged_rows;
|
||||
for (size_t i = 0; i < num_columns; ++i)
|
||||
merged_columns[i]->insertFrom(*(*last_negative.columns)[i], last_negative.row_num);
|
||||
|
||||
if (out_row_sources_buf)
|
||||
{
|
||||
/// true flag value means "skip row"
|
||||
current_row_sources[last_positive_pos].setSkipFlag(false);
|
||||
current_row_sources[last_negative_pos].setSkipFlag(false);
|
||||
}
|
||||
}
|
||||
/// Input rows exactly cancel out.
|
||||
return;
|
||||
}
|
||||
else
|
||||
|
||||
if (count_positive <= count_negative)
|
||||
{
|
||||
if (count_positive <= count_negative)
|
||||
{
|
||||
++merged_rows;
|
||||
for (size_t i = 0; i < num_columns; ++i)
|
||||
merged_columns[i]->insertFrom(*(*first_negative.columns)[i], first_negative.row_num);
|
||||
++merged_rows;
|
||||
for (size_t i = 0; i < num_columns; ++i)
|
||||
merged_columns[i]->insertFrom(*(*first_negative.columns)[i], first_negative.row_num);
|
||||
|
||||
if (out_row_sources_buf)
|
||||
current_row_sources[first_negative_pos].setSkipFlag(false);
|
||||
}
|
||||
if (out_row_sources_buf)
|
||||
current_row_sources[first_negative_pos].setSkipFlag(false);
|
||||
}
|
||||
|
||||
if (count_positive >= count_negative)
|
||||
{
|
||||
++merged_rows;
|
||||
for (size_t i = 0; i < num_columns; ++i)
|
||||
merged_columns[i]->insertFrom(*(*last_positive.columns)[i], last_positive.row_num);
|
||||
if (count_positive >= count_negative)
|
||||
{
|
||||
++merged_rows;
|
||||
for (size_t i = 0; i < num_columns; ++i)
|
||||
merged_columns[i]->insertFrom(*(*last_positive.columns)[i], last_positive.row_num);
|
||||
|
||||
if (out_row_sources_buf)
|
||||
current_row_sources[last_positive_pos].setSkipFlag(false);
|
||||
}
|
||||
if (out_row_sources_buf)
|
||||
current_row_sources[last_positive_pos].setSkipFlag(false);
|
||||
}
|
||||
|
||||
if (!(count_positive == count_negative || count_positive + 1 == count_negative || count_positive == count_negative + 1))
|
||||
{
|
||||
if (count_incorrect_data < MAX_ERROR_MESSAGES)
|
||||
reportIncorrectData();
|
||||
++count_incorrect_data;
|
||||
}
|
||||
if (!(count_positive == count_negative || count_positive + 1 == count_negative || count_positive == count_negative + 1))
|
||||
{
|
||||
if (count_incorrect_data < MAX_ERROR_MESSAGES)
|
||||
reportIncorrectData();
|
||||
++count_incorrect_data;
|
||||
}
|
||||
|
||||
if (out_row_sources_buf)
|
||||
@ -117,7 +102,7 @@ Block CollapsingSortedBlockInputStream::readImpl()
|
||||
if (merged_columns.empty())
|
||||
return {};
|
||||
|
||||
merge(merged_columns, queue);
|
||||
merge(merged_columns, queue_without_collation);
|
||||
return header.cloneWithColumns(std::move(merged_columns));
|
||||
}
|
||||
|
||||
@ -211,7 +196,7 @@ void CollapsingSortedBlockInputStream::merge(MutableColumns & merged_columns, st
|
||||
}
|
||||
|
||||
/// Write data for last primary key.
|
||||
insertRows(merged_columns, merged_rows, true);
|
||||
insertRows(merged_columns, merged_rows);
|
||||
|
||||
finished = true;
|
||||
}
|
||||
|
@ -74,7 +74,7 @@ private:
|
||||
void merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue);
|
||||
|
||||
/// Output to result rows for the current primary key.
|
||||
void insertRows(MutableColumns & merged_columns, size_t & merged_rows, bool last_in_stream = false);
|
||||
void insertRows(MutableColumns & merged_columns, size_t & merged_rows);
|
||||
|
||||
void reportIncorrectData();
|
||||
};
|
||||
|
@ -102,7 +102,7 @@ Block GraphiteRollupSortedBlockInputStream::readImpl()
|
||||
if (merged_columns.empty())
|
||||
return Block();
|
||||
|
||||
merge(merged_columns, queue);
|
||||
merge(merged_columns, queue_without_collation);
|
||||
return header.cloneWithColumns(std::move(merged_columns));
|
||||
}
|
||||
|
||||
|
@ -188,7 +188,7 @@ static bool handleOverflowMode(OverflowMode mode, const String & message, int co
|
||||
default:
|
||||
throw Exception("Logical error: unknown overflow mode", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
bool IProfilingBlockInputStream::checkTimeLimit()
|
||||
|
@ -183,7 +183,7 @@ MergeSortingBlocksBlockInputStream::MergeSortingBlocksBlockInputStream(
|
||||
if (!has_collation)
|
||||
{
|
||||
for (size_t i = 0; i < cursors.size(); ++i)
|
||||
queue.push(SortCursor(&cursors[i]));
|
||||
queue_without_collation.push(SortCursor(&cursors[i]));
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -206,7 +206,7 @@ Block MergeSortingBlocksBlockInputStream::readImpl()
|
||||
}
|
||||
|
||||
return !has_collation
|
||||
? mergeImpl<SortCursor>(queue)
|
||||
? mergeImpl<SortCursor>(queue_without_collation)
|
||||
: mergeImpl<SortCursorWithCollation>(queue_with_collation);
|
||||
}
|
||||
|
||||
|
@ -55,7 +55,7 @@ private:
|
||||
|
||||
bool has_collation = false;
|
||||
|
||||
std::priority_queue<SortCursor> queue;
|
||||
std::priority_queue<SortCursor> queue_without_collation;
|
||||
std::priority_queue<SortCursorWithCollation> queue_with_collation;
|
||||
|
||||
/** Two different cursors are supported - with and without Collation.
|
||||
|
@ -320,7 +320,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(MemoryTracker
|
||||
* - or, if no next blocks, set 'exhausted' flag.
|
||||
*/
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(parallel_merge_data->get_next_blocks_mutex);
|
||||
std::lock_guard<std::mutex> lock_next_blocks(parallel_merge_data->get_next_blocks_mutex);
|
||||
|
||||
if (parallel_merge_data->exhausted || parallel_merge_data->finish)
|
||||
break;
|
||||
@ -330,7 +330,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(MemoryTracker
|
||||
if (!blocks_to_merge || blocks_to_merge->empty())
|
||||
{
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(parallel_merge_data->merged_blocks_mutex);
|
||||
std::unique_lock<std::mutex> lock_merged_blocks(parallel_merge_data->merged_blocks_mutex);
|
||||
parallel_merge_data->exhausted = true;
|
||||
}
|
||||
|
||||
@ -344,9 +344,9 @@ void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(MemoryTracker
|
||||
: blocks_to_merge->front().info.bucket_num;
|
||||
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(parallel_merge_data->merged_blocks_mutex);
|
||||
std::unique_lock<std::mutex> lock_merged_blocks(parallel_merge_data->merged_blocks_mutex);
|
||||
|
||||
parallel_merge_data->have_space.wait(lock, [this]
|
||||
parallel_merge_data->have_space.wait(lock_merged_blocks, [this]
|
||||
{
|
||||
return parallel_merge_data->merged_blocks.size() < merging_threads
|
||||
|| parallel_merge_data->finish;
|
||||
|
@ -58,7 +58,7 @@ void MergingSortedBlockInputStream::init(MutableColumns & merged_columns)
|
||||
if (has_collation)
|
||||
initQueue(queue_with_collation);
|
||||
else
|
||||
initQueue(queue);
|
||||
initQueue(queue_without_collation);
|
||||
}
|
||||
|
||||
/// Let's check that all source blocks have the same structure.
|
||||
@ -105,7 +105,7 @@ Block MergingSortedBlockInputStream::readImpl()
|
||||
if (has_collation)
|
||||
merge(merged_columns, queue_with_collation);
|
||||
else
|
||||
merge(merged_columns, queue);
|
||||
merge(merged_columns, queue_without_collation);
|
||||
|
||||
return header.cloneWithColumns(std::move(merged_columns));
|
||||
}
|
||||
@ -200,7 +200,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::
|
||||
|
||||
// std::cerr << "copied columns\n";
|
||||
|
||||
size_t merged_rows = merged_columns.at(0)->size();
|
||||
merged_rows = merged_columns.at(0)->size();
|
||||
|
||||
if (limit && total_merged_rows + merged_rows > limit)
|
||||
{
|
||||
|
@ -2,17 +2,8 @@
|
||||
|
||||
#include <queue>
|
||||
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic push
|
||||
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
|
||||
#endif
|
||||
|
||||
#include <boost/smart_ptr/intrusive_ptr.hpp>
|
||||
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic pop
|
||||
#endif
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
#include <Core/Row.h>
|
||||
@ -161,7 +152,7 @@ protected:
|
||||
CursorImpls cursors;
|
||||
|
||||
using Queue = std::priority_queue<SortCursor>;
|
||||
Queue queue;
|
||||
Queue queue_without_collation;
|
||||
|
||||
using QueueWithCollation = std::priority_queue<SortCursorWithCollation>;
|
||||
QueueWithCollation queue_with_collation;
|
||||
|
@ -58,21 +58,21 @@ RemoteBlockInputStream::RemoteBlockInputStream(
|
||||
|
||||
create_multiplexed_connections = [this, pool, throttler]()
|
||||
{
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
const Settings & current_settings = context.getSettingsRef();
|
||||
|
||||
std::vector<IConnectionPool::Entry> connections;
|
||||
if (main_table)
|
||||
{
|
||||
auto try_results = pool->getManyChecked(&settings, pool_mode, *main_table);
|
||||
auto try_results = pool->getManyChecked(¤t_settings, pool_mode, *main_table);
|
||||
connections.reserve(try_results.size());
|
||||
for (auto & try_result : try_results)
|
||||
connections.emplace_back(std::move(try_result.entry));
|
||||
}
|
||||
else
|
||||
connections = pool->getMany(&settings, pool_mode);
|
||||
connections = pool->getMany(¤t_settings, pool_mode);
|
||||
|
||||
return std::make_unique<MultiplexedConnections>(
|
||||
std::move(connections), settings, throttler, append_extra_info);
|
||||
std::move(connections), current_settings, throttler, append_extra_info);
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -44,7 +44,7 @@ Block ReplacingSortedBlockInputStream::readImpl()
|
||||
if (merged_columns.empty())
|
||||
return Block();
|
||||
|
||||
merge(merged_columns, queue);
|
||||
merge(merged_columns, queue_without_collation);
|
||||
return header.cloneWithColumns(std::move(merged_columns));
|
||||
}
|
||||
|
||||
|
@ -195,7 +195,7 @@ SummingSortedBlockInputStream::SummingSortedBlockInputStream(
|
||||
}
|
||||
|
||||
|
||||
void SummingSortedBlockInputStream::insertCurrentRowIfNeeded(MutableColumns & merged_columns, bool force_insertion)
|
||||
void SummingSortedBlockInputStream::insertCurrentRowIfNeeded(MutableColumns & merged_columns)
|
||||
{
|
||||
for (auto & desc : columns_to_aggregate)
|
||||
{
|
||||
@ -237,9 +237,9 @@ void SummingSortedBlockInputStream::insertCurrentRowIfNeeded(MutableColumns & me
|
||||
desc.merged_column->insertDefault();
|
||||
}
|
||||
|
||||
/// If it is "zero" row and it is not the last row of the result block, then
|
||||
/// rollback the insertion (at this moment we need rollback only cols from columns_to_aggregate)
|
||||
if (!force_insertion && current_row_is_zero)
|
||||
/// If it is "zero" row, then rollback the insertion
|
||||
/// (at this moment we need rollback only cols from columns_to_aggregate)
|
||||
if (current_row_is_zero)
|
||||
{
|
||||
for (auto & desc : columns_to_aggregate)
|
||||
desc.merged_column->popBack(1);
|
||||
@ -252,7 +252,6 @@ void SummingSortedBlockInputStream::insertCurrentRowIfNeeded(MutableColumns & me
|
||||
|
||||
/// Update per-block and per-group flags
|
||||
++merged_rows;
|
||||
output_is_non_empty = true;
|
||||
}
|
||||
|
||||
|
||||
@ -287,7 +286,7 @@ Block SummingSortedBlockInputStream::readImpl()
|
||||
desc.merged_column = header.safeGetByPosition(desc.column_numbers[0]).column->cloneEmpty();
|
||||
}
|
||||
|
||||
merge(merged_columns, queue);
|
||||
merge(merged_columns, queue_without_collation);
|
||||
Block res = header.cloneWithColumns(std::move(merged_columns));
|
||||
|
||||
/// Place aggregation results into block.
|
||||
@ -333,7 +332,7 @@ void SummingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::
|
||||
{
|
||||
if (!current_key.empty())
|
||||
/// Write the data for the previous group.
|
||||
insertCurrentRowIfNeeded(merged_columns, false);
|
||||
insertCurrentRowIfNeeded(merged_columns);
|
||||
|
||||
if (merged_rows >= max_block_size)
|
||||
{
|
||||
@ -393,7 +392,7 @@ void SummingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::
|
||||
|
||||
/// We will write the data for the last group, if it is non-zero.
|
||||
/// If it is zero, and without it the output stream will be empty, we will write it anyway.
|
||||
insertCurrentRowIfNeeded(merged_columns, !output_is_non_empty);
|
||||
insertCurrentRowIfNeeded(merged_columns);
|
||||
finished = true;
|
||||
}
|
||||
|
||||
|
@ -134,7 +134,6 @@ private:
|
||||
Row current_row;
|
||||
bool current_row_is_zero = true; /// Are all summed columns zero (or empty)? It is updated incrementally.
|
||||
|
||||
bool output_is_non_empty = false; /// Have we given out at least one row as a result.
|
||||
size_t merged_rows = 0; /// Number of rows merged into current result block
|
||||
|
||||
/** We support two different cursors - with Collation and without.
|
||||
@ -143,8 +142,7 @@ private:
|
||||
void merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue);
|
||||
|
||||
/// Insert the summed row for the current group into the result and updates some of per-block flags if the row is not "zero".
|
||||
/// If force_insertion=true, then the row will be inserted even if it is "zero"
|
||||
void insertCurrentRowIfNeeded(MutableColumns & merged_columns, bool force_insertion);
|
||||
void insertCurrentRowIfNeeded(MutableColumns & merged_columns);
|
||||
|
||||
/// Returns true if merge result is not empty
|
||||
bool mergeMap(const MapDescription & map, Row & row, SortCursor & cursor);
|
||||
|
@ -15,11 +15,11 @@ namespace ErrorCodes
|
||||
|
||||
VersionedCollapsingSortedBlockInputStream::VersionedCollapsingSortedBlockInputStream(
|
||||
const BlockInputStreams & inputs_, const SortDescription & description_,
|
||||
const String & sign_column_, size_t max_block_size_, bool can_collapse_all_rows_,
|
||||
const String & sign_column_, size_t max_block_size_,
|
||||
WriteBuffer * out_row_sources_buf_)
|
||||
: MergingSortedBlockInputStream(inputs_, description_, max_block_size_, 0, out_row_sources_buf_)
|
||||
, max_rows_in_queue(std::min(std::max<size_t>(3, max_block_size_), MAX_ROWS_IN_MULTIVERSION_QUEUE) - 2)
|
||||
, current_keys(max_rows_in_queue + 1), can_collapse_all_rows(can_collapse_all_rows_)
|
||||
, current_keys(max_rows_in_queue + 1)
|
||||
{
|
||||
sign_column_number = header.getPositionByName(sign_column_);
|
||||
}
|
||||
@ -76,7 +76,7 @@ Block VersionedCollapsingSortedBlockInputStream::readImpl()
|
||||
if (merged_columns.empty())
|
||||
return {};
|
||||
|
||||
merge(merged_columns, queue);
|
||||
merge(merged_columns, queue_without_collation);
|
||||
return header.cloneWithColumns(std::move(merged_columns));
|
||||
}
|
||||
|
||||
@ -130,10 +130,7 @@ void VersionedCollapsingSortedBlockInputStream::merge(MutableColumns & merged_co
|
||||
{
|
||||
update_queue(current);
|
||||
|
||||
/// If all the rows was collapsed, we still want to give at least one block in the result.
|
||||
/// If queue is empty then don't collapse two last rows.
|
||||
if (sign == sign_in_queue || (!can_collapse_all_rows && blocks_written == 0
|
||||
&& merged_rows == 0 && queue.empty() && current_keys.size() == 1))
|
||||
if (sign == sign_in_queue)
|
||||
current_keys.pushBack(next_key);
|
||||
else
|
||||
{
|
||||
|
@ -176,7 +176,7 @@ public:
|
||||
/// max_rows_in_queue should be about max_block_size_ if we won't store a lot of extra blocks (RowRef holds SharedBlockPtr).
|
||||
VersionedCollapsingSortedBlockInputStream(
|
||||
const BlockInputStreams & inputs_, const SortDescription & description_,
|
||||
const String & sign_column_, size_t max_block_size_, bool can_collapse_all_rows_,
|
||||
const String & sign_column_, size_t max_block_size_,
|
||||
WriteBuffer * out_row_sources_buf_ = nullptr);
|
||||
|
||||
String getName() const override { return "VersionedCollapsingSorted"; }
|
||||
@ -203,8 +203,6 @@ private:
|
||||
/// Sources of rows for VERTICAL merge algorithm. Size equals to (size + number of gaps) in current_keys.
|
||||
std::queue<RowSourcePart> current_row_sources;
|
||||
|
||||
const bool can_collapse_all_rows;
|
||||
|
||||
void merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue);
|
||||
|
||||
/// Output to result row for the current primary key.
|
||||
|
@ -212,9 +212,9 @@ void DataTypeAggregateFunction::deserializeTextQuoted(IColumn & column, ReadBuff
|
||||
}
|
||||
|
||||
|
||||
void DataTypeAggregateFunction::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
|
||||
void DataTypeAggregateFunction::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
writeJSONString(serializeToString(function, column, row_num), ostr);
|
||||
writeJSONString(serializeToString(function, column, row_num), ostr, settings);
|
||||
}
|
||||
|
||||
|
||||
|
@ -97,7 +97,7 @@ DataTypeEnum<Type>::DataTypeEnum(const Values & values_) : values{values_}
|
||||
});
|
||||
|
||||
fillMaps();
|
||||
name = generateName(values);
|
||||
type_name = generateName(values);
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
@ -145,9 +145,9 @@ template <typename Type>
|
||||
void DataTypeEnum<Type>::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
|
||||
{
|
||||
/// NOTE It would be nice to do without creating a temporary object - at least extract std::string out.
|
||||
std::string name;
|
||||
readEscapedString(name, istr);
|
||||
static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(name)));
|
||||
std::string field_name;
|
||||
readEscapedString(field_name, istr);
|
||||
static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(field_name)));
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
@ -159,15 +159,15 @@ void DataTypeEnum<Type>::serializeTextQuoted(const IColumn & column, size_t row_
|
||||
template <typename Type>
|
||||
void DataTypeEnum<Type>::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
|
||||
{
|
||||
std::string name;
|
||||
readQuotedStringWithSQLStyle(name, istr);
|
||||
static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(name)));
|
||||
std::string field_name;
|
||||
readQuotedStringWithSQLStyle(field_name, istr);
|
||||
static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(field_name)));
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
void DataTypeEnum<Type>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
|
||||
void DataTypeEnum<Type>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
writeJSONString(getNameForValue(static_cast<const ColumnType &>(column).getData()[row_num]), ostr);
|
||||
writeJSONString(getNameForValue(static_cast<const ColumnType &>(column).getData()[row_num]), ostr, settings);
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
@ -179,9 +179,9 @@ void DataTypeEnum<Type>::serializeTextXML(const IColumn & column, size_t row_num
|
||||
template <typename Type>
|
||||
void DataTypeEnum<Type>::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
|
||||
{
|
||||
std::string name;
|
||||
readJSONString(name, istr);
|
||||
static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(name)));
|
||||
std::string field_name;
|
||||
readJSONString(field_name, istr);
|
||||
static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(field_name)));
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
@ -193,9 +193,9 @@ void DataTypeEnum<Type>::serializeTextCSV(const IColumn & column, size_t row_num
|
||||
template <typename Type>
|
||||
void DataTypeEnum<Type>::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||
{
|
||||
std::string name;
|
||||
readCSVString(name, istr, settings.csv);
|
||||
static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(name)));
|
||||
std::string field_name;
|
||||
readCSVString(field_name, istr, settings.csv);
|
||||
static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(field_name)));
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
@ -237,7 +237,7 @@ void DataTypeEnum<Type>::insertDefaultInto(IColumn & column) const
|
||||
template <typename Type>
|
||||
bool DataTypeEnum<Type>::equals(const IDataType & rhs) const
|
||||
{
|
||||
return typeid(rhs) == typeid(*this) && name == static_cast<const DataTypeEnum<Type> &>(rhs).name;
|
||||
return typeid(rhs) == typeid(*this) && type_name == static_cast<const DataTypeEnum<Type> &>(rhs).type_name;
|
||||
}
|
||||
|
||||
|
||||
@ -346,14 +346,14 @@ static DataTypePtr create(const ASTPtr & arguments)
|
||||
throw Exception("Elements of Enum data type must be of form: 'name' = number, where name is string literal and number is an integer",
|
||||
ErrorCodes::UNEXPECTED_AST_STRUCTURE);
|
||||
|
||||
const String & name = name_literal->value.get<String>();
|
||||
const String & field_name = name_literal->value.get<String>();
|
||||
const auto value = value_literal->value.get<typename NearestFieldType<FieldType>::Type>();
|
||||
|
||||
if (value > std::numeric_limits<FieldType>::max() || value < std::numeric_limits<FieldType>::min())
|
||||
throw Exception{"Value " + toString(value) + " for element '" + name + "' exceeds range of " + EnumName<FieldType>::value,
|
||||
throw Exception{"Value " + toString(value) + " for element '" + field_name + "' exceeds range of " + EnumName<FieldType>::value,
|
||||
ErrorCodes::ARGUMENT_OUT_OF_BOUND};
|
||||
|
||||
values.emplace_back(name, value);
|
||||
values.emplace_back(field_name, value);
|
||||
}
|
||||
|
||||
return std::make_shared<DataTypeEnum>(values);
|
||||
|
@ -53,7 +53,7 @@ private:
|
||||
Values values;
|
||||
NameToValueMap name_to_value_map;
|
||||
ValueToNameMap value_to_name_map;
|
||||
std::string name;
|
||||
std::string type_name;
|
||||
|
||||
static std::string generateName(const Values & values);
|
||||
void fillMaps();
|
||||
@ -62,7 +62,7 @@ public:
|
||||
explicit DataTypeEnum(const Values & values_);
|
||||
|
||||
const Values & getValues() const { return values; }
|
||||
std::string getName() const override { return name; }
|
||||
std::string getName() const override { return type_name; }
|
||||
const char * getFamilyName() const override;
|
||||
|
||||
const StringRef & getNameForValue(const FieldType & value) const
|
||||
@ -74,11 +74,11 @@ public:
|
||||
return it->second;
|
||||
}
|
||||
|
||||
FieldType getValue(StringRef name) const
|
||||
FieldType getValue(StringRef field_name) const
|
||||
{
|
||||
const auto it = name_to_value_map.find(name);
|
||||
const auto it = name_to_value_map.find(field_name);
|
||||
if (it == std::end(name_to_value_map))
|
||||
throw Exception{"Unknown element '" + name.toString() + "' for type " + getName(), ErrorCodes::LOGICAL_ERROR};
|
||||
throw Exception{"Unknown element '" + field_name.toString() + "' for type " + getName(), ErrorCodes::LOGICAL_ERROR};
|
||||
|
||||
return it->second;
|
||||
}
|
||||
|
@ -168,10 +168,10 @@ void DataTypeFixedString::deserializeTextQuoted(IColumn & column, ReadBuffer & i
|
||||
}
|
||||
|
||||
|
||||
void DataTypeFixedString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
|
||||
void DataTypeFixedString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
const char * pos = reinterpret_cast<const char *>(&static_cast<const ColumnFixedString &>(column).getChars()[n * row_num]);
|
||||
writeJSONString(pos, pos + n, ostr);
|
||||
writeJSONString(pos, pos + n, ostr, settings);
|
||||
}
|
||||
|
||||
|
||||
|
@ -262,9 +262,9 @@ void DataTypeString::deserializeTextQuoted(IColumn & column, ReadBuffer & istr,
|
||||
}
|
||||
|
||||
|
||||
void DataTypeString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
|
||||
void DataTypeString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
writeJSONString(static_cast<const ColumnString &>(column).getDataAt(row_num), ostr);
|
||||
writeJSONString(static_cast<const ColumnString &>(column).getDataAt(row_num), ostr, settings);
|
||||
}
|
||||
|
||||
|
||||
|
@ -22,6 +22,6 @@ namespace Nested
|
||||
|
||||
/// Collect Array columns in a form of `column_name.element_name` to single Array(Tuple(...)) column.
|
||||
NamesAndTypesList collect(const NamesAndTypesList & names_and_types);
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -39,15 +39,15 @@ Tables DatabaseDictionary::loadTables()
|
||||
Tables tables;
|
||||
for (const auto & pair : dictionaries)
|
||||
{
|
||||
const std::string & name = pair.first;
|
||||
if (deleted_tables.count(name))
|
||||
const std::string & dict_name = pair.first;
|
||||
if (deleted_tables.count(dict_name))
|
||||
continue;
|
||||
auto dict_ptr = std::static_pointer_cast<IDictionaryBase>(pair.second.loadable);
|
||||
if (dict_ptr)
|
||||
{
|
||||
const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
|
||||
auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
|
||||
tables[name] = StorageDictionary::create(name, ColumnsDescription{columns}, dictionary_structure, name);
|
||||
tables[dict_name] = StorageDictionary::create(dict_name, ColumnsDescription{columns}, dictionary_structure, dict_name);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -101,12 +101,12 @@ void CacheDictionary::isInImpl(
|
||||
{
|
||||
/// Transform all children to parents until ancestor id or null_value will be reached.
|
||||
|
||||
size_t size = out.size();
|
||||
memset(out.data(), 0xFF, size); /// 0xFF means "not calculated"
|
||||
size_t out_size = out.size();
|
||||
memset(out.data(), 0xFF, out_size); /// 0xFF means "not calculated"
|
||||
|
||||
const auto null_value = std::get<UInt64>(hierarchical_attribute->null_values);
|
||||
|
||||
PaddedPODArray<Key> children(size);
|
||||
PaddedPODArray<Key> children(out_size);
|
||||
PaddedPODArray<Key> parents(child_ids.begin(), child_ids.end());
|
||||
|
||||
while (true)
|
||||
@ -115,7 +115,7 @@ void CacheDictionary::isInImpl(
|
||||
size_t parents_idx = 0;
|
||||
size_t new_children_idx = 0;
|
||||
|
||||
while (out_idx < size)
|
||||
while (out_idx < out_size)
|
||||
{
|
||||
/// Already calculated
|
||||
if (out[out_idx] != 0xFF)
|
||||
@ -203,7 +203,7 @@ void CacheDictionary::isInConstantVector(
|
||||
}
|
||||
|
||||
/// Assuming short hierarchy, so linear search is Ok.
|
||||
for (size_t i = 0, size = out.size(); i < size; ++i)
|
||||
for (size_t i = 0, out_size = out.size(); i < out_size; ++i)
|
||||
out[i] = std::find(ancestors.begin(), ancestors.end(), ancestor_ids[i]) != ancestors.end();
|
||||
}
|
||||
|
||||
@ -936,12 +936,12 @@ void CacheDictionary::setAttributeValue(Attribute & attribute, const Key idx, co
|
||||
if (string_ref.data && string_ref.data != null_value_ref.data())
|
||||
string_arena->free(const_cast<char *>(string_ref.data), string_ref.size);
|
||||
|
||||
const auto size = string.size();
|
||||
if (size != 0)
|
||||
const auto str_size = string.size();
|
||||
if (str_size != 0)
|
||||
{
|
||||
auto string_ptr = string_arena->alloc(size + 1);
|
||||
std::copy(string.data(), string.data() + size + 1, string_ptr);
|
||||
string_ref = StringRef{string_ptr, size};
|
||||
auto string_ptr = string_arena->alloc(str_size + 1);
|
||||
std::copy(string.data(), string.data() + str_size + 1, string_ptr);
|
||||
string_ref = StringRef{string_ptr, str_size};
|
||||
}
|
||||
else
|
||||
string_ref = {};
|
||||
|
@ -307,13 +307,13 @@ private:
|
||||
|
||||
/// buffer[column_size * cat_features_count] -> char * => cat_features[column_size][cat_features_count] -> char *
|
||||
void fillCatFeaturesBuffer(const char *** cat_features, const char ** buffer,
|
||||
size_t column_size, size_t cat_features_count) const
|
||||
size_t column_size, size_t cat_features_count_current) const
|
||||
{
|
||||
for (size_t i = 0; i < column_size; ++i)
|
||||
{
|
||||
*cat_features = buffer;
|
||||
++cat_features;
|
||||
buffer += cat_features_count;
|
||||
buffer += cat_features_count_current;
|
||||
}
|
||||
}
|
||||
|
||||
@ -321,7 +321,7 @@ private:
|
||||
/// * CalcModelPredictionFlat if no cat features
|
||||
/// * CalcModelPrediction if all cat features are strings
|
||||
/// * CalcModelPredictionWithHashedCatFeatures if has int cat features.
|
||||
ColumnPtr evalImpl(const ColumnRawPtrs & columns, size_t float_features_count, size_t cat_features_count,
|
||||
ColumnPtr evalImpl(const ColumnRawPtrs & columns, size_t float_features_count_current, size_t cat_features_count_current,
|
||||
bool cat_features_are_strings) const
|
||||
{
|
||||
std::string error_msg = "Error occurred while applying CatBoost model: ";
|
||||
@ -334,12 +334,12 @@ private:
|
||||
PODArray<const float *> float_features(column_size);
|
||||
auto float_features_buf = float_features.data();
|
||||
/// Store all float data into single column. float_features is a list of pointers to it.
|
||||
auto float_features_col = placeNumericColumns<float>(columns, 0, float_features_count, float_features_buf);
|
||||
auto float_features_col = placeNumericColumns<float>(columns, 0, float_features_count_current, float_features_buf);
|
||||
|
||||
if (cat_features_count == 0)
|
||||
if (cat_features_count_current == 0)
|
||||
{
|
||||
if (!api->CalcModelPredictionFlat(handle->get(), column_size,
|
||||
float_features_buf, float_features_count,
|
||||
float_features_buf, float_features_count_current,
|
||||
result_buf, column_size))
|
||||
{
|
||||
|
||||
@ -352,18 +352,18 @@ private:
|
||||
if (cat_features_are_strings)
|
||||
{
|
||||
/// cat_features_holder stores pointers to ColumnString data or fixed_strings_data.
|
||||
PODArray<const char *> cat_features_holder(cat_features_count * column_size);
|
||||
PODArray<const char *> cat_features_holder(cat_features_count_current * column_size);
|
||||
PODArray<const char **> cat_features(column_size);
|
||||
auto cat_features_buf = cat_features.data();
|
||||
|
||||
fillCatFeaturesBuffer(cat_features_buf, cat_features_holder.data(), column_size, cat_features_count);
|
||||
fillCatFeaturesBuffer(cat_features_buf, cat_features_holder.data(), column_size, cat_features_count_current);
|
||||
/// Fixed strings are stored without termination zero, so have to copy data into fixed_strings_data.
|
||||
auto fixed_strings_data = placeStringColumns(columns, float_features_count,
|
||||
cat_features_count, cat_features_holder.data());
|
||||
auto fixed_strings_data = placeStringColumns(columns, float_features_count_current,
|
||||
cat_features_count_current, cat_features_holder.data());
|
||||
|
||||
if (!api->CalcModelPrediction(handle->get(), column_size,
|
||||
float_features_buf, float_features_count,
|
||||
cat_features_buf, cat_features_count,
|
||||
float_features_buf, float_features_count_current,
|
||||
cat_features_buf, cat_features_count_current,
|
||||
result_buf, column_size))
|
||||
{
|
||||
throw Exception(error_msg + api->GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL);
|
||||
@ -373,13 +373,13 @@ private:
|
||||
{
|
||||
PODArray<const int *> cat_features(column_size);
|
||||
auto cat_features_buf = cat_features.data();
|
||||
auto cat_features_col = placeNumericColumns<int>(columns, float_features_count,
|
||||
cat_features_count, cat_features_buf);
|
||||
calcHashes(columns, float_features_count, cat_features_count, cat_features_buf);
|
||||
auto cat_features_col = placeNumericColumns<int>(columns, float_features_count_current,
|
||||
cat_features_count_current, cat_features_buf);
|
||||
calcHashes(columns, float_features_count_current, cat_features_count_current, cat_features_buf);
|
||||
if (!api->CalcModelPredictionWithHashedCatFeatures(
|
||||
handle->get(), column_size,
|
||||
float_features_buf, float_features_count,
|
||||
cat_features_buf, cat_features_count,
|
||||
float_features_buf, float_features_count_current,
|
||||
cat_features_buf, cat_features_count_current,
|
||||
result_buf, column_size))
|
||||
{
|
||||
throw Exception(error_msg + api->GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL);
|
||||
@ -453,7 +453,7 @@ CatBoostModel::CatBoostModel(std::string name_, std::string model_path_, std::st
|
||||
{
|
||||
try
|
||||
{
|
||||
init(lib_path);
|
||||
init();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
@ -463,7 +463,7 @@ CatBoostModel::CatBoostModel(std::string name_, std::string model_path_, std::st
|
||||
creation_time = std::chrono::system_clock::now();
|
||||
}
|
||||
|
||||
void CatBoostModel::init(const std::string & lib_path)
|
||||
void CatBoostModel::init()
|
||||
{
|
||||
api_provider = getCatBoostWrapperHolder(lib_path);
|
||||
api = &api_provider->getAPI();
|
||||
|
@ -80,7 +80,7 @@ private:
|
||||
std::chrono::time_point<std::chrono::system_clock> creation_time;
|
||||
std::exception_ptr creation_exception;
|
||||
|
||||
void init(const std::string & lib_path);
|
||||
void init();
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -28,12 +28,12 @@ void ComplexKeyCacheDictionary::setAttributeValue(Attribute & attribute, const s
|
||||
if (string_ref.data && string_ref.data != null_value_ref.data())
|
||||
string_arena->free(const_cast<char *>(string_ref.data), string_ref.size);
|
||||
|
||||
const auto size = string.size();
|
||||
if (size != 0)
|
||||
const auto str_size = string.size();
|
||||
if (str_size != 0)
|
||||
{
|
||||
auto string_ptr = string_arena->alloc(size + 1);
|
||||
std::copy(string.data(), string.data() + size + 1, string_ptr);
|
||||
string_ref = StringRef{string_ptr, size};
|
||||
auto string_ptr = string_arena->alloc(str_size + 1);
|
||||
std::copy(string.data(), string.data() + str_size + 1, string_ptr);
|
||||
string_ref = StringRef{string_ptr, str_size};
|
||||
}
|
||||
else
|
||||
string_ref = {};
|
||||
|
@ -223,47 +223,47 @@ Block DictionaryBlockInputStream<DictionaryType, Key>::getBlock(size_t start, si
|
||||
template <typename DictionaryType, typename Key>
|
||||
template <typename Type, typename Container>
|
||||
void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
|
||||
DictionaryGetter<Type> getter, const PaddedPODArray<Key> & ids,
|
||||
DictionaryGetter<Type> getter, const PaddedPODArray<Key> & ids_to_fill,
|
||||
const Columns & /*keys*/, const DataTypes & /*data_types*/,
|
||||
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const
|
||||
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dict) const
|
||||
{
|
||||
(dictionary.*getter)(attribute.name, ids, container);
|
||||
(dict.*getter)(attribute.name, ids_to_fill, container);
|
||||
}
|
||||
|
||||
template <typename DictionaryType, typename Key>
|
||||
template <typename Container>
|
||||
void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
|
||||
DictionaryStringGetter getter, const PaddedPODArray<Key> & ids,
|
||||
DictionaryStringGetter getter, const PaddedPODArray<Key> & ids_to_fill,
|
||||
const Columns & /*keys*/, const DataTypes & /*data_types*/,
|
||||
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const
|
||||
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dict) const
|
||||
{
|
||||
(dictionary.*getter)(attribute.name, ids, container);
|
||||
(dict.*getter)(attribute.name, ids_to_fill, container);
|
||||
}
|
||||
|
||||
template <typename DictionaryType, typename Key>
|
||||
template <typename Type, typename Container>
|
||||
void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
|
||||
GetterByKey<Type> getter, const PaddedPODArray<Key> & /*ids*/,
|
||||
GetterByKey<Type> getter, const PaddedPODArray<Key> & /*ids_to_fill*/,
|
||||
const Columns & keys, const DataTypes & data_types,
|
||||
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const
|
||||
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dict) const
|
||||
{
|
||||
(dictionary.*getter)(attribute.name, keys, data_types, container);
|
||||
(dict.*getter)(attribute.name, keys, data_types, container);
|
||||
}
|
||||
|
||||
template <typename DictionaryType, typename Key>
|
||||
template <typename Container>
|
||||
void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
|
||||
StringGetterByKey getter, const PaddedPODArray<Key> & /*ids*/,
|
||||
StringGetterByKey getter, const PaddedPODArray<Key> & /*ids_to_fill*/,
|
||||
const Columns & keys, const DataTypes & data_types,
|
||||
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const
|
||||
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dict) const
|
||||
{
|
||||
(dictionary.*getter)(attribute.name, keys, data_types, container);
|
||||
(dict.*getter)(attribute.name, keys, data_types, container);
|
||||
}
|
||||
|
||||
template <typename DictionaryType, typename Key>
|
||||
template <template <typename> class Getter, typename StringGetter>
|
||||
Block DictionaryBlockInputStream<DictionaryType, Key>::fillBlock(
|
||||
const PaddedPODArray<Key> & ids, const Columns & keys, const DataTypes & types, ColumnsWithTypeAndName && view) const
|
||||
const PaddedPODArray<Key> & ids_to_fill, const Columns & keys, const DataTypes & types, ColumnsWithTypeAndName && view) const
|
||||
{
|
||||
std::unordered_set<std::string> names(column_names.begin(), column_names.end());
|
||||
|
||||
@ -283,7 +283,7 @@ Block DictionaryBlockInputStream<DictionaryType, Key>::fillBlock(
|
||||
const DictionaryStructure & structure = dictionary->getStructure();
|
||||
|
||||
if (structure.id && names.find(structure.id->name) != names.end())
|
||||
block_columns.emplace_back(getColumnFromIds(ids), std::make_shared<DataTypeUInt64>(), structure.id->name);
|
||||
block_columns.emplace_back(getColumnFromIds(ids_to_fill), std::make_shared<DataTypeUInt64>(), structure.id->name);
|
||||
|
||||
for (const auto idx : ext::range(0, structure.attributes.size()))
|
||||
{
|
||||
@ -293,7 +293,7 @@ Block DictionaryBlockInputStream<DictionaryType, Key>::fillBlock(
|
||||
ColumnPtr column;
|
||||
#define GET_COLUMN_FORM_ATTRIBUTE(TYPE) \
|
||||
column = getColumnFromAttribute<TYPE, Getter<TYPE>>( \
|
||||
&DictionaryType::get##TYPE, ids, keys, data_types, attribute, *dictionary)
|
||||
&DictionaryType::get##TYPE, ids_to_fill, keys, data_types, attribute, *dictionary)
|
||||
switch (attribute.underlying_type)
|
||||
{
|
||||
case AttributeUnderlyingType::UInt8:
|
||||
@ -346,37 +346,37 @@ Block DictionaryBlockInputStream<DictionaryType, Key>::fillBlock(
|
||||
template <typename DictionaryType, typename Key>
|
||||
template <typename AttributeType, typename Getter>
|
||||
ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromAttribute(
|
||||
Getter getter, const PaddedPODArray<Key> & ids,
|
||||
Getter getter, const PaddedPODArray<Key> & ids_to_fill,
|
||||
const Columns & keys, const DataTypes & data_types,
|
||||
const DictionaryAttribute & attribute, const DictionaryType & dictionary) const
|
||||
const DictionaryAttribute & attribute, const DictionaryType & dict) const
|
||||
{
|
||||
auto size = ids.size();
|
||||
auto size = ids_to_fill.size();
|
||||
if (!keys.empty())
|
||||
size = keys.front()->size();
|
||||
auto column_vector = ColumnVector<AttributeType>::create(size);
|
||||
callGetter(getter, ids, keys, data_types, column_vector->getData(), attribute, dictionary);
|
||||
callGetter(getter, ids_to_fill, keys, data_types, column_vector->getData(), attribute, dict);
|
||||
return std::move(column_vector);
|
||||
}
|
||||
|
||||
template <typename DictionaryType, typename Key>
|
||||
template <typename Getter>
|
||||
ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromStringAttribute(
|
||||
Getter getter, const PaddedPODArray<Key> & ids,
|
||||
Getter getter, const PaddedPODArray<Key> & ids_to_fill,
|
||||
const Columns & keys, const DataTypes & data_types,
|
||||
const DictionaryAttribute& attribute, const DictionaryType& dictionary) const
|
||||
const DictionaryAttribute& attribute, const DictionaryType & dict) const
|
||||
{
|
||||
auto column_string = ColumnString::create();
|
||||
auto ptr = column_string.get();
|
||||
callGetter(getter, ids, keys, data_types, ptr, attribute, dictionary);
|
||||
callGetter(getter, ids_to_fill, keys, data_types, ptr, attribute, dict);
|
||||
return std::move(column_string);
|
||||
}
|
||||
|
||||
template <typename DictionaryType, typename Key>
|
||||
ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromIds(const PaddedPODArray<Key> & ids) const
|
||||
ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromIds(const PaddedPODArray<Key> & ids_to_fill) const
|
||||
{
|
||||
auto column_vector = ColumnVector<UInt64>::create();
|
||||
column_vector->getData().reserve(ids.size());
|
||||
for (UInt64 id : ids)
|
||||
column_vector->getData().reserve(ids_to_fill.size());
|
||||
for (UInt64 id : ids_to_fill)
|
||||
column_vector->insert(id);
|
||||
return std::move(column_vector);
|
||||
}
|
||||
|
@ -7,7 +7,7 @@ namespace DB
|
||||
class DictionaryBlockInputStreamBase : public IProfilingBlockInputStream
|
||||
{
|
||||
protected:
|
||||
Block block;
|
||||
//Block block;
|
||||
|
||||
DictionaryBlockInputStreamBase(size_t rows_count, size_t max_block_size);
|
||||
|
||||
|
@ -20,10 +20,7 @@
|
||||
#include <Dictionaries/MongoDBDictionarySource.h>
|
||||
#endif
|
||||
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-parameter"
|
||||
#include <Poco/Data/ODBC/Connector.h>
|
||||
#pragma GCC diagnostic pop
|
||||
#include <Dictionaries/ODBCDictionarySource.h>
|
||||
#endif
|
||||
#if USE_MYSQL
|
||||
|
@ -240,22 +240,22 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
|
||||
const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
|
||||
const bool hierarchy_allowed, const bool allow_null_values)
|
||||
{
|
||||
Poco::Util::AbstractConfiguration::Keys keys;
|
||||
config.keys(config_prefix, keys);
|
||||
Poco::Util::AbstractConfiguration::Keys config_elems;
|
||||
config.keys(config_prefix, config_elems);
|
||||
auto has_hierarchy = false;
|
||||
|
||||
std::vector<DictionaryAttribute> attributes;
|
||||
std::vector<DictionaryAttribute> res_attributes;
|
||||
|
||||
const FormatSettings format_settings;
|
||||
|
||||
for (const auto & key : keys)
|
||||
for (const auto & config_elem : config_elems)
|
||||
{
|
||||
if (!startsWith(key.data(), "attribute"))
|
||||
if (!startsWith(config_elem.data(), "attribute"))
|
||||
continue;
|
||||
|
||||
const auto prefix = config_prefix + '.' + key + '.';
|
||||
const auto prefix = config_prefix + '.' + config_elem + '.';
|
||||
Poco::Util::AbstractConfiguration::Keys attribute_keys;
|
||||
config.keys(config_prefix + '.' + key, attribute_keys);
|
||||
config.keys(config_prefix + '.' + config_elem, attribute_keys);
|
||||
|
||||
checkAttributeKeys(attribute_keys);
|
||||
|
||||
@ -300,12 +300,12 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
|
||||
|
||||
has_hierarchy = has_hierarchy || hierarchical;
|
||||
|
||||
attributes.emplace_back(DictionaryAttribute{
|
||||
res_attributes.emplace_back(DictionaryAttribute{
|
||||
name, underlying_type, type, expression, null_value, hierarchical, injective, is_object_id
|
||||
});
|
||||
}
|
||||
|
||||
return attributes;
|
||||
return res_attributes;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -5,13 +5,10 @@
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-parameter"
|
||||
#include <Poco/MongoDB/Connection.h>
|
||||
#include <Poco/MongoDB/Cursor.h>
|
||||
#include <Poco/MongoDB/Element.h>
|
||||
#include <Poco/MongoDB/ObjectId.h>
|
||||
#pragma GCC diagnostic pop
|
||||
#include <Poco/MongoDB/Connection.h>
|
||||
#include <Poco/MongoDB/Cursor.h>
|
||||
#include <Poco/MongoDB/Element.h>
|
||||
#include <Poco/MongoDB/ObjectId.h>
|
||||
|
||||
#include <Dictionaries/DictionaryStructure.h>
|
||||
#include <Dictionaries/MongoDBBlockInputStream.h>
|
||||
|
@ -2,14 +2,11 @@
|
||||
#if USE_POCO_MONGODB
|
||||
#include <Poco/Util/AbstractConfiguration.h>
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-parameter"
|
||||
#include <Poco/MongoDB/Connection.h>
|
||||
#include <Poco/MongoDB/Database.h>
|
||||
#include <Poco/MongoDB/Cursor.h>
|
||||
#include <Poco/MongoDB/Array.h>
|
||||
#include <Poco/MongoDB/ObjectId.h>
|
||||
#pragma GCC diagnostic pop
|
||||
#include <Poco/MongoDB/Connection.h>
|
||||
#include <Poco/MongoDB/Database.h>
|
||||
#include <Poco/MongoDB/Cursor.h>
|
||||
#include <Poco/MongoDB/Array.h>
|
||||
#include <Poco/MongoDB/ObjectId.h>
|
||||
|
||||
#include <Poco/Version.h>
|
||||
|
||||
|
@ -167,10 +167,10 @@ std::string MySQLDictionarySource::quoteForLike(const std::string s)
|
||||
|
||||
LocalDateTime MySQLDictionarySource::getLastModification() const
|
||||
{
|
||||
LocalDateTime update_time{std::time(nullptr)};
|
||||
LocalDateTime modification_time{std::time(nullptr)};
|
||||
|
||||
if (dont_check_update_time)
|
||||
return update_time;
|
||||
return modification_time;
|
||||
|
||||
try
|
||||
{
|
||||
@ -190,8 +190,8 @@ LocalDateTime MySQLDictionarySource::getLastModification() const
|
||||
|
||||
if (!update_time_value.isNull())
|
||||
{
|
||||
update_time = update_time_value.getDateTime();
|
||||
LOG_TRACE(log, "Got update time: " << update_time);
|
||||
modification_time = update_time_value.getDateTime();
|
||||
LOG_TRACE(log, "Got modification time: " << modification_time);
|
||||
}
|
||||
|
||||
/// fetch remaining rows to avoid "commands out of sync" error
|
||||
@ -211,15 +211,15 @@ LocalDateTime MySQLDictionarySource::getLastModification() const
|
||||
}
|
||||
|
||||
/// we suppose failure to get modification time is not an error, therefore return current time
|
||||
return update_time;
|
||||
return modification_time;
|
||||
}
|
||||
|
||||
std::string MySQLDictionarySource::doInvalidateQuery(const std::string & request) const
|
||||
{
|
||||
Block sample_block;
|
||||
Block invalidate_sample_block;
|
||||
ColumnPtr column(ColumnString::create());
|
||||
sample_block.insert(ColumnWithTypeAndName(column, std::make_shared<DataTypeString>(), "Sample Block"));
|
||||
MySQLBlockInputStream block_input_stream(pool.Get(), request, sample_block, 1);
|
||||
invalidate_sample_block.insert(ColumnWithTypeAndName(column, std::make_shared<DataTypeString>(), "Sample Block"));
|
||||
MySQLBlockInputStream block_input_stream(pool.Get(), request, invalidate_sample_block, 1);
|
||||
return readInvalidateQuery(block_input_stream);
|
||||
}
|
||||
|
||||
|
@ -4,12 +4,9 @@
|
||||
#include <DataStreams/IProfilingBlockInputStream.h>
|
||||
#include <Dictionaries/ExternalResultDescription.h>
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-parameter"
|
||||
#include <Poco/Data/Session.h>
|
||||
#include <Poco/Data/Statement.h>
|
||||
#include <Poco/Data/RecordSet.h>
|
||||
#pragma GCC diagnostic pop
|
||||
#include <Poco/Data/Session.h>
|
||||
#include <Poco/Data/Statement.h>
|
||||
#include <Poco/Data/RecordSet.h>
|
||||
|
||||
#include <string>
|
||||
|
||||
|
@ -145,10 +145,10 @@ bool ODBCDictionarySource::isModified() const
|
||||
|
||||
std::string ODBCDictionarySource::doInvalidateQuery(const std::string & request) const
|
||||
{
|
||||
Block sample_block;
|
||||
Block invalidate_sample_block;
|
||||
ColumnPtr column(ColumnString::create());
|
||||
sample_block.insert(ColumnWithTypeAndName(column, std::make_shared<DataTypeString>(), "Sample Block"));
|
||||
ODBCBlockInputStream block_input_stream(pool->get(), request, sample_block, 1);
|
||||
invalidate_sample_block.insert(ColumnWithTypeAndName(column, std::make_shared<DataTypeString>(), "Sample Block"));
|
||||
ODBCBlockInputStream block_input_stream(pool->get(), request, invalidate_sample_block, 1);
|
||||
return readInvalidateQuery(block_input_stream);
|
||||
}
|
||||
|
||||
|
@ -1,13 +1,12 @@
|
||||
#pragma once
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-parameter"
|
||||
#include <Poco/Data/SessionPool.h>
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
#include <Dictionaries/IDictionarySource.h>
|
||||
#include <Dictionaries/ExternalQueryBuilder.h>
|
||||
#include <Dictionaries/DictionaryStructure.h>
|
||||
|
||||
|
||||
namespace Poco
|
||||
{
|
||||
namespace Util
|
||||
|
@ -189,22 +189,22 @@ std::string validateODBCConnectionString(const std::string & connection_string)
|
||||
{
|
||||
reconstructed_connection_string += '{';
|
||||
|
||||
const char * pos = value.data();
|
||||
const char * end = pos + value.size();
|
||||
const char * value_pos = value.data();
|
||||
const char * value_end = value_pos + value.size();
|
||||
while (true)
|
||||
{
|
||||
const char * next_pos = find_first_symbols<'}'>(pos, end);
|
||||
const char * next_pos = find_first_symbols<'}'>(value_pos, value_end);
|
||||
|
||||
if (next_pos == end)
|
||||
if (next_pos == value_end)
|
||||
{
|
||||
reconstructed_connection_string.append(pos, next_pos - pos);
|
||||
reconstructed_connection_string.append(value_pos, next_pos - value_pos);
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
reconstructed_connection_string.append(pos, next_pos - pos);
|
||||
reconstructed_connection_string.append(value_pos, next_pos - value_pos);
|
||||
reconstructed_connection_string.append("}}");
|
||||
pos = next_pos + 1;
|
||||
value_pos = next_pos + 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -60,6 +60,7 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, WriteBuffer &
|
||||
FormatSettings format_settings;
|
||||
format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers;
|
||||
format_settings.json.quote_denormals = settings.output_format_json_quote_denormals;
|
||||
format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes;
|
||||
format_settings.csv.delimiter = settings.format_csv_delimiter;
|
||||
format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes;
|
||||
format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes;
|
||||
@ -119,6 +120,7 @@ void registerOutputFormatJSON(FormatFactory & factory);
|
||||
void registerOutputFormatJSONCompact(FormatFactory & factory);
|
||||
void registerOutputFormatXML(FormatFactory & factory);
|
||||
void registerOutputFormatODBCDriver(FormatFactory & factory);
|
||||
void registerOutputFormatODBCDriver2(FormatFactory & factory);
|
||||
void registerOutputFormatNull(FormatFactory & factory);
|
||||
|
||||
/// Input only formats.
|
||||
@ -152,6 +154,7 @@ FormatFactory::FormatFactory()
|
||||
registerOutputFormatJSONCompact(*this);
|
||||
registerOutputFormatXML(*this);
|
||||
registerOutputFormatODBCDriver(*this);
|
||||
registerOutputFormatODBCDriver2(*this);
|
||||
registerOutputFormatNull(*this);
|
||||
}
|
||||
|
||||
|
@ -17,6 +17,7 @@ struct FormatSettings
|
||||
{
|
||||
bool quote_64bit_integers = true;
|
||||
bool quote_denormals = true;
|
||||
bool escape_forward_slashes = true;
|
||||
};
|
||||
|
||||
JSON json;
|
||||
|
@ -18,7 +18,7 @@ JSONEachRowRowOutputStream::JSONEachRowRowOutputStream(WriteBuffer & ostr_, cons
|
||||
for (size_t i = 0; i < columns; ++i)
|
||||
{
|
||||
WriteBufferFromString out(fields[i]);
|
||||
writeJSONString(sample.getByPosition(i).name, out);
|
||||
writeJSONString(sample.getByPosition(i).name, out, settings);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -21,7 +21,7 @@ JSONRowOutputStream::JSONRowOutputStream(WriteBuffer & ostr_, const Block & samp
|
||||
need_validate_utf8 = true;
|
||||
|
||||
WriteBufferFromOwnString out;
|
||||
writeJSONString(fields[i].name, out);
|
||||
writeJSONString(fields[i].name, out, settings);
|
||||
|
||||
fields[i].name = out.str();
|
||||
}
|
||||
@ -50,7 +50,7 @@ void JSONRowOutputStream::writePrefix()
|
||||
writeString(fields[i].name, *ostr);
|
||||
writeCString(",\n", *ostr);
|
||||
writeCString("\t\t\t\"type\": ", *ostr);
|
||||
writeJSONString(fields[i].type->getName(), *ostr);
|
||||
writeJSONString(fields[i].type->getName(), *ostr, settings);
|
||||
writeChar('\n', *ostr);
|
||||
|
||||
writeCString("\t\t}", *ostr);
|
||||
@ -149,7 +149,7 @@ void JSONRowOutputStream::writeTotals()
|
||||
writeCString(",\n", *ostr);
|
||||
|
||||
writeCString("\t\t", *ostr);
|
||||
writeJSONString(column.name, *ostr);
|
||||
writeJSONString(column.name, *ostr, settings);
|
||||
writeCString(": ", *ostr);
|
||||
column.type->serializeTextJSON(*column.column.get(), 0, *ostr, settings);
|
||||
}
|
||||
@ -176,7 +176,7 @@ static void writeExtremesElement(const char * title, const Block & extremes, siz
|
||||
writeCString(",\n", ostr);
|
||||
|
||||
writeCString("\t\t\t", ostr);
|
||||
writeJSONString(column.name, ostr);
|
||||
writeJSONString(column.name, ostr, settings);
|
||||
writeCString(": ", ostr);
|
||||
column.type->serializeTextJSON(*column.column.get(), row_num, ostr, settings);
|
||||
}
|
||||
|
96
dbms/src/Formats/ODBCDriver2BlockOutputStream.cpp
Normal file
96
dbms/src/Formats/ODBCDriver2BlockOutputStream.cpp
Normal file
@ -0,0 +1,96 @@
|
||||
#include <Core/Block.h>
|
||||
#include <Formats/FormatFactory.h>
|
||||
#include <Formats/ODBCDriver2BlockOutputStream.h>
|
||||
#include <IO/WriteBuffer.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
|
||||
#include <Core/iostream_debug_helpers.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
ODBCDriver2BlockOutputStream::ODBCDriver2BlockOutputStream(
|
||||
WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings)
|
||||
: out(out_), header(header_), format_settings(format_settings)
|
||||
{
|
||||
}
|
||||
|
||||
void ODBCDriver2BlockOutputStream::flush()
|
||||
{
|
||||
out.next();
|
||||
}
|
||||
|
||||
void writeODBCString(WriteBuffer & out, const std::string & str)
|
||||
{
|
||||
writeIntBinary(Int32(str.size()), out);
|
||||
out.write(str.data(), str.size());
|
||||
}
|
||||
|
||||
void ODBCDriver2BlockOutputStream::write(const Block & block)
|
||||
{
|
||||
const size_t rows = block.rows();
|
||||
const size_t columns = block.columns();
|
||||
String text_value;
|
||||
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
{
|
||||
for (size_t j = 0; j < columns; ++j)
|
||||
{
|
||||
text_value.resize(0);
|
||||
const ColumnWithTypeAndName & col = block.getByPosition(j);
|
||||
|
||||
if (col.column->isNullAt(i))
|
||||
{
|
||||
writeIntBinary(Int32(-1), out);
|
||||
}
|
||||
else
|
||||
{
|
||||
{
|
||||
WriteBufferFromString text_out(text_value);
|
||||
col.type->serializeText(*col.column, i, text_out, format_settings);
|
||||
}
|
||||
writeODBCString(out, text_value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ODBCDriver2BlockOutputStream::writePrefix()
|
||||
{
|
||||
const size_t columns = header.columns();
|
||||
|
||||
/// Number of header rows.
|
||||
writeIntBinary(Int32(2), out);
|
||||
|
||||
/// Names of columns.
|
||||
/// Number of columns + 1 for first name column.
|
||||
writeIntBinary(Int32(columns + 1), out);
|
||||
writeODBCString(out, "name");
|
||||
for (size_t i = 0; i < columns; ++i)
|
||||
{
|
||||
const ColumnWithTypeAndName & col = header.getByPosition(i);
|
||||
writeODBCString(out, col.name);
|
||||
}
|
||||
|
||||
/// Types of columns.
|
||||
writeIntBinary(Int32(columns + 1), out);
|
||||
writeODBCString(out, "type");
|
||||
for (size_t i = 0; i < columns; ++i)
|
||||
{
|
||||
const ColumnWithTypeAndName & col = header.getByPosition(i);
|
||||
writeODBCString(out, col.type->getName());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void registerOutputFormatODBCDriver2(FormatFactory & factory)
|
||||
{
|
||||
factory.registerOutputFormat(
|
||||
"ODBCDriver2", [](WriteBuffer & buf, const Block & sample, const Context &, const FormatSettings & format_settings)
|
||||
{
|
||||
return std::make_shared<ODBCDriver2BlockOutputStream>(buf, sample, format_settings);
|
||||
});
|
||||
}
|
||||
|
||||
}
|
44
dbms/src/Formats/ODBCDriver2BlockOutputStream.h
Normal file
44
dbms/src/Formats/ODBCDriver2BlockOutputStream.h
Normal file
@ -0,0 +1,44 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <Core/Block.h>
|
||||
#include <DataStreams/IBlockOutputStream.h>
|
||||
#include <Formats/FormatSettings.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
class WriteBuffer;
|
||||
|
||||
|
||||
/** A data format designed to simplify the implementation of the ODBC driver.
|
||||
* ODBC driver is designed to be build for different platforms without dependencies from the main code,
|
||||
* so the format is made that way so that it can be as easy as possible to parse it.
|
||||
* A header is displayed with the required information.
|
||||
* The data is then output in the order of the rows. Each value is displayed as follows: length in Int32 format (-1 for NULL), then data in text form.
|
||||
*/
|
||||
class ODBCDriver2BlockOutputStream : public IBlockOutputStream
|
||||
{
|
||||
public:
|
||||
ODBCDriver2BlockOutputStream(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings);
|
||||
|
||||
Block getHeader() const override
|
||||
{
|
||||
return header;
|
||||
}
|
||||
void write(const Block & block) override;
|
||||
void writePrefix() override;
|
||||
|
||||
void flush() override;
|
||||
std::string getContentType() const override
|
||||
{
|
||||
return "application/octet-stream";
|
||||
}
|
||||
|
||||
private:
|
||||
WriteBuffer & out;
|
||||
const Block header;
|
||||
const FormatSettings format_settings;
|
||||
};
|
||||
|
||||
}
|
@ -20,7 +20,10 @@ void registerFunctionsHashing(FunctionFactory & factory)
|
||||
factory.registerFunction<FunctionIntHash32>();
|
||||
factory.registerFunction<FunctionIntHash64>();
|
||||
factory.registerFunction<FunctionURLHash>();
|
||||
factory.registerFunction<FunctionMurmurHash2>();
|
||||
factory.registerFunction<FunctionMurmurHash2_32>();
|
||||
factory.registerFunction<FunctionMurmurHash2_64>();
|
||||
factory.registerFunction<FunctionMurmurHash3_32>();
|
||||
factory.registerFunction<FunctionMurmurHash3_64>();
|
||||
factory.registerFunction<FunctionMurmurHash3_128>();
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -6,6 +6,7 @@
|
||||
#include <farmhash.h>
|
||||
#include <metrohash.h>
|
||||
#include <murmurhash2.h>
|
||||
#include <murmurhash3.h>
|
||||
|
||||
#include <Poco/ByteOrder.h>
|
||||
|
||||
@ -64,6 +65,8 @@ namespace ErrorCodes
|
||||
|
||||
struct HalfMD5Impl
|
||||
{
|
||||
using ReturnType = UInt64;
|
||||
|
||||
static UInt64 apply(const char * begin, size_t size)
|
||||
{
|
||||
union
|
||||
@ -139,6 +142,8 @@ struct SHA256Impl
|
||||
|
||||
struct SipHash64Impl
|
||||
{
|
||||
using ReturnType = UInt64;
|
||||
|
||||
static UInt64 apply(const char * begin, size_t size)
|
||||
{
|
||||
return sipHash64(begin, size);
|
||||
@ -179,63 +184,6 @@ struct IntHash64Impl
|
||||
};
|
||||
|
||||
|
||||
template <typename Impl, typename Name>
|
||||
class FunctionStringHash64 : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = Name::name;
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionStringHash64>(); }
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
if (!arguments[0]->isString())
|
||||
throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
}
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override
|
||||
{
|
||||
if (const ColumnString * col_from = checkAndGetColumn<ColumnString>(block.getByPosition(arguments[0]).column.get()))
|
||||
{
|
||||
auto col_to = ColumnUInt64::create();
|
||||
|
||||
const typename ColumnString::Chars_t & data = col_from->getChars();
|
||||
const typename ColumnString::Offsets & offsets = col_from->getOffsets();
|
||||
typename ColumnUInt64::Container & vec_to = col_to->getData();
|
||||
size_t size = offsets.size();
|
||||
vec_to.resize(size);
|
||||
|
||||
ColumnString::Offset current_offset = 0;
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
vec_to[i] = Impl::apply(
|
||||
reinterpret_cast<const char *>(&data[current_offset]),
|
||||
offsets[i] - current_offset - 1);
|
||||
|
||||
current_offset = offsets[i];
|
||||
}
|
||||
|
||||
block.getByPosition(result).column = std::move(col_to);
|
||||
}
|
||||
else
|
||||
throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
|
||||
+ " of first argument of function " + Name::name,
|
||||
ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template <typename Impl>
|
||||
class FunctionStringHashFixedString : public IFunction
|
||||
{
|
||||
@ -622,12 +570,12 @@ public:
|
||||
};
|
||||
|
||||
|
||||
template <typename Impl>
|
||||
class FunctionStringHash32 : public IFunction
|
||||
template <typename Impl, typename Name>
|
||||
class FunctionStringHash : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = Impl::name;
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionStringHash32>(); }
|
||||
static constexpr auto name = Name::name;
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionStringHash>(); }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
@ -635,14 +583,15 @@ public:
|
||||
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & /* arguments */) const override { return std::make_shared<DataTypeUInt32>(); }
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments */) const override
|
||||
{ return std::make_shared<DataTypeNumber<ToType>>(); }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
|
||||
{
|
||||
auto col_to = ColumnUInt32::create(input_rows_count);
|
||||
ColumnUInt32::Container & vec_to = col_to->getData();
|
||||
auto col_to = ColumnVector<ToType>::create(input_rows_count);
|
||||
typename ColumnVector<ToType>::Container & vec_to = col_to->getData();
|
||||
|
||||
const ColumnWithTypeAndName & col = block.getByPosition(arguments[0]);
|
||||
const IDataType * from_type = col.type.get();
|
||||
@ -671,8 +620,10 @@ public:
|
||||
block.getByPosition(result).column = std::move(col_to);
|
||||
}
|
||||
private:
|
||||
using ToType = typename Impl::ReturnType;
|
||||
|
||||
template <typename FromType>
|
||||
void executeIntType(const IColumn * column, ColumnUInt32::Container & vec_to)
|
||||
void executeIntType(const IColumn * column, typename ColumnVector<ToType>::Container & vec_to)
|
||||
{
|
||||
if (const ColumnVector<FromType> * col_from = checkAndGetColumn<ColumnVector<FromType>>(column))
|
||||
{
|
||||
@ -680,7 +631,7 @@ private:
|
||||
size_t size = vec_from.size();
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
vec_to[i] = Impl::Hash32(reinterpret_cast<const char *>(&vec_from[i]), sizeof(FromType));
|
||||
vec_to[i] = Impl::apply(reinterpret_cast<const char *>(&vec_from[i]), sizeof(FromType));
|
||||
}
|
||||
}
|
||||
else
|
||||
@ -689,7 +640,7 @@ private:
|
||||
ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
|
||||
void executeString(const IColumn * column, ColumnUInt32::Container & vec_to)
|
||||
void executeString(const IColumn * column, typename ColumnVector<ToType>::Container & vec_to)
|
||||
{
|
||||
if (const ColumnString * col_from = checkAndGetColumn<ColumnString>(column))
|
||||
{
|
||||
@ -700,7 +651,7 @@ private:
|
||||
ColumnString::Offset current_offset = 0;
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
vec_to[i] = Impl::Hash32(
|
||||
vec_to[i] = Impl::apply(
|
||||
reinterpret_cast<const char *>(&data[current_offset]),
|
||||
offsets[i] - current_offset - 1);
|
||||
|
||||
@ -713,7 +664,7 @@ private:
|
||||
size_t n = col_from->getN();
|
||||
size_t size = data.size() / n;
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
vec_to[i] = Impl::Hash32(reinterpret_cast<const char *>(&data[i * n]), n);
|
||||
vec_to[i] = Impl::apply(reinterpret_cast<const char *>(&data[i * n]), n);
|
||||
}
|
||||
else
|
||||
throw Exception("Illegal column " + column->getName()
|
||||
@ -730,15 +681,69 @@ private:
|
||||
* in ClickHouse as is. For example, it is needed to reproduce the behaviour
|
||||
* for NGINX a/b testing module: https://nginx.ru/en/docs/http/ngx_http_split_clients_module.html
|
||||
*/
|
||||
struct MurmurHash2Impl
|
||||
struct MurmurHash2Impl32
|
||||
{
|
||||
static constexpr auto name = "murmurHash2_32";
|
||||
static UInt32 Hash32(const char * data, const size_t size)
|
||||
using ReturnType = UInt32;
|
||||
|
||||
static UInt32 apply(const char * data, const size_t size)
|
||||
{
|
||||
return MurmurHash2(data, size, 0);
|
||||
}
|
||||
};
|
||||
|
||||
struct MurmurHash2Impl64
|
||||
{
|
||||
using ReturnType = UInt64;
|
||||
|
||||
static UInt64 apply(const char * data, const size_t size)
|
||||
{
|
||||
return MurmurHash64A(data, size, 0);
|
||||
}
|
||||
};
|
||||
|
||||
struct MurmurHash3Impl32
|
||||
{
|
||||
using ReturnType = UInt32;
|
||||
|
||||
static UInt32 apply(const char * data, const size_t size)
|
||||
{
|
||||
union
|
||||
{
|
||||
UInt32 h;
|
||||
char bytes[sizeof(h)];
|
||||
};
|
||||
MurmurHash3_x86_32(data, size, 0, bytes);
|
||||
return h;
|
||||
}
|
||||
};
|
||||
|
||||
struct MurmurHash3Impl64
|
||||
{
|
||||
using ReturnType = UInt64;
|
||||
|
||||
static UInt64 apply(const char * data, const size_t size)
|
||||
{
|
||||
union
|
||||
{
|
||||
UInt64 h[2];
|
||||
char bytes[16];
|
||||
};
|
||||
MurmurHash3_x64_128(data, size, 0, bytes);
|
||||
return h[0] ^ h[1];
|
||||
}
|
||||
};
|
||||
|
||||
struct MurmurHash3Impl128
|
||||
{
|
||||
static constexpr auto name = "murmurHash3_128";
|
||||
enum { length = 16 };
|
||||
|
||||
static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
|
||||
{
|
||||
MurmurHash3_x64_128(begin, size, 0, out_char_data);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
struct URLHashImpl
|
||||
{
|
||||
@ -933,6 +938,12 @@ struct NameHalfMD5 { static constexpr auto name = "halfMD5"; };
|
||||
struct NameSipHash64 { static constexpr auto name = "sipHash64"; };
|
||||
struct NameIntHash32 { static constexpr auto name = "intHash32"; };
|
||||
struct NameIntHash64 { static constexpr auto name = "intHash64"; };
|
||||
struct NameMurmurHash2_32 { static constexpr auto name = "murmurHash2_32"; };
|
||||
struct NameMurmurHash2_64 { static constexpr auto name = "murmurHash2_64"; };
|
||||
struct NameMurmurHash3_32 { static constexpr auto name = "murmurHash3_32"; };
|
||||
struct NameMurmurHash3_64 { static constexpr auto name = "murmurHash3_64"; };
|
||||
struct NameMurmurHash3_128 { static constexpr auto name = "murmurHash3_128"; };
|
||||
|
||||
|
||||
struct ImplCityHash64
|
||||
{
|
||||
@ -973,8 +984,8 @@ struct ImplMetroHash64
|
||||
}
|
||||
};
|
||||
|
||||
using FunctionHalfMD5 = FunctionStringHash64<HalfMD5Impl, NameHalfMD5>;
|
||||
using FunctionSipHash64 = FunctionStringHash64<SipHash64Impl, NameSipHash64>;
|
||||
using FunctionHalfMD5 = FunctionStringHash<HalfMD5Impl, NameHalfMD5>;
|
||||
using FunctionSipHash64 = FunctionStringHash<SipHash64Impl, NameSipHash64>;
|
||||
using FunctionIntHash32 = FunctionIntHash<IntHash32Impl, NameIntHash32>;
|
||||
using FunctionIntHash64 = FunctionIntHash<IntHash64Impl, NameIntHash64>;
|
||||
using FunctionMD5 = FunctionStringHashFixedString<MD5Impl>;
|
||||
@ -985,5 +996,9 @@ using FunctionSipHash128 = FunctionStringHashFixedString<SipHash128Impl>;
|
||||
using FunctionCityHash64 = FunctionNeighbourhoodHash64<ImplCityHash64>;
|
||||
using FunctionFarmHash64 = FunctionNeighbourhoodHash64<ImplFarmHash64>;
|
||||
using FunctionMetroHash64 = FunctionNeighbourhoodHash64<ImplMetroHash64>;
|
||||
using FunctionMurmurHash2 = FunctionStringHash32<MurmurHash2Impl>;
|
||||
using FunctionMurmurHash2_32 = FunctionStringHash<MurmurHash2Impl32, NameMurmurHash2_32>;
|
||||
using FunctionMurmurHash2_64 = FunctionStringHash<MurmurHash2Impl64, NameMurmurHash2_64>;
|
||||
using FunctionMurmurHash3_32 = FunctionStringHash<MurmurHash3Impl32, NameMurmurHash3_32>;
|
||||
using FunctionMurmurHash3_64 = FunctionStringHash<MurmurHash3Impl64, NameMurmurHash3_64>;
|
||||
using FunctionMurmurHash3_128 = FunctionStringHashFixedString<MurmurHash3Impl128>;
|
||||
}
|
||||
|
@ -16,18 +16,18 @@
|
||||
*/
|
||||
|
||||
#if USE_VECTORCLASS
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic push
|
||||
#pragma clang diagnostic ignored "-Wshift-negative-value"
|
||||
#endif
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic push
|
||||
#pragma clang diagnostic ignored "-Wshift-negative-value"
|
||||
#endif
|
||||
|
||||
#include <vectorf128.h> // Y_IGNORE
|
||||
#include <vectormath_exp.h> // Y_IGNORE
|
||||
#include <vectormath_trig.h> // Y_IGNORE
|
||||
#include <vectorf128.h> // Y_IGNORE
|
||||
#include <vectormath_exp.h> // Y_IGNORE
|
||||
#include <vectormath_trig.h> // Y_IGNORE
|
||||
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic pop
|
||||
#endif
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic pop
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user