thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into base64-update

Browse Source
This commit is contained in:
Alexey Milovidov 2020-01-11 14:03:18 +03:00
commit 83869b78ca
468 changed files with 9558 additions and 4514 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/CODEOWNERS vendored
View File

@ -1,4 +1,2 @@
dbms/* @ClickHouse/core-assigner
utils/* @ClickHouse/core-assigner
docs/* @ClickHouse/docs
docs/zh/* @ClickHouse/docs-zh

3
.gitmodules vendored
View File

@ -134,3 +134,6 @@
[submodule "contrib/libc-headers"]
path = contrib/libc-headers
url = https://github.com/ClickHouse-Extras/libc-headers.git
[submodule "contrib/ryu"]
path = contrib/ryu
url = https://github.com/ClickHouse-Extras/ryu.git

838
CHANGELOG.md
View File

File diff suppressed because it is too large Load Diff

24
CMakeLists.txt
View File

@ -95,6 +95,8 @@ if (CMAKE_GENERATOR STREQUAL "Ninja")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always")
endif ()
include (cmake/add_warning.cmake)
if (NOT MSVC)
set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror is also added inside directories with our own code.
endif ()
@ -176,7 +178,9 @@ if (ARCH_NATIVE)
set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
endif ()
set (CMAKE_CXX_STANDARD 17)
# cmake < 3.12 doesn't supoprt 20. We'll set CMAKE_CXX_FLAGS for now
# set (CMAKE_CXX_STANDARD 20)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++2a")
set (CMAKE_CXX_EXTENSIONS 0) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS
set (CMAKE_CXX_STANDARD_REQUIRED ON)
@ -208,7 +212,7 @@ set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3
if (COMPILER_CLANG)
# Exception unwinding doesn't work in clang release build without this option
# TODO investigate if contrib/libcxxabi is out of date
# TODO investigate that
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer")
endif ()
@ -222,8 +226,8 @@ else ()
set(NOT_UNBUNDLED 1)
endif ()
# Using system libs can cause lot of warnings in includes (on macro expansion).
if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32)
# Using system libs can cause a lot of warnings in includes (on macro expansion).
if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN) OR ARCH_32)
option (NO_WERROR "Disable -Werror compiler option" ON)
endif ()
@ -248,8 +252,16 @@ endif ()
string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
if (NOT MAKE_STATIC_LIBRARIES)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
if (MAKE_STATIC_LIBRARIES)
set (CMAKE_POSITION_INDEPENDENT_CODE OFF)
if (OS_LINUX)
# Slightly more efficient code can be generated
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-pie")
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -fno-pie")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-no-pie")
endif ()
else ()
set (CMAKE_POSITION_INDEPENDENT_CODE ON)
endif ()
# Using "include-what-you-use" tool.

4
README.md
View File

@ -11,3 +11,7 @@ ClickHouse is an open-source column-oriented database management system that all
* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
* [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any.
* You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person.
## Upcoming Events
* [ClickHouse Meetup in San Francisco](https://www.eventbrite.com/e/clickhouse-february-meetup-registration-88496227599) on February 5.

18
cmake/add_warning.cmake Normal file
View File

@ -0,0 +1,18 @@
include (CheckCXXSourceCompiles)
# Try to add -Wflag if compiler supports it
macro (add_warning flag)
string (REPLACE "-" "_" underscored_flag ${flag})
string (REPLACE "+" "x" underscored_flag ${underscored_flag})
check_cxx_compiler_flag("-W${flag}" SUPPORTS_FLAG_${underscored_flag})
if (SUPPORTS_FLAG_${underscored_flag})
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W${flag}")
else ()
message (WARNING "Flag -W${flag} is unsupported")
endif ()
endmacro ()
# Try to add -Wno flag if compiler supports it
macro (no_warning flag)
add_warning(no-${flag})
endmacro ()

9
cmake/darwin/default_libs.cmake
View File

@ -13,11 +13,12 @@ set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
# Minimal supported SDK version
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mmacosx-version-min=10.14")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mmacosx-version-min=10.14")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mmacosx-version-min=10.15")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mmacosx-version-min=10.15")
set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -mmacosx-version-min=10.15")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -mmacosx-version-min=10.14")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -mmacosx-version-min=10.14")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -mmacosx-version-min=10.15")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -mmacosx-version-min=10.15")
# Global libraries

1
cmake/darwin/toolchain-x86_64.cmake
View File

@ -2,6 +2,7 @@ set (CMAKE_SYSTEM_NAME "Darwin")
set (CMAKE_SYSTEM_PROCESSOR "x86_64")
set (CMAKE_C_COMPILER_TARGET "x86_64-apple-darwin")
set (CMAKE_CXX_COMPILER_TARGET "x86_64-apple-darwin")
set (CMAKE_ASM_COMPILER_TARGET "x86_64-apple-darwin")
set (CMAKE_OSX_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/darwin-x86_64")
set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake

6
cmake/find/icu.cmake
View File

@ -1,4 +1,8 @@
option(ENABLE_ICU "Enable ICU" ${ENABLE_LIBRARIES})
if (OS_LINUX)
option(ENABLE_ICU "Enable ICU" ${ENABLE_LIBRARIES})
else ()
option(ENABLE_ICU "Enable ICU" 0)
endif ()
if (ENABLE_ICU)

1
cmake/target.cmake
View File

@ -13,7 +13,6 @@ if (CMAKE_CROSSCOMPILING)
if (OS_DARWIN)
# FIXME: broken dependencies
set (USE_SNAPPY OFF CACHE INTERNAL "")
set (ENABLE_SSL OFF CACHE INTERNAL "")
set (ENABLE_PROTOBUF OFF CACHE INTERNAL "")
set (ENABLE_PARQUET OFF CACHE INTERNAL "")
set (ENABLE_READLINE OFF CACHE INTERNAL "")

6
contrib/CMakeLists.txt vendored
View File

@ -2,10 +2,10 @@
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -std=c++1z")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -std=c++1z")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
endif ()
set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1)
@ -32,6 +32,8 @@ if (USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY)
add_subdirectory (double-conversion-cmake)
endif ()
add_subdirectory (ryu-cmake)
if (USE_INTERNAL_CITYHASH_LIBRARY)
add_subdirectory (cityhash102)
endif ()

2
contrib/arrow-cmake/CMakeLists.txt
View File

@ -1,5 +1,7 @@
include(ExternalProject)
set (CMAKE_CXX_STANDARD 17)
# === thrift
set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp)

2
contrib/capnproto-cmake/CMakeLists.txt
View File

@ -1,5 +1,7 @@
set (CAPNPROTO_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/src)
set (CMAKE_CXX_STANDARD 17)
set (KJ_SRCS
${CAPNPROTO_SOURCE_DIR}/kj/array.c++
${CAPNPROTO_SOURCE_DIR}/kj/common.c++

2
contrib/googletest vendored

@ -1 +1 @@
Subproject commit d175c8bf823e709d570772b038757fadf63bc632
Subproject commit 703bd9caab50b139428cea1aaff9974ebee5742e

2
contrib/icu-cmake/CMakeLists.txt
View File

@ -1,6 +1,8 @@
set(ICU_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/icu/icu4c/source)
set(ICUDATA_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/icudata/)
set (CMAKE_CXX_STANDARD 17)
# These lists of sources were generated from build log of the original ICU build system (configure + make).
set(ICUUC_SOURCES

2
contrib/libc-headers vendored

@ -1 +1 @@
Subproject commit cd82fd9d8eefe50a47a0adf7c617c3ea7d558d11
Subproject commit 9676d2645a713e679dc981ffd84dee99fcd68b8e

2
contrib/libcxx vendored

@ -1 +1 @@
Subproject commit f7c63235238a71b7e0563fab8c7c5ec1b54831f6
Subproject commit a8c453300879d0bf255f9d5959d42e2c8aac1bfb

12
contrib/libcxx-cmake/CMakeLists.txt
View File

@ -47,15 +47,19 @@ add_library(cxx ${SRCS})
target_include_directories(cxx SYSTEM BEFORE PUBLIC $<BUILD_INTERFACE:${LIBCXX_SOURCE_DIR}/include>)
target_compile_definitions(cxx PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI)
# Enable capturing stack traces for all exceptions.
if (USE_UNWIND)
target_compile_definitions(cxx PUBLIC -DSTD_EXCEPTION_HAS_STACK_TRACE=1)
endif ()
target_compile_options(cxx PUBLIC $<$<COMPILE_LANGUAGE:CXX>:-nostdinc++>)
check_cxx_compiler_flag(-Wreserved-id-macro HAVE_WARNING_RESERVED_ID_MACRO)
if (HAVE_WARNING_RESERVED_ID_MACRO)
if (SUPPORTS_FLAG_no_reserved_id_macro)
target_compile_options(cxx PUBLIC -Wno-reserved-id-macro)
endif ()
check_cxx_compiler_flag(-Wctad-maybe-unsupported HAVE_WARNING_CTAD_MAYBE_UNSUPPORTED)
if (HAVE_WARNING_CTAD_MAYBE_UNSUPPORTED)
if (SUPPORTS_FLAG_no_ctad_maybe_unsupported)
target_compile_options(cxx PUBLIC -Wno-ctad-maybe-unsupported)
endif ()

5
contrib/libcxxabi-cmake/CMakeLists.txt
View File

@ -32,6 +32,11 @@ target_compile_definitions(cxxabi PRIVATE -D_LIBCPP_BUILDING_LIBRARY)
target_compile_options(cxxabi PRIVATE -nostdinc++ -fno-sanitize=undefined -Wno-macro-redefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast.
target_link_libraries(cxxabi PUBLIC ${EXCEPTION_HANDLING_LIBRARY})
# Enable capturing stack traces for all exceptions.
if (USE_UNWIND)
target_compile_definitions(cxxabi PUBLIC -DSTD_EXCEPTION_HAS_STACK_TRACE=1)
endif ()
install(
TARGETS cxxabi
EXPORT global

8
contrib/libhdfs3-cmake/CMake/Platform.cmake
View File

@ -7,10 +7,14 @@ ELSE(CMAKE_SYSTEM_NAME STREQUAL "Linux")
ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
IF(CMAKE_COMPILER_IS_GNUCXX)
EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_COMPILER_VERSION)
EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} -dumpfullversion OUTPUT_VARIABLE GCC_COMPILER_VERSION)
IF (NOT GCC_COMPILER_VERSION)
MESSAGE(FATAL_ERROR "Cannot get gcc version")
EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_COMPILER_VERSION)
IF (NOT GCC_COMPILER_VERSION)
MESSAGE(FATAL_ERROR "Cannot get gcc version")
ENDIF (NOT GCC_COMPILER_VERSION)
ENDIF (NOT GCC_COMPILER_VERSION)
STRING(REGEX MATCHALL "[0-9]+" GCC_COMPILER_VERSION ${GCC_COMPILER_VERSION})

26
contrib/openssl-cmake/CMakeLists.txt
View File

@ -1,16 +1,6 @@
set(OPENSSL_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/openssl)
set(OPENSSL_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/openssl)
#file(READ ${CMAKE_CURRENT_SOURCE_DIR}/${OPENSSL_SOURCE_DIR}/ssl/VERSION SSL_VERSION)
#string(STRIP ${SSL_VERSION} SSL_VERSION)
#string(REPLACE ":" "." SSL_VERSION ${SSL_VERSION})
#string(REGEX REPLACE "\\..*" "" SSL_MAJOR_VERSION ${SSL_VERSION})
#file(READ ${CMAKE_CURRENT_SOURCE_DIR}/${OPENSSL_SOURCE_DIR}/crypto/VERSION CRYPTO_VERSION)
#string(STRIP ${CRYPTO_VERSION} CRYPTO_VERSION)
#string(REPLACE ":" "." CRYPTO_VERSION ${CRYPTO_VERSION})
#string(REGEX REPLACE "\\..*" "" CRYPTO_MAJOR_VERSION ${CRYPTO_VERSION})
set(OPENSSLDIR "/etc/ssl" CACHE PATH "Set the default openssl directory")
set(OPENSSL_ENGINESDIR "/usr/lib/engines-3" CACHE PATH "Set the default openssl directory for engines")
set(OPENSSL_MODULESDIR "/usr/local/lib/ossl-modules" CACHE PATH "Set the default openssl directory for modules")
@ -27,19 +17,27 @@ elseif(ARCH_AARCH64)
endif()
enable_language(ASM)
if (COMPILER_CLANG)
add_definitions(-Wno-unused-command-line-argument)
endif ()
if (ARCH_AMD64)
if (OS_DARWIN)
set (OPENSSL_SYSTEM "macosx")
endif ()
macro(perl_generate_asm FILE_IN FILE_OUT)
get_filename_component(DIRNAME ${FILE_OUT} DIRECTORY)
file(MAKE_DIRECTORY ${DIRNAME})
add_custom_command(OUTPUT ${FILE_OUT}
COMMAND /usr/bin/env perl ${FILE_IN} ${FILE_OUT}
COMMAND /usr/bin/env perl ${FILE_IN} ${OPENSSL_SYSTEM} ${FILE_OUT}
# ASM code has broken unwind tables (CFI), strip them.
# Otherwise asynchronous unwind (that we use for query profiler)
# will lead to segfault while trying to interpret wrong "CFA expression".
COMMAND sed -i -e '/^\.cfi_/d' ${FILE_OUT})
endmacro()
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/aes-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/aes/aes-x86_64.s)
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/aesni-mb-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/aes/aesni-mb-x86_64.s)
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/aesni-sha1-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/aes/aesni-sha1-x86_64.s)
@ -70,12 +68,17 @@ if (ARCH_AMD64)
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha512-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha256-x86_64.s) # This is not a mistake
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha512-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha512-x86_64.s)
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/whrlpool/asm/wp-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/whrlpool/wp-x86_64.s)
elseif (ARCH_AARCH64)
macro(perl_generate_asm FILE_IN FILE_OUT)
get_filename_component(DIRNAME ${FILE_OUT} DIRECTORY)
file(MAKE_DIRECTORY ${DIRNAME})
add_custom_command(OUTPUT ${FILE_OUT}
COMMAND /usr/bin/env perl ${FILE_IN} "linux64" ${FILE_OUT})
# Hope that the ASM code for AArch64 doesn't have broken CFI. Otherwise, add the same sed as for x86_64.
endmacro()
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/aesv8-armx.pl ${OPENSSL_BINARY_DIR}/crypto/aes/aesv8-armx.S)
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/vpaes-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/aes/vpaes-armv8.S)
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/bn/asm/armv8-mont.pl ${OPENSSL_BINARY_DIR}/crypto/bn/armv8-mont.S)
@ -88,6 +91,7 @@ elseif (ARCH_AARCH64)
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha1-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha1-armv8.S)
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha512-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha256-armv8.S) # This is not a mistake
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha512-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha512-armv8.S)
endif ()
set(CRYPTO_SRCS

1
contrib/ryu vendored Submodule

@ -0,0 +1 @@
Subproject commit 5b4a853534b47438b4d97935370f6b2397137c2b

10
contrib/ryu-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,10 @@
SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/ryu)
add_library(ryu
${LIBRARY_DIR}/ryu/d2fixed.c
${LIBRARY_DIR}/ryu/d2s.c
${LIBRARY_DIR}/ryu/f2s.c
${LIBRARY_DIR}/ryu/generic_128.c
)
target_include_directories(ryu SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}")

2
contrib/zlib-ng vendored

@ -1 +1 @@
Subproject commit 5673222fbd37ea89afb2ea73096f9bf5ec68ea31
Subproject commit bba56a73be249514acfbc7d49aa2a68994dad8ab

90
dbms/CMakeLists.txt
View File

@ -45,36 +45,75 @@ endif ()
option (WEVERYTHING "Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang." ON)
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpedantic -Wno-vla-extension -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types")
if (COMPILER_CLANG)
add_warning(pedantic)
no_warning(gnu-anonymous-struct)
no_warning(nested-anon-types)
no_warning(vla-extension)
no_warning(zero-length-array)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow -Wshadow-uncaptured-local -Wextra-semi -Wcomma -Winconsistent-missing-destructor-override -Wunused-exception-parameter -Wcovered-switch-default -Wold-style-cast -Wrange-loop-analysis -Wunused-member-function -Wunreachable-code -Wunreachable-code-return -Wnewline-eof -Wembedded-directive -Wgnu-case-range -Wunused-macros -Wconditional-uninitialized -Wdeprecated -Wundef -Wreserved-id-macro -Wredundant-parens -Wzero-as-null-pointer-constant")
add_warning(comma)
add_warning(conditional-uninitialized)
add_warning(covered-switch-default)
add_warning(deprecated)
add_warning(embedded-directive)
add_warning(empty-init-stmt) # linux-only
add_warning(extra-semi-stmt) # linux-only
add_warning(extra-semi)
add_warning(gnu-case-range)
add_warning(inconsistent-missing-destructor-override)
add_warning(newline-eof)
add_warning(old-style-cast)
add_warning(range-loop-analysis)
add_warning(redundant-parens)
add_warning(reserved-id-macro)
add_warning(shadow-field) # clang 8+
add_warning(shadow-uncaptured-local)
add_warning(shadow)
add_warning(string-plus-int) # clang 8+
add_warning(undef)
add_warning(unreachable-code-return)
add_warning(unreachable-code)
add_warning(unused-exception-parameter)
add_warning(unused-macros)
add_warning(unused-member-function)
add_warning(zero-as-null-pointer-constant)
if (WEVERYTHING)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-padded -Wno-switch-enum -Wno-deprecated-dynamic-exception-spec -Wno-float-equal -Wno-weak-vtables -Wno-shift-sign-overflow -Wno-sign-conversion -Wno-conversion -Wno-exit-time-destructors -Wno-undefined-func-template -Wno-documentation-unknown-command -Wno-missing-variable-declarations -Wno-unused-template -Wno-global-constructors -Wno-c99-extensions -Wno-missing-prototypes -Wno-weak-template-vtables -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types -Wno-double-promotion -Wno-disabled-macro-expansion -Wno-vla-extension -Wno-vla -Wno-packed")
add_warning(everything)
no_warning(c++98-compat-pedantic)
no_warning(c++98-compat)
no_warning(c99-extensions)
no_warning(conversion)
no_warning(ctad-maybe-unsupported) # clang 9+, linux-only
no_warning(deprecated-dynamic-exception-spec)
no_warning(disabled-macro-expansion)
no_warning(documentation-unknown-command)
no_warning(double-promotion)
no_warning(exit-time-destructors)
no_warning(float-equal)
no_warning(global-constructors)
no_warning(gnu-anonymous-struct)
no_warning(missing-prototypes)
no_warning(missing-variable-declarations)
no_warning(nested-anon-types)
no_warning(packed)
no_warning(padded)
no_warning(return-std-move-in-c++11) # clang 7+
no_warning(shift-sign-overflow)
no_warning(sign-conversion)
no_warning(switch-enum)
no_warning(undefined-func-template)
no_warning(unused-template)
no_warning(vla-extension)
no_warning(vla)
no_warning(weak-template-vtables)
no_warning(weak-vtables)
no_warning(zero-length-array)
# TODO Enable conversion, sign-conversion, double-promotion warnings.
endif ()
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7)
if (WEVERYTHING)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-std-move-in-c++11")
endif ()
endif ()
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow-field -Wstring-plus-int")
if(NOT APPLE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra-semi-stmt -Wempty-init-stmt")
endif()
endif ()
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9)
if (WEVERYTHING AND NOT APPLE)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-ctad-maybe-unsupported")
endif ()
endif ()
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
elseif (COMPILER_GCC)
# Add compiler options only to c++ compiler
function(add_cxx_compile_options option)
add_compile_options("$<$<STREQUAL:$<TARGET_PROPERTY:LINKER_LANGUAGE>,CXX>:${option}>")
@ -156,7 +195,7 @@ if (USE_DEBUG_HELPERS)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${INCLUDE_DEBUG_HELPERS}")
endif ()
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
if (COMPILER_GCC)
# If we leave this optimization enabled, gcc-7 replaces a pair of SSE intrinsics (16 byte load, store) with a call to memcpy.
# It leads to slow code. This is compiler bug. It looks like this:
#
@ -330,6 +369,7 @@ target_link_libraries (clickhouse_common_io
${LINK_LIBRARIES_ONLY_ON_X86_64}
PUBLIC
${DOUBLE_CONVERSION_LIBRARIES}
ryu
PUBLIC
${Poco_Net_LIBRARY}
${Poco_Util_LIBRARY}

12
dbms/cmake/version.cmake
View File

@ -1,11 +1,11 @@
# This strings autochanged from release_lib.sh:
set(VERSION_REVISION 54430)
set(VERSION_MAJOR 19)
set(VERSION_MINOR 19)
set(VERSION_REVISION 54431)
set(VERSION_MAJOR 20)
set(VERSION_MINOR 1)
set(VERSION_PATCH 1)
set(VERSION_GITHASH 8bd9709d1dec3366e35d2efeab213435857f67a9)
set(VERSION_DESCRIBE v19.19.1.1-prestable)
set(VERSION_STRING 19.19.1.1)
set(VERSION_GITHASH 51d4c8a53be94504e3607b2232e12e5ef7a8ec28)
set(VERSION_DESCRIBE v20.1.1.1-prestable)
set(VERSION_STRING 20.1.1.1)
# end of autochange
set(VERSION_EXTRA "" CACHE STRING "")

85
dbms/programs/client/Client.cpp
View File

@ -300,7 +300,7 @@ private:
&& std::string::npos == embedded_stack_trace_pos)
{
std::cerr << "Stack trace:" << std::endl
<< e.getStackTrace().toString();
<< e.getStackTraceString();
}
/// If exception code isn't zero, we should return non-zero return code anyway.
@ -327,6 +327,78 @@ private:
|| (now.month() == 1 && now.day() <= 5);
}
bool isChineseNewYearMode(const String & local_tz)
{
/// Days of Dec. 20 in Chinese calendar starting from year 2019 to year 2105
static constexpr UInt16 chineseNewYearIndicators[]
= {18275, 18659, 19014, 19368, 19752, 20107, 20491, 20845, 21199, 21583, 21937, 22292, 22676, 23030, 23414, 23768, 24122, 24506,
24860, 25215, 25599, 25954, 26308, 26692, 27046, 27430, 27784, 28138, 28522, 28877, 29232, 29616, 29970, 30354, 30708, 31062,
31446, 31800, 32155, 32539, 32894, 33248, 33632, 33986, 34369, 34724, 35078, 35462, 35817, 36171, 36555, 36909, 37293, 37647,
38002, 38386, 38740, 39095, 39479, 39833, 40187, 40571, 40925, 41309, 41664, 42018, 42402, 42757, 43111, 43495, 43849, 44233,
44587, 44942, 45326, 45680, 46035, 46418, 46772, 47126, 47510, 47865, 48249, 48604, 48958, 49342};
static constexpr size_t N = sizeof(chineseNewYearIndicators) / sizeof(chineseNewYearIndicators[0]);
/// All time zone names are acquired from https://www.iana.org/time-zones
static constexpr const char * chineseNewYearTimeZoneIndicators[] = {
/// Time zones celebrating Chinese new year.
"Asia/Shanghai",
"Asia/Chongqing",
"Asia/Harbin",
"Asia/Urumqi",
"Asia/Hong_Kong",
"Asia/Chungking",
"Asia/Macao",
"Asia/Macau",
"Asia/Taipei",
"Asia/Singapore",
/// Time zones celebrating Chinese new year but with different festival names. Let's not print the message for now.
// "Asia/Brunei",
// "Asia/Ho_Chi_Minh",
// "Asia/Hovd",
// "Asia/Jakarta",
// "Asia/Jayapura",
// "Asia/Kashgar",
// "Asia/Kuala_Lumpur",
// "Asia/Kuching",
// "Asia/Makassar",
// "Asia/Pontianak",
// "Asia/Pyongyang",
// "Asia/Saigon",
// "Asia/Seoul",
// "Asia/Ujung_Pandang",
// "Asia/Ulaanbaatar",
// "Asia/Ulan_Bator",
};
static constexpr size_t M = sizeof(chineseNewYearTimeZoneIndicators) / sizeof(chineseNewYearTimeZoneIndicators[0]);
time_t current_time = time(nullptr);
if (chineseNewYearTimeZoneIndicators + M
== std::find_if(chineseNewYearTimeZoneIndicators, chineseNewYearTimeZoneIndicators + M, [&local_tz](const char * tz)
{
return tz == local_tz;
}))
return false;
/// It's bad to be intrusive.
if (current_time % 3 != 0)
return false;
auto days = DateLUT::instance().toDayNum(current_time).toUnderType();
for (auto i = 0ul; i < N; ++i)
{
auto d = chineseNewYearIndicators[i];
/// Let's celebrate until Lantern Festival
if (d <= days && d + 25u >= days)
return true;
else if (d > days)
return false;
}
return false;
}
int mainImpl()
{
UseSSL use_ssl;
@ -374,7 +446,7 @@ private:
connect();
/// Initialize DateLUT here to avoid counting time spent here as query execution time.
DateLUT::instance();
const auto local_tz = DateLUT::instance().getTimeZone();
if (!context.getSettingsRef().use_client_time_zone)
{
const auto & time_zone = connection->getServerTimezone(connection_parameters.timeouts);
@ -540,7 +612,12 @@ private:
loop();
std::cout << (isNewYearMode() ? "Happy new year." : "Bye.") << std::endl;
if (isNewYearMode())
std::cout << "Happy new year." << std::endl;
else if (isChineseNewYearMode(local_tz))
std::cout << "Happy Chinese new year. 春节快乐!" << std::endl;
else
std::cout << "Bye." << std::endl;
return 0;
}
else
@ -714,7 +791,7 @@ private:
if (config().getBool("stacktrace", false))
std::cerr << "Stack trace:" << std::endl
<< e.getStackTrace().toString() << std::endl;
<< e.getStackTraceString() << std::endl;
std::cerr << std::endl;

2
dbms/programs/odbc-bridge/MainHandler.cpp
View File

@ -115,7 +115,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne
catch (const Exception & ex)
{
process_error("Invalid 'columns' parameter in request body '" + ex.message() + "'");
LOG_WARNING(log, ex.getStackTrace().toString());
LOG_WARNING(log, ex.getStackTraceString());
return;
}

12
dbms/programs/performance-test/PerformanceTest.cpp
View File

@ -85,16 +85,6 @@ bool PerformanceTest::checkPreconditions() const
for (const std::string & precondition : preconditions)
{
if (precondition == "flush_disk_cache")
{
if (system(
"(>&2 echo 'Flushing disk cache...') && (sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches') && (>&2 echo 'Flushed.')"))
{
LOG_WARNING(log, "Failed to flush disk cache");
return false;
}
}
if (precondition == "ram_size")
{
size_t ram_size_needed = config->getUInt64("preconditions.ram_size");
@ -337,7 +327,7 @@ void PerformanceTest::runQueries(
{
statistics.exception = "Code: " + std::to_string(e.code()) + ", e.displayText() = " + e.displayText();
LOG_WARNING(log, "Code: " << e.code() << ", e.displayText() = " << e.displayText()
<< ", Stack trace:\n\n" << e.getStackTrace().toString());
<< ", Stack trace:\n\n" << e.getStackTraceString());
}
if (!statistics.got_SIGINT)

42
dbms/programs/performance-test/PerformanceTestInfo.cpp
View File

@ -45,21 +45,11 @@ namespace fs = std::filesystem;
PerformanceTestInfo::PerformanceTestInfo(
XMLConfigurationPtr config,
const std::string & profiles_file_,
const Settings & global_settings_)
: profiles_file(profiles_file_)
, settings(global_settings_)
: settings(global_settings_)
{
path = config->getString("path");
test_name = fs::path(path).stem().string();
if (config->has("main_metric"))
{
Strings main_metrics;
config->keys("main_metric", main_metrics);
if (main_metrics.size())
main_metric = main_metrics[0];
}
applySettings(config);
extractQueries(config);
extractAuxiliaryQueries(config);
@ -75,38 +65,8 @@ void PerformanceTestInfo::applySettings(XMLConfigurationPtr config)
SettingsChanges settings_to_apply;
Strings config_settings;
config->keys("settings", config_settings);
auto settings_contain = [&config_settings] (const std::string & setting)
{
auto position = std::find(config_settings.begin(), config_settings.end(), setting);
return position != config_settings.end();
};
/// Preprocess configuration file
if (settings_contain("profile"))
{
if (!profiles_file.empty())
{
std::string profile_name = config->getString("settings.profile");
XMLConfigurationPtr profiles_config(new XMLConfiguration(profiles_file));
Strings profile_settings;
profiles_config->keys("profiles." + profile_name, profile_settings);
extractSettings(profiles_config, "profiles." + profile_name, profile_settings, settings_to_apply);
}
}
extractSettings(config, "settings", config_settings, settings_to_apply);
settings.applyChanges(settings_to_apply);
if (settings_contain("average_rows_speed_precision"))
TestStats::avg_rows_speed_precision =
config->getDouble("settings.average_rows_speed_precision");
if (settings_contain("average_bytes_speed_precision"))
TestStats::avg_bytes_speed_precision =
config->getDouble("settings.average_bytes_speed_precision");
}
}

4
dbms/programs/performance-test/PerformanceTestInfo.h
View File

@ -26,15 +26,13 @@ using StringToVector = std::map<std::string, Strings>;
class PerformanceTestInfo
{
public:
PerformanceTestInfo(XMLConfigurationPtr config, const std::string & profiles_file_, const Settings & global_settings_);
PerformanceTestInfo(XMLConfigurationPtr config, const Settings & global_settings_);
std::string test_name;
std::string path;
std::string main_metric;
Strings queries;
std::string profiles_file;
Settings settings;
ExecutionType exec_type;
StringToVector substitutions;

7
dbms/programs/performance-test/PerformanceTestSuite.cpp
View File

@ -64,7 +64,6 @@ public:
const std::string & password_,
const Settings & cmd_settings,
const bool lite_output_,
const std::string & profiles_file_,
Strings && input_files_,
Strings && tests_tags_,
Strings && skip_tags_,
@ -86,7 +85,6 @@ public:
, skip_names_regexp(std::move(skip_names_regexp_))
, query_indexes(query_indexes_)
, lite_output(lite_output_)
, profiles_file(profiles_file_)
, input_files(input_files_)
, log(&Poco::Logger::get("PerformanceTestSuite"))
{
@ -139,7 +137,6 @@ private:
using XMLConfigurationPtr = Poco::AutoPtr<XMLConfiguration>;
bool lite_output;
std::string profiles_file;
Strings input_files;
std::vector<XMLConfigurationPtr> tests_configurations;
@ -197,7 +194,7 @@ private:
std::pair<std::string, bool> runTest(XMLConfigurationPtr & test_config)
{
PerformanceTestInfo info(test_config, profiles_file, global_context.getSettingsRef());
PerformanceTestInfo info(test_config, global_context.getSettingsRef());
LOG_INFO(log, "Config for test '" << info.test_name << "' parsed");
PerformanceTest current(test_config, connection, timeouts, interrupt_listener, info, global_context, query_indexes[info.path]);
@ -332,7 +329,6 @@ try
desc.add_options()
("help", "produce help message")
("lite", "use lite version of output")
("profiles-file", value<std::string>()->default_value(""), "Specify a file with global profiles")
("host,h", value<std::string>()->default_value("localhost"), "")
("port", value<UInt16>()->default_value(9000), "")
("secure,s", "Use TLS connection")
@ -401,7 +397,6 @@ try
options["password"].as<std::string>(),
cmd_settings,
options.count("lite") > 0,
options["profiles-file"].as<std::string>(),
std::move(input_files),
std::move(tests_tags),
std::move(skip_tags),

41
dbms/programs/performance-test/ReportBuilder.cpp
View File

@ -17,23 +17,25 @@ namespace DB
namespace
{
const std::regex QUOTE_REGEX{"\""};
std::string getMainMetric(const PerformanceTestInfo & test_info)
{
std::string main_metric;
if (test_info.main_metric.empty())
if (test_info.exec_type == ExecutionType::Loop)
main_metric = "min_time";
else
main_metric = "rows_per_second";
if (test_info.exec_type == ExecutionType::Loop)
return "min_time";
else
main_metric = test_info.main_metric;
return main_metric;
return "rows_per_second";
}
bool isASCIIString(const std::string & str)
{
return std::all_of(str.begin(), str.end(), isASCII);
}
String jsonString(const String & str, FormatSettings & settings)
{
WriteBufferFromOwnString buffer;
writeJSONString(str, buffer, settings);
return std::move(buffer.str());
}
}
ReportBuilder::ReportBuilder(const std::string & server_version_)
@ -55,6 +57,8 @@ std::string ReportBuilder::buildFullReport(
std::vector<TestStats> & stats,
const std::vector<std::size_t> & queries_to_run) const
{
FormatSettings settings;
JSONString json_output;
json_output.set("hostname", hostname);
@ -65,22 +69,18 @@ std::string ReportBuilder::buildFullReport(
json_output.set("time", getCurrentTime());
json_output.set("test_name", test_info.test_name);
json_output.set("path", test_info.path);
json_output.set("main_metric", getMainMetric(test_info));
if (test_info.substitutions.size())
if (!test_info.substitutions.empty())
{
JSONString json_parameters(2); /// here, 2 is the size of \t padding
for (auto it = test_info.substitutions.begin(); it != test_info.substitutions.end(); ++it)
for (auto & [parameter, values] : test_info.substitutions)
{
std::string parameter = it->first;
Strings values = it->second;
std::ostringstream array_string;
array_string << "[";
for (size_t i = 0; i != values.size(); ++i)
{
array_string << '"' << std::regex_replace(values[i], QUOTE_REGEX, "\\\"") << '"';
array_string << jsonString(values[i], settings);
if (i != values.size() - 1)
{
array_string << ", ";
@ -110,13 +110,12 @@ std::string ReportBuilder::buildFullReport(
JSONString runJSON;
auto query = std::regex_replace(test_info.queries[query_index], QUOTE_REGEX, "\\\"");
runJSON.set("query", query);
runJSON.set("query", jsonString(test_info.queries[query_index], settings), false);
runJSON.set("query_index", query_index);
if (!statistics.exception.empty())
{
if (isASCIIString(statistics.exception))
runJSON.set("exception", std::regex_replace(statistics.exception, QUOTE_REGEX, "\\\""));
runJSON.set("exception", jsonString(statistics.exception, settings), false);
else
runJSON.set("exception", "Some exception occured with non ASCII message. This may produce invalid JSON. Try reproduce locally.");
}
@ -183,7 +182,7 @@ std::string ReportBuilder::buildCompactReport(
std::vector<TestStats> & stats,
const std::vector<std::size_t> & queries_to_run) const
{
FormatSettings settings;
std::ostringstream output;
for (size_t query_index = 0; query_index < test_info.queries.size(); ++query_index)
@ -194,7 +193,7 @@ std::string ReportBuilder::buildCompactReport(
for (size_t number_of_launch = 0; number_of_launch < test_info.times_to_run; ++number_of_launch)
{
if (test_info.queries.size() > 1)
output << "query \"" << test_info.queries[query_index] << "\", ";
output << "query " << jsonString(test_info.queries[query_index], settings) << ", ";
output << "run " << std::to_string(number_of_launch + 1) << ": ";

61
dbms/programs/server/HTTPHandler.cpp
View File

@ -20,8 +20,6 @@
#include <Compression/CompressedReadBuffer.h>
#include <Compression/CompressedWriteBuffer.h>
#include <IO/ReadBufferFromIStream.h>
#include <IO/ZlibInflatingReadBuffer.h>
#include <IO/BrotliReadBuffer.h>
#include <IO/ReadBufferFromString.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteBufferFromHTTPServerResponse.h>
@ -300,32 +298,24 @@ void HTTPHandler::processQuery(
/// The client can pass a HTTP header indicating supported compression method (gzip or deflate).
String http_response_compression_methods = request.get("Accept-Encoding", "");
bool client_supports_http_compression = false;
CompressionMethod http_response_compression_method {};
CompressionMethod http_response_compression_method = CompressionMethod::None;
if (!http_response_compression_methods.empty())
{
/// If client supports brotli - it's preferred.
/// Both gzip and deflate are supported. If the client supports both, gzip is preferred.
/// NOTE parsing of the list of methods is slightly incorrect.
if (std::string::npos != http_response_compression_methods.find("gzip"))
{
client_supports_http_compression = true;
http_response_compression_method = CompressionMethod::Gzip;
}
else if (std::string::npos != http_response_compression_methods.find("deflate"))
{
client_supports_http_compression = true;
http_response_compression_method = CompressionMethod::Zlib;
}
#if USE_BROTLI
else if (http_response_compression_methods == "br")
{
client_supports_http_compression = true;
if (std::string::npos != http_response_compression_methods.find("br"))
http_response_compression_method = CompressionMethod::Brotli;
}
#endif
else if (std::string::npos != http_response_compression_methods.find("gzip"))
http_response_compression_method = CompressionMethod::Gzip;
else if (std::string::npos != http_response_compression_methods.find("deflate"))
http_response_compression_method = CompressionMethod::Zlib;
}
bool client_supports_http_compression = http_response_compression_method != CompressionMethod::None;
/// Client can pass a 'compress' flag in the query string. In this case the query result is
/// compressed using internal algorithm. This is not reflected in HTTP headers.
bool internal_compression = params.getParsed<bool>("compress", false);
@ -344,8 +334,8 @@ void HTTPHandler::processQuery(
unsigned keep_alive_timeout = config.getUInt("keep_alive_timeout", 10);
used_output.out = std::make_shared<WriteBufferFromHTTPServerResponse>(
request, response, keep_alive_timeout,
client_supports_http_compression, http_response_compression_method, buffer_size_http);
request, response, keep_alive_timeout, client_supports_http_compression, http_response_compression_method);
if (internal_compression)
used_output.out_maybe_compressed = std::make_shared<CompressedWriteBuffer>(*used_output.out);
else
@ -400,32 +390,9 @@ void HTTPHandler::processQuery(
std::unique_ptr<ReadBuffer> in_post_raw = std::make_unique<ReadBufferFromIStream>(istr);
/// Request body can be compressed using algorithm specified in the Content-Encoding header.
std::unique_ptr<ReadBuffer> in_post;
String http_request_compression_method_str = request.get("Content-Encoding", "");
if (!http_request_compression_method_str.empty())
{
if (http_request_compression_method_str == "gzip")
{
in_post = std::make_unique<ZlibInflatingReadBuffer>(std::move(in_post_raw), CompressionMethod::Gzip);
}
else if (http_request_compression_method_str == "deflate")
{
in_post = std::make_unique<ZlibInflatingReadBuffer>(std::move(in_post_raw), CompressionMethod::Zlib);
}
#if USE_BROTLI
else if (http_request_compression_method_str == "br")
{
in_post = std::make_unique<BrotliReadBuffer>(std::move(in_post_raw));
}
#endif
else
{
throw Exception("Unknown Content-Encoding of HTTP request: " + http_request_compression_method_str,
ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
}
}
else
in_post = std::move(in_post_raw);
std::unique_ptr<ReadBuffer> in_post = wrapReadBufferWithCompressionMethod(
std::make_unique<ReadBufferFromIStream>(istr), chooseCompressionMethod({}, http_request_compression_method_str));
/// The data can also be compressed using incompatible internal algorithm. This is indicated by
/// 'decompress' query parameter.

11
dbms/programs/server/Server.cpp
View File

@ -947,6 +947,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
});
/// try to load dictionaries immediately, throw on error and die
ext::scope_guard dictionaries_xmls, models_xmls;
try
{
if (!config().getBool("dictionaries_lazy_load", true))
@ -954,12 +955,10 @@ int Server::main(const std::vector<std::string> & /*args*/)
global_context->tryCreateEmbeddedDictionaries();
global_context->getExternalDictionariesLoader().enableAlwaysLoadEverything(true);
}
auto dictionaries_repository = std::make_unique<ExternalLoaderXMLConfigRepository>(config(), "dictionaries_config");
global_context->getExternalDictionariesLoader().addConfigRepository("", std::move(dictionaries_repository));
auto models_repository = std::make_unique<ExternalLoaderXMLConfigRepository>(config(), "models_config");
global_context->getExternalModelsLoader().addConfigRepository("", std::move(models_repository));
dictionaries_xmls = global_context->getExternalDictionariesLoader().addConfigRepository(
std::make_unique<ExternalLoaderXMLConfigRepository>(config(), "dictionaries_config"));
models_xmls = global_context->getExternalModelsLoader().addConfigRepository(
std::make_unique<ExternalLoaderXMLConfigRepository>(config(), "models_config"));
}
catch (...)
{

16
dbms/programs/server/TCPHandler.cpp
View File

@ -112,7 +112,7 @@ void TCPHandler::runImpl()
{
Exception e("Database " + backQuote(default_database) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE);
LOG_ERROR(log, "Code: " << e.code() << ", e.displayText() = " << e.displayText()
<< ", Stack trace:\n\n" << e.getStackTrace().toString());
<< ", Stack trace:\n\n" << e.getStackTraceString());
sendException(e, connection_context.getSettingsRef().calculate_text_stack_trace);
return;
}
@ -158,7 +158,7 @@ void TCPHandler::runImpl()
/** An exception during the execution of request (it must be sent over the network to the client).
* The client will be able to accept it, if it did not happen while sending another packet and the client has not disconnected yet.
*/
std::unique_ptr<Exception> exception;
std::optional<DB::Exception> exception;
bool network_error = false;
bool send_exception_with_stack_trace = connection_context.getSettingsRef().calculate_text_stack_trace;
@ -280,7 +280,7 @@ void TCPHandler::runImpl()
catch (const Exception & e)
{
state.io.onException();
exception.reset(e.clone());
exception.emplace(e);
if (e.code() == ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT)
throw;
@ -298,22 +298,22 @@ void TCPHandler::runImpl()
* We will try to send exception to the client in any case - see below.
*/
state.io.onException();
exception = std::make_unique<Exception>(e.displayText(), ErrorCodes::POCO_EXCEPTION);
exception.emplace(Exception::CreateFromPoco, e);
}
catch (const Poco::Exception & e)
{
state.io.onException();
exception = std::make_unique<Exception>(e.displayText(), ErrorCodes::POCO_EXCEPTION);
exception.emplace(Exception::CreateFromPoco, e);
}
catch (const std::exception & e)
{
state.io.onException();
exception = std::make_unique<Exception>(e.what(), ErrorCodes::STD_EXCEPTION);
exception.emplace(Exception::CreateFromSTD, e);
}
catch (...)
{
state.io.onException();
exception = std::make_unique<Exception>("Unknown exception", ErrorCodes::UNKNOWN_EXCEPTION);
exception.emplace("Unknown exception", ErrorCodes::UNKNOWN_EXCEPTION);
}
try
@ -546,7 +546,7 @@ void TCPHandler::processOrdinaryQueryWithProcessors(size_t num_threads)
auto & pipeline = state.io.pipeline;
if (pipeline.getMaxThreads())
num_threads = pipeline.getMaxThreads();
num_threads = std::min(num_threads, pipeline.getMaxThreads());
/// Send header-block, to allow client to prepare output format for data to send.
{

7
dbms/programs/server/config.d/part_log.xml Normal file
View File

@ -0,0 +1,7 @@
<yandex>
<part_log>
<database>system</database>
<table>part_log</table>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</part_log>
</yandex>

23
dbms/programs/server/users.xml
View File

@ -83,30 +83,7 @@
<!-- Quota for user. -->
<quota>default</quota>
<!-- Example of row level security policy. -->
<!-- <databases>
<test>
<filtered_table1>
<filter>a = 1</filter>
</filtered_table1>
<filtered_table2>
<filter>a + b &lt; 1 or c - d &gt; 5</filter>
</filtered_table2>
</test>
</databases> -->
</default>
<!-- Example of user with readonly access. -->
<!-- <readonly>
<password></password>
<networks incl="networks" replace="replace">
<ip>::1</ip>
<ip>127.0.0.1</ip>
</networks>
<profile>readonly</profile>
<quota>default</quota>
</readonly> -->
</users>
<!-- Quotas. -->

37
dbms/src/Access/RowPolicyContextFactory.cpp
View File

@ -101,9 +101,6 @@ namespace
public:
void add(const ASTPtr & condition, bool is_restrictive)
{
if (!condition)
return;
if (is_restrictive)
restrictions.push_back(condition);
else
@ -139,29 +136,32 @@ void RowPolicyContextFactory::PolicyInfo::setPolicy(const RowPolicyPtr & policy_
for (auto index : ext::range_with_static_cast<ConditionIndex>(0, MAX_CONDITION_INDEX))
{
parsed_conditions[index] = nullptr;
const String & condition = policy->conditions[index];
if (condition.empty())
continue;
auto previous_range = std::pair(std::begin(policy->conditions), std::begin(policy->conditions) + index);
auto previous_it = std::find(previous_range.first, previous_range.second, condition);
if (previous_it != previous_range.second)
{
/// The condition is already parsed before.
parsed_conditions[index] = parsed_conditions[previous_it - previous_range.first];
continue;
}
else
/// Try to parse the condition.
try
{
/// Try to parse the condition.
try
{
ParserExpression parser;
parsed_conditions[index] = parseQuery(parser, condition, 0);
}
catch (...)
{
tryLogCurrentException(
&Poco::Logger::get("RowPolicy"),
String("Could not parse the condition ") + RowPolicy::conditionIndexToString(index) + " of row policy "
+ backQuote(policy->getFullName()));
}
ParserExpression parser;
parsed_conditions[index] = parseQuery(parser, condition, 0);
}
catch (...)
{
tryLogCurrentException(
&Poco::Logger::get("RowPolicy"),
String("Could not parse the condition ") + RowPolicy::conditionIndexToString(index) + " of row policy "
+ backQuote(policy->getFullName()));
}
}
}
@ -290,7 +290,8 @@ void RowPolicyContextFactory::mixConditionsForContext(RowPolicyContext & context
auto & mixers = map_of_mixers[std::pair{policy.getDatabase(), policy.getTableName()}];
mixers.policy_ids.push_back(policy_id);
for (auto index : ext::range(0, MAX_CONDITION_INDEX))
mixers.mixers[index].add(info.parsed_conditions[index], policy.isRestrictive());
if (info.parsed_conditions[index])
mixers.mixers[index].add(info.parsed_conditions[index], policy.isRestrictive());
}
}

20
dbms/src/Access/UsersConfigAccessStorage.cpp
View File

@ -135,13 +135,25 @@ namespace
for (const String & database : databases)
{
const String database_config = databases_config + "." + database;
Poco::Util::AbstractConfiguration::Keys table_names;
config.keys(database_config, table_names);
Poco::Util::AbstractConfiguration::Keys keys_in_database_config;
config.keys(database_config, keys_in_database_config);
/// Read table properties
for (const String & table_name : table_names)
for (const String & key_in_database_config : keys_in_database_config)
{
const auto filter_config = database_config + "." + table_name + ".filter";
String table_name = key_in_database_config;
String filter_config = database_config + "." + table_name + ".filter";
if (key_in_database_config.starts_with("table["))
{
const auto table_name_config = database_config + "." + table_name + "[@name]";
if (config.has(table_name_config))
{
table_name = config.getString(table_name_config);
filter_config = database_config + ".table[@name='" + table_name + "']";
}
}
if (config.has(filter_config))
{
try

4
dbms/src/Client/Connection.cpp
View File

@ -774,9 +774,7 @@ std::unique_ptr<Exception> Connection::receiveException()
{
//LOG_TRACE(log_wrapper.get(), "Receiving exception");
Exception e;
readException(e, *in, "Received from " + getDescription());
return std::unique_ptr<Exception>{ e.clone() };
return std::make_unique<Exception>(readException(*in, "Received from " + getDescription()));
}

2
dbms/src/Common/ErrorCodes.cpp
View File

@ -138,7 +138,6 @@ namespace ErrorCodes
extern const int FUNCTION_IS_SPECIAL = 129;
extern const int CANNOT_READ_ARRAY_FROM_TEXT = 130;
extern const int TOO_LARGE_STRING_SIZE = 131;
extern const int CANNOT_CREATE_TABLE_FROM_METADATA = 132;
extern const int AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS = 133;
extern const int PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS = 134;
extern const int ZERO_ARRAY_OR_TUPLE_INDEX = 135;
@ -474,7 +473,6 @@ namespace ErrorCodes
extern const int NOT_ENOUGH_PRIVILEGES = 497;
extern const int LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED = 498;
extern const int S3_ERROR = 499;
extern const int CANNOT_CREATE_DICTIONARY_FROM_METADATA = 500;
extern const int CANNOT_CREATE_DATABASE = 501;
extern const int CANNOT_SIGQUEUE = 502;
extern const int AGGREGATE_FUNCTION_THROW = 503;

60
dbms/src/Common/Exception.cpp
View File

@ -23,8 +23,60 @@ namespace ErrorCodes
extern const int UNKNOWN_EXCEPTION;
extern const int CANNOT_TRUNCATE_FILE;
extern const int NOT_IMPLEMENTED;
extern const int LOGICAL_ERROR;
}
Exception::Exception()
{
}
Exception::Exception(const std::string & msg, int code)
: Poco::Exception(msg, code)
{
// In debug builds, treat LOGICAL_ERROR as an assertion failure.
assert(code != ErrorCodes::LOGICAL_ERROR);
}
Exception::Exception(CreateFromPocoTag, const Poco::Exception & exc)
: Poco::Exception(exc.displayText(), ErrorCodes::POCO_EXCEPTION)
{
#ifdef STD_EXCEPTION_HAS_STACK_TRACE
set_stack_trace(exc.get_stack_trace_frames(), exc.get_stack_trace_size());
#endif
}
Exception::Exception(CreateFromSTDTag, const std::exception & exc)
: Poco::Exception(String(typeid(exc).name()) + ": " + String(exc.what()), ErrorCodes::STD_EXCEPTION)
{
#ifdef STD_EXCEPTION_HAS_STACK_TRACE
set_stack_trace(exc.get_stack_trace_frames(), exc.get_stack_trace_size());
#endif
}
std::string getExceptionStackTraceString(const std::exception & e)
{
#ifdef STD_EXCEPTION_HAS_STACK_TRACE
return StackTrace::toString(e.get_stack_trace_frames(), 0, e.get_stack_trace_size());
#else
if (const auto * db_exception = dynamic_cast<const Exception *>(&e))
return db_exception->getStackTraceString();
return {};
#endif
}
std::string Exception::getStackTraceString() const
{
#ifdef STD_EXCEPTION_HAS_STACK_TRACE
return StackTrace::toString(get_stack_trace_frames(), 0, get_stack_trace_size());
#else
return trace.toString();
#endif
}
std::string errnoToString(int code, int e)
{
const size_t buf_size = 128;
@ -141,6 +193,7 @@ std::string getCurrentExceptionMessage(bool with_stacktrace, bool check_embedded
{
stream << "Poco::Exception. Code: " << ErrorCodes::POCO_EXCEPTION << ", e.code() = " << e.code()
<< ", e.displayText() = " << e.displayText()
<< (with_stacktrace ? getExceptionStackTraceString(e) : "")
<< (with_extra_info ? getExtraExceptionInfo(e) : "")
<< " (version " << VERSION_STRING << VERSION_OFFICIAL;
}
@ -157,8 +210,9 @@ std::string getCurrentExceptionMessage(bool with_stacktrace, bool check_embedded
name += " (demangling status: " + toString(status) + ")";
stream << "std::exception. Code: " << ErrorCodes::STD_EXCEPTION << ", type: " << name << ", e.what() = " << e.what()
<< (with_extra_info ? getExtraExceptionInfo(e) : "")
<< ", version = " << VERSION_STRING << VERSION_OFFICIAL;
<< (with_stacktrace ? getExceptionStackTraceString(e) : "")
<< (with_extra_info ? getExtraExceptionInfo(e) : "")
<< ", version = " << VERSION_STRING << VERSION_OFFICIAL;
}
catch (...) {}
}
@ -261,7 +315,7 @@ std::string getExceptionMessage(const Exception & e, bool with_stacktrace, bool
stream << "Code: " << e.code() << ", e.displayText() = " << text;
if (with_stacktrace && !has_embedded_stack_trace)
stream << ", Stack trace (when copying this message, always include the lines below):\n\n" << e.getStackTrace().toString();
stream << ", Stack trace (when copying this message, always include the lines below):\n\n" << e.getStackTraceString();
}
catch (...) {}

18
dbms/src/Common/Exception.h
View File

@ -22,13 +22,14 @@ namespace ErrorCodes
class Exception : public Poco::Exception
{
public:
Exception() {} /// For deferred initialization.
Exception(const std::string & msg, int code) : Poco::Exception(msg, code) {}
Exception(const std::string & msg, const Exception & nested_exception, int code)
: Poco::Exception(msg, nested_exception, code), trace(nested_exception.trace) {}
Exception();
Exception(const std::string & msg, int code);
enum CreateFromPocoTag { CreateFromPoco };
Exception(CreateFromPocoTag, const Poco::Exception & exc) : Poco::Exception(exc.displayText(), ErrorCodes::POCO_EXCEPTION) {}
enum CreateFromSTDTag { CreateFromSTD };
Exception(CreateFromPocoTag, const Poco::Exception & exc);
Exception(CreateFromSTDTag, const std::exception & exc);
Exception * clone() const override { return new Exception(*this); }
void rethrow() const override { throw *this; }
@ -38,15 +39,20 @@ public:
/// Add something to the existing message.
void addMessage(const std::string & arg) { extendedMessage(arg); }
const StackTrace & getStackTrace() const { return trace; }
std::string getStackTraceString() const;
private:
#ifndef STD_EXCEPTION_HAS_STACK_TRACE
StackTrace trace;
#endif
const char * className() const throw() override { return "DB::Exception"; }
};
std::string getExceptionStackTraceString(const std::exception & e);
/// Contains an additional member `saved_errno`. See the throwFromErrno function.
class ErrnoException : public Exception
{

33
dbms/src/Common/LRUCache.h
View File

@ -23,11 +23,10 @@ struct TrivialWeightFunction
};
/// Thread-safe cache that evicts entries which are not used for a long time or are expired.
/// Thread-safe cache that evicts entries which are not used for a long time.
/// WeightFunction is a functor that takes Mapped as a parameter and returns "weight" (approximate size)
/// of that value.
/// Cache starts to evict entries when their total weight exceeds max_size and when expiration time of these
/// entries is due.
/// Cache starts to evict entries when their total weight exceeds max_size.
/// Value weight should not change after insertion.
template <typename TKey, typename TMapped, typename HashFunction = std::hash<TKey>, typename WeightFunction = TrivialWeightFunction<TMapped>>
class LRUCache
@ -36,15 +35,13 @@ public:
using Key = TKey;
using Mapped = TMapped;
using MappedPtr = std::shared_ptr<Mapped>;
using Delay = std::chrono::seconds;
private:
using Clock = std::chrono::steady_clock;
using Timestamp = Clock::time_point;
public:
LRUCache(size_t max_size_, const Delay & expiration_delay_ = Delay::zero())
: max_size(std::max(static_cast<size_t>(1), max_size_)), expiration_delay(expiration_delay_) {}
LRUCache(size_t max_size_)
: max_size(std::max(static_cast<size_t>(1), max_size_)) {}
MappedPtr get(const Key & key)
{
@ -167,16 +164,9 @@ protected:
struct Cell
{
bool expired(const Timestamp & last_timestamp, const Delay & delay) const
{
return (delay == Delay::zero()) ||
((last_timestamp > timestamp) && ((last_timestamp - timestamp) > delay));
}
MappedPtr value;
size_t size;
LRUQueueIterator queue_iterator;
Timestamp timestamp;
};
using Cells = std::unordered_map<Key, Cell, HashFunction>;
@ -257,7 +247,6 @@ private:
/// Total weight of values.
size_t current_size = 0;
const size_t max_size;
const Delay expiration_delay;
std::atomic<size_t> hits {0};
std::atomic<size_t> misses {0};
@ -273,7 +262,6 @@ private:
}
Cell & cell = it->second;
updateCellTimestamp(cell);
/// Move the key to the end of the queue. The iterator remains valid.
queue.splice(queue.end(), queue, cell.queue_iterator);
@ -303,18 +291,11 @@ private:
cell.value = mapped;
cell.size = cell.value ? weight_function(*cell.value) : 0;
current_size += cell.size;
updateCellTimestamp(cell);
removeOverflow(cell.timestamp);
removeOverflow();
}
void updateCellTimestamp(Cell & cell)
{
if (expiration_delay != Delay::zero())
cell.timestamp = Clock::now();
}
void removeOverflow(const Timestamp & last_timestamp)
void removeOverflow()
{
size_t current_weight_lost = 0;
size_t queue_size = cells.size();
@ -330,8 +311,6 @@ private:
}
const auto & cell = it->second;
if (!cell.expired(last_timestamp, expiration_delay))
break;
current_size -= cell.size;
current_weight_lost += cell.size;

2
dbms/src/Common/ProfileEvents.cpp
View File

@ -37,6 +37,8 @@
M(CreatedReadBufferOrdinary, "") \
M(CreatedReadBufferAIO, "") \
M(CreatedReadBufferAIOFailed, "") \
M(CreatedReadBufferMMap, "") \
M(CreatedReadBufferMMapFailed, "") \
M(CreatedWriteBufferOrdinary, "") \
M(CreatedWriteBufferAIO, "") \
M(CreatedWriteBufferAIOFailed, "") \

14
dbms/src/Common/StackTrace.cpp
View File

@ -4,6 +4,7 @@
#include <Common/Elf.h>
#include <Common/SymbolIndex.h>
#include <Common/config.h>
#include <Common/MemorySanitizer.h>
#include <common/SimpleCache.h>
#include <common/demangle.h>
#include <Core/Defines.h>
@ -226,6 +227,7 @@ void StackTrace::tryCapture()
size = 0;
#if USE_UNWIND
size = unw_backtrace(frames.data(), capacity);
__msan_unpoison(frames.data(), size * sizeof(frames[0]));
#endif
}
@ -328,3 +330,15 @@ std::string StackTrace::toString() const
static SimpleCache<decltype(toStringImpl), &toStringImpl> func_cached;
return func_cached(frames, offset, size);
}
std::string StackTrace::toString(void ** frames_, size_t offset, size_t size)
{
__msan_unpoison(frames_, size * sizeof(*frames_));
StackTrace::Frames frames_copy{};
for (size_t i = 0; i < size; ++i)
frames_copy[i] = frames_[i];
static SimpleCache<decltype(toStringImpl), &toStringImpl> func_cached;
return func_cached(frames_copy, offset, size);
}

2
dbms/src/Common/StackTrace.h
View File

@ -41,6 +41,8 @@ public:
const Frames & getFrames() const;
std::string toString() const;
static std::string toString(void ** frames, size_t offset, size_t size);
void toStringEveryLine(std::function<void(const std::string &)> callback) const;
protected:

3
dbms/src/Common/tests/CMakeLists.txt
View File

@ -13,9 +13,6 @@ target_link_libraries (sip_hash_perf PRIVATE clickhouse_common_io)
add_executable (auto_array auto_array.cpp)
target_link_libraries (auto_array PRIVATE clickhouse_common_io)
add_executable (lru_cache lru_cache.cpp)
target_link_libraries (lru_cache PRIVATE clickhouse_common_io)
add_executable (hash_table hash_table.cpp)
target_link_libraries (hash_table PRIVATE clickhouse_common_io)

317
dbms/src/Common/tests/lru_cache.cpp
View File

@ -1,317 +0,0 @@
#include <Common/LRUCache.h>
#include <Common/Exception.h>
#include <iostream>
#include <string>
#include <thread>
#include <chrono>
#include <functional>
namespace
{
void run();
void runTest(unsigned int num, const std::function<bool()> & func);
bool test1();
bool test2();
bool test_concurrent();
#define ASSERT_CHECK(cond, res) \
do \
{ \
if (!(cond)) \
{ \
std::cout << __FILE__ << ":" << __LINE__ << ":" \
<< "Assertion " << #cond << " failed.\n"; \
if ((res)) { (res) = false; } \
} \
} \
while (0)
void run()
{
const std::vector<std::function<bool()>> tests =
{
test1,
test2,
test_concurrent
};
unsigned int num = 0;
for (const auto & test : tests)
{
++num;
runTest(num, test);
}
}
void runTest(unsigned int num, const std::function<bool()> & func)
{
bool ok;
try
{
ok = func();
}
catch (const DB::Exception & ex)
{
ok = false;
std::cout << "Caught exception " << ex.displayText() << "\n";
}
catch (const std::exception & ex)
{
ok = false;
std::cout << "Caught exception " << ex.what() << "\n";
}
catch (...)
{
ok = false;
std::cout << "Caught unhandled exception\n";
}
if (ok)
std::cout << "Test " << num << " passed\n";
else
std::cout << "Test " << num << " failed\n";
}
struct Weight
{
size_t operator()(const std::string & s) const
{
return s.size();
}
};
bool test1()
{
using Cache = DB::LRUCache<std::string, std::string, std::hash<std::string>, Weight>;
using MappedPtr = Cache::MappedPtr;
auto ptr = [](const std::string & s)
{
return MappedPtr(new std::string(s));
};
Cache cache(10);
bool res = true;
ASSERT_CHECK(!cache.get("asd"), res);
cache.set("asd", ptr("qwe"));
ASSERT_CHECK((*cache.get("asd") == "qwe"), res);
cache.set("zxcv", ptr("12345"));
cache.set("01234567891234567", ptr("--"));
ASSERT_CHECK((*cache.get("zxcv") == "12345"), res);
ASSERT_CHECK((*cache.get("asd") == "qwe"), res);
ASSERT_CHECK((*cache.get("01234567891234567") == "--"), res);
ASSERT_CHECK(!cache.get("123x"), res);
cache.set("321x", ptr("+"));
ASSERT_CHECK(!cache.get("zxcv"), res);
ASSERT_CHECK((*cache.get("asd") == "qwe"), res);
ASSERT_CHECK((*cache.get("01234567891234567") == "--"), res);
ASSERT_CHECK(!cache.get("123x"), res);
ASSERT_CHECK((*cache.get("321x") == "+"), res);
ASSERT_CHECK((cache.weight() == 6), res);
ASSERT_CHECK((cache.count() == 3), res);
return res;
}
bool test2()
{
using namespace std::literals;
using Cache = DB::LRUCache<std::string, std::string, std::hash<std::string>, Weight>;
using MappedPtr = Cache::MappedPtr;
auto ptr = [](const std::string & s)
{
return MappedPtr(new std::string(s));
};
Cache cache(10, 3s);
bool res = true;
ASSERT_CHECK(!cache.get("asd"), res);
cache.set("asd", ptr("qwe"));
ASSERT_CHECK((*cache.get("asd") == "qwe"), res);
cache.set("zxcv", ptr("12345"));
cache.set("01234567891234567", ptr("--"));
ASSERT_CHECK((*cache.get("zxcv") == "12345"), res);
ASSERT_CHECK((*cache.get("asd") == "qwe"), res);
ASSERT_CHECK((*cache.get("01234567891234567") == "--"), res);
ASSERT_CHECK(!cache.get("123x"), res);
cache.set("321x", ptr("+"));
ASSERT_CHECK((cache.get("zxcv")), res);
ASSERT_CHECK((*cache.get("asd") == "qwe"), res);
ASSERT_CHECK((*cache.get("01234567891234567") == "--"), res);
ASSERT_CHECK(!cache.get("123x"), res);
ASSERT_CHECK((*cache.get("321x") == "+"), res);
ASSERT_CHECK((cache.weight() == 11), res);
ASSERT_CHECK((cache.count() == 4), res);
std::this_thread::sleep_for(5s);
cache.set("123x", ptr("2769"));
ASSERT_CHECK(!cache.get("zxcv"), res);
ASSERT_CHECK((*cache.get("asd") == "qwe"), res);
ASSERT_CHECK((*cache.get("01234567891234567") == "--"), res);
ASSERT_CHECK((*cache.get("321x") == "+"), res);
ASSERT_CHECK((cache.weight() == 10), res);
ASSERT_CHECK((cache.count() == 4), res);
return res;
}
bool test_concurrent()
{
using namespace std::literals;
using Cache = DB::LRUCache<std::string, std::string, std::hash<std::string>, Weight>;
Cache cache(2);
bool res = true;
auto load_func = [](const std::string & result, std::chrono::seconds sleep_for, bool throw_exc)
{
std::this_thread::sleep_for(sleep_for);
if (throw_exc)
throw std::runtime_error("Exception!");
return std::make_shared<std::string>(result);
};
/// Case 1: Both threads are able to load the value.
std::pair<Cache::MappedPtr, bool> result1;
std::thread thread1([&]()
{
result1 = cache.getOrSet("key", [&]() { return load_func("val1", 1s, false); });
});
std::pair<Cache::MappedPtr, bool> result2;
std::thread thread2([&]()
{
result2 = cache.getOrSet("key", [&]() { return load_func("val2", 1s, false); });
});
thread1.join();
thread2.join();
ASSERT_CHECK((result1.first == result2.first), res);
ASSERT_CHECK((result1.second != result2.second), res);
/// Case 2: One thread throws an exception during loading.
cache.reset();
bool thrown = false;
thread1 = std::thread([&]()
{
try
{
cache.getOrSet("key", [&]() { return load_func("val1", 2s, true); });
}
catch (...)
{
thrown = true;
}
});
thread2 = std::thread([&]()
{
std::this_thread::sleep_for(1s);
result2 = cache.getOrSet("key", [&]() { return load_func("val2", 1s, false); });
});
thread1.join();
thread2.join();
ASSERT_CHECK((thrown == true), res);
ASSERT_CHECK((result2.second == true), res);
ASSERT_CHECK((result2.first.get() == cache.get("key").get()), res);
ASSERT_CHECK((*result2.first == "val2"), res);
/// Case 3: All threads throw an exception.
cache.reset();
bool thrown1 = false;
thread1 = std::thread([&]()
{
try
{
cache.getOrSet("key", [&]() { return load_func("val1", 1s, true); });
}
catch (...)
{
thrown1 = true;
}
});
bool thrown2 = false;
thread2 = std::thread([&]()
{
try
{
cache.getOrSet("key", [&]() { return load_func("val1", 1s, true); });
}
catch (...)
{
thrown2 = true;
}
});
thread1.join();
thread2.join();
ASSERT_CHECK((thrown1 == true), res);
ASSERT_CHECK((thrown2 == true), res);
ASSERT_CHECK((cache.get("key") == nullptr), res);
/// Case 4: Concurrent reset.
cache.reset();
thread1 = std::thread([&]()
{
result1 = cache.getOrSet("key", [&]() { return load_func("val1", 2s, false); });
});
std::this_thread::sleep_for(1s);
cache.reset();
thread1.join();
ASSERT_CHECK((result1.second == true), res);
ASSERT_CHECK((*result1.first == "val1"), res);
ASSERT_CHECK((cache.get("key") == nullptr), res);
return res;
}
}
int main()
{
run();
return 0;
}

7
dbms/src/Compression/CachedCompressedReadBuffer.cpp
View File

@ -19,7 +19,7 @@ void CachedCompressedReadBuffer::initInput()
{
if (!file_in)
{
file_in = createReadBufferFromFileBase(path, estimated_size, aio_threshold, buf_size);
file_in = createReadBufferFromFileBase(path, estimated_size, aio_threshold, mmap_threshold, buf_size);
compressed_in = file_in.get();
if (profile_callback)
@ -73,10 +73,11 @@ bool CachedCompressedReadBuffer::nextImpl()
CachedCompressedReadBuffer::CachedCompressedReadBuffer(
const std::string & path_, UncompressedCache * cache_, size_t estimated_size_, size_t aio_threshold_,
const std::string & path_, UncompressedCache * cache_,
size_t estimated_size_, size_t aio_threshold_, size_t mmap_threshold_,
size_t buf_size_)
: ReadBuffer(nullptr, 0), path(path_), cache(cache_), buf_size(buf_size_), estimated_size(estimated_size_),
aio_threshold(aio_threshold_), file_pos(0)
aio_threshold(aio_threshold_), mmap_threshold(mmap_threshold_), file_pos(0)
{
}

4
dbms/src/Compression/CachedCompressedReadBuffer.h
View File

@ -26,6 +26,7 @@ private:
size_t buf_size;
size_t estimated_size;
size_t aio_threshold;
size_t mmap_threshold;
std::unique_ptr<ReadBufferFromFileBase> file_in;
size_t file_pos;
@ -42,7 +43,8 @@ private:
public:
CachedCompressedReadBuffer(
const std::string & path_, UncompressedCache * cache_, size_t estimated_size_, size_t aio_threshold_,
const std::string & path_, UncompressedCache * cache_,
size_t estimated_size_, size_t aio_threshold_, size_t mmap_threshold_,
size_t buf_size_ = DBMS_DEFAULT_BUFFER_SIZE);

4
dbms/src/Compression/CompressedReadBufferFromFile.cpp
View File

@ -33,9 +33,9 @@ bool CompressedReadBufferFromFile::nextImpl()
CompressedReadBufferFromFile::CompressedReadBufferFromFile(
const std::string & path, size_t estimated_size, size_t aio_threshold, size_t buf_size)
const std::string & path, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, size_t buf_size)
: BufferWithOwnMemory<ReadBuffer>(0),
p_file_in(createReadBufferFromFileBase(path, estimated_size, aio_threshold, buf_size)),
p_file_in(createReadBufferFromFileBase(path, estimated_size, aio_threshold, mmap_threshold, buf_size)),
file_in(*p_file_in)
{
compressed_in = &file_in;

2
dbms/src/Compression/CompressedReadBufferFromFile.h
View File

@ -30,7 +30,7 @@ private:
public:
CompressedReadBufferFromFile(
const std::string & path, size_t estimated_size, size_t aio_threshold, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE);
const std::string & path, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE);
void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block);

87
dbms/src/Compression/CompressionCodecDoubleDelta.cpp
View File

@ -26,7 +26,7 @@ extern const int CANNOT_DECOMPRESS;
namespace
{
Int64 getMaxValueForByteSize(UInt8 byte_size)
inline Int64 getMaxValueForByteSize(Int8 byte_size)
{
switch (byte_size)
{
@ -51,11 +51,56 @@ struct WriteSpec
const UInt8 data_bits;
};
const std::array<UInt8, 5> DELTA_SIZES{7, 9, 12, 32, 64};
// delta size prefix and data lengths based on few high bits peeked from binary stream
static const WriteSpec WRITE_SPEC_LUT[32] = {
// 0b0 - 1-bit prefix, no data to read
/* 00000 */ {1, 0b0, 0},
/* 00001 */ {1, 0b0, 0},
/* 00010 */ {1, 0b0, 0},
/* 00011 */ {1, 0b0, 0},
/* 00100 */ {1, 0b0, 0},
/* 00101 */ {1, 0b0, 0},
/* 00110 */ {1, 0b0, 0},
/* 00111 */ {1, 0b0, 0},
/* 01000 */ {1, 0b0, 0},
/* 01001 */ {1, 0b0, 0},
/* 01010 */ {1, 0b0, 0},
/* 01011 */ {1, 0b0, 0},
/* 01100 */ {1, 0b0, 0},
/* 01101 */ {1, 0b0, 0},
/* 01110 */ {1, 0b0, 0},
/* 01111 */ {1, 0b0, 0},
// 0b10 - 2 bit prefix, 7 bits of data
/* 10000 */ {2, 0b10, 7},
/* 10001 */ {2, 0b10, 7},
/* 10010 */ {2, 0b10, 7},
/* 10011 */ {2, 0b10, 7},
/* 10100 */ {2, 0b10, 7},
/* 10101 */ {2, 0b10, 7},
/* 10110 */ {2, 0b10, 7},
/* 10111 */ {2, 0b10, 7},
// 0b110 - 3 bit prefix, 9 bits of data
/* 11000 */ {3, 0b110, 9},
/* 11001 */ {3, 0b110, 9},
/* 11010 */ {3, 0b110, 9},
/* 11011 */ {3, 0b110, 9},
// 0b1110 - 4 bit prefix, 12 bits of data
/* 11100 */ {4, 0b1110, 12},
/* 11101 */ {4, 0b1110, 12},
// 5-bit prefixes
/* 11110 */ {5, 0b11110, 32},
/* 11111 */ {5, 0b11111, 64},
};
template <typename T>
WriteSpec getDeltaWriteSpec(const T & value)
{
// TODO: to speed up things a bit by counting number of leading zeroes instead of doing lots of comparisons
if (value > -63 && value < 64)
{
return WriteSpec{2, 0b10, 7};
@ -107,14 +152,15 @@ UInt32 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size)
template <typename ValueType>
UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest)
{
// Since only unsinged int has granted 2-compliment overflow handling, we are doing math here on unsigned types.
// To simplify and booletproof code, we operate enforce ValueType to be unsigned too.
// Since only unsinged int has granted 2-complement overflow handling,
// we are doing math here only on unsigned types.
// To simplify and booletproof code, we enforce ValueType to be unsigned too.
static_assert(is_unsigned_v<ValueType>, "ValueType must be unsigned.");
using UnsignedDeltaType = ValueType;
// We use signed delta type to turn huge unsigned values into smaller signed:
// ffffffff => -1
using SignedDeltaType = typename std::make_signed<UnsignedDeltaType>::type;
using SignedDeltaType = typename std::make_signed_t<UnsignedDeltaType>;
if (source_size % sizeof(ValueType) != 0)
throw Exception("Cannot compress, data size " + toString(source_size)
@ -149,8 +195,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest)
prev_value = curr_value;
}
WriteBuffer buffer(dest, getCompressedDataSize(sizeof(ValueType), source_size - sizeof(ValueType)*2));
BitWriter writer(buffer);
BitWriter writer(dest, getCompressedDataSize(sizeof(ValueType), source_size - sizeof(ValueType)*2));
int item = 2;
for (; source < source_end; source += sizeof(ValueType), ++item)
@ -170,7 +215,8 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest)
else
{
const SignedDeltaType signed_dd = static_cast<SignedDeltaType>(double_delta);
const auto sign = std::signbit(signed_dd);
const auto sign = signed_dd < 0;
// -1 shirnks dd down to fit into number of bits, and there can't be 0, so it is OK.
const auto abs_value = static_cast<UnsignedDeltaType>(std::abs(signed_dd) - 1);
const auto write_spec = getDeltaWriteSpec(signed_dd);
@ -183,7 +229,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest)
writer.flush();
return sizeof(items_count) + sizeof(prev_value) + sizeof(prev_delta) + buffer.count();
return sizeof(items_count) + sizeof(prev_value) + sizeof(prev_delta) + writer.count() / 8;
}
template <typename ValueType>
@ -220,35 +266,28 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest)
dest += sizeof(prev_value);
}
ReadBufferFromMemory buffer(source, source_size - sizeof(prev_value) - sizeof(prev_delta) - sizeof(items_count));
BitReader reader(buffer);
BitReader reader(source, source_size - sizeof(prev_value) - sizeof(prev_delta) - sizeof(items_count));
// since data is tightly packed, up to 1 bit per value, and last byte is padded with zeroes,
// we have to keep track of items to avoid reading more that there is.
for (UInt32 items_read = 2; items_read < items_count && !reader.eof(); ++items_read)
{
UnsignedDeltaType double_delta = 0;
if (reader.readBit() == 1)
{
UInt8 i = 0;
for (; i < sizeof(DELTA_SIZES) - 1; ++i)
{
const auto next_bit = reader.readBit();
if (next_bit == 0)
{
break;
}
}
static_assert(sizeof(WRITE_SPEC_LUT)/sizeof(WRITE_SPEC_LUT[0]) == 32); // 5-bit prefix lookup table
const auto write_spec = WRITE_SPEC_LUT[reader.peekByte() >> (8 - 5)]; // only 5 high bits of peeked byte value
reader.skipBufferedBits(write_spec.prefix_bits); // discard the prefix value, since we've already used it
if (write_spec.data_bits != 0)
{
const UInt8 sign = reader.readBit();
SignedDeltaType signed_dd = static_cast<SignedDeltaType>(reader.readBits(DELTA_SIZES[i] - 1) + 1);
SignedDeltaType signed_dd = static_cast<SignedDeltaType>(reader.readBits(write_spec.data_bits - 1) + 1);
if (sign)
{
signed_dd *= -1;
}
double_delta = static_cast<UnsignedDeltaType>(signed_dd);
}
// else if first bit is zero, no need to read more data.
const UnsignedDeltaType delta = double_delta + prev_delta;
const ValueType curr_value = prev_value + delta;

86
dbms/src/Compression/CompressionCodecDoubleDelta.h
View File

@ -5,6 +5,92 @@
namespace DB
{
/** DoubleDelta column codec implementation.
*
* Based on Gorilla paper: http://www.vldb.org/pvldb/vol8/p1816-teller.pdf, which was extended
* to support 64bit types. The drawback is 1 extra bit for 32-byte wide deltas: 5-bit prefix
* instead of 4-bit prefix.
*
* This codec is best used against monotonic integer sequences with constant (or almost contant)
* stride, like event timestamp for some monitoring application.
*
* Given input sequence a: [a0, a1, ... an]:
*
* First, write number of items (sizeof(int32)*8 bits): n
* Then write first item as is (sizeof(a[0])*8 bits): a[0]
* Second item is written as delta (sizeof(a[0])*8 bits): a[1] - a[0]
* Loop over remaining items and calculate double delta:
* double_delta = a[i] - 2 * a[i - 1] + a[i - 2]
* Write it in compact binary form with `BitWriter`
* if double_delta == 0:
* write 1bit: 0
* else if -63 < double_delta < 64:
* write 2 bit prefix: 10
* write sign bit (1 if signed): x
* write 7-1 bits of abs(double_delta - 1): xxxxxx
* else if -255 < double_delta < 256:
* write 3 bit prefix: 110
* write sign bit (1 if signed): x
* write 9-1 bits of abs(double_delta - 1): xxxxxxxx
* else if -2047 < double_delta < 2048:
* write 4 bit prefix: 1110
* write sign bit (1 if signed): x
* write 12-1 bits of abs(double_delta - 1): xxxxxxxxxxx
* else if double_delta fits into 32-bit int:
* write 5 bit prefix: 11110
* write sign bit (1 if signed): x
* write 32-1 bits of abs(double_delta - 1): xxxxxxxxxxx...
* else
* write 5 bit prefix: 11111
* write sign bit (1 if signed): x
* write 64-1 bits of abs(double_delta - 1): xxxxxxxxxxx...
*
* @example sequence of UInt8 values [1, 2, 3, 4, 5, 6, 7, 8, 9 10] is encoded as (codec header is ommited):
*
* .- 4-byte little-endian sequence length (10 == 0xa)
* | .- 1 byte (sizeof(UInt8) a[0] : 0x01
* | | .- 1 byte of delta: a[1] - a[0] = 2 - 1 = 1 : 0x01
* | | | .- 8 zero bits since double delta for remaining 8 elements was 0 : 0x00
* v_______________v___v___v___
* \x0a\x00\x00\x00\x01\x01\x00
*
* @example sequence of Int16 values [-10, 10, -20, 20, -40, 40] is encoded as:
*
* .- 4-byte little endian sequence length = 6 : 0x00000006
* | .- 2 bytes (sizeof(Int16) a[0] as UInt16 = -10 : 0xfff6
* | | .- 2 bytes of delta: a[1] - a[0] = 10 - (-10) = 20 : 0x0014
* | | | .- 4 encoded double deltas (see below)
* v_______________ v______ v______ v______________________
* \x06\x00\x00\x00\xf6\xff\x14\x00\xb8\xe2\x2e\xb1\xe4\x58
*
* 4 binary encoded double deltas (\xb8\xe2\x2e\xb1\xe4\x58):
* double_delta (DD) = -20 - 2 * 10 + (-10) = -50
* .- 2-bit prefix : 0b10
* | .- sign-bit : 0b1
* | |.- abs(DD - 1) = 49 : 0b110001
* | ||
* | || DD = 20 - 2 * (-20) + 10 = 70
* | || .- 3-bit prefix : 0b110
* | || | .- sign bit : 0b0
* | || | |.- abs(DD - 1) = 69 : 0b1000101
* | || | ||
* | || | || DD = -40 - 2 * 20 + (-20) = -100
* | || | || .- 3-bit prefix : 0b110
* | || | || | .- sign-bit : 0b0
* | || | || | |.- abs(DD - 1) = 99 : 0b1100011
* | || | || | ||
* | || | || | || DD = 40 - 2 * (-40) + 20 = 140
* | || | || | || .- 3-bit prefix : 0b110
* | || | || | || | .- sign bit : 0b0
* | || | || | || | |.- abs(DD - 1) = 139 : 0b10001011
* | || | || | || | ||
* V_vv______V__vv________V____vv_______V__vv________,- padding bits
* 10111000 11100010 00101110 10110001 11100100 01011000
*
* Please also see unit tests for:
* * Examples on what output `BitWriter` produces on predefined input.
* * Compatibility tests solidifying encoded binary output on set of predefined sequences.
*/
class CompressionCodecDoubleDelta : public ICompressionCodec
{
public:

8
dbms/src/Compression/CompressionCodecGorilla.cpp
View File

@ -112,8 +112,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest,
dest += sizeof(prev_value);
}
WriteBuffer buffer(dest, dest_end - dest);
BitWriter writer(buffer);
BitWriter writer(dest, dest_end - dest);
while (source < source_end)
{
@ -148,7 +147,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest,
writer.flush();
return sizeof(items_count) + sizeof(prev_value) + buffer.count();
return sizeof(items_count) + sizeof(prev_value) + writer.count() / 8;
}
template <typename T>
@ -174,8 +173,7 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest)
dest += sizeof(prev_value);
}
ReadBufferFromMemory buffer(source, source_size - sizeof(items_count) - sizeof(prev_value));
BitReader reader(buffer);
BitReader reader(source, source_size - sizeof(items_count) - sizeof(prev_value));
binary_value_info prev_xored_info{0, 0, 0};

83
dbms/src/Compression/CompressionCodecGorilla.h
View File

@ -5,6 +5,89 @@
namespace DB
{
/** Gorilla column codec implementation.
*
* Based on Gorilla paper: http://www.vldb.org/pvldb/vol8/p1816-teller.pdf
*
* This codec is best used against monotonic floating sequences, like CPU usage percentage
* or any other gauge.
*
* Given input sequence a: [a0, a1, ... an]
*
* First, write number of items (sizeof(int32)*8 bits): n
* Then write first item as is (sizeof(a[0])*8 bits): a[0]
* Loop over remaining items and calculate xor_diff:
* xor_diff = a[i] ^ a[i - 1] (e.g. 00000011'10110100)
* Write it in compact binary form with `BitWriter`
* if xor_diff == 0:
* write 1 bit: 0
* else:
* calculate leading zero bits (lzb)
* and trailing zero bits (tzb) of xor_diff,
* compare to lzb and tzb of previous xor_diff
* (X = sizeof(a[i]) * 8, e.g. X = 16, lzb = 6, tzb = 2)
* if lzb >= prev_lzb && tzb >= prev_tzb:
* (e.g. prev_lzb=4, prev_tzb=1)
* write 2 bit prefix: 0b10
* write xor_diff >> prev_tzb (X - prev_lzb - prev_tzb bits):0b00111011010
* (where X = sizeof(a[i]) * 8, e.g. 16)
* else:
* write 2 bit prefix: 0b11
* write 5 bits of lzb: 0b00110
* write 6 bits of (X - lzb - tzb)=(16-6-2)=8: 0b001000
* write (X - lzb - tzb) non-zero bits of xor_diff: 0b11101101
* prev_lzb = lzb
* prev_tzb = tzb
*
* @example sequence of Float32 values [0.1, 0.1, 0.11, 0.2, 0.1] is encoded as:
*
* .- 4-byte little endian sequence length: 5 : 0x00000005
* | .- 4 byte (sizeof(Float32) a[0] as UInt32 : -10 : 0xcdcccc3d
* | | .- 4 encoded xor diffs (see below)
* v_______________ v______________ v__________________________________________________
* \x05\x00\x00\x00\xcd\xcc\xcc\x3d\x6a\x5a\xd8\xb6\x3c\xcd\x75\xb1\x6c\x77\x00\x00\x00
*
* 4 binary encoded xor diffs (\x6a\x5a\xd8\xb6\x3c\xcd\x75\xb1\x6c\x77\x00\x00\x00):
*
* ...........................................
* a[i-1] = 00111101110011001100110011001101
* a[i] = 00111101110011001100110011001101
* xor_diff = 00000000000000000000000000000000
* .- 1-bit prefix : 0b0
* |
* | ...........................................
* | a[i-1] = 00111101110011001100110011001101
* ! a[i] = 00111101111000010100011110101110
* | xor_diff = 00000000001011011000101101100011
* | lzb = 10
* | tzb = 0
* |.- 2-bit prefix : 0b11
* || .- lzb (10) : 0b1010
* || | .- data length (32-10-0): 22 : 0b010110
* || | | .- data : 0b1011011000101101100011
* || | | |
* || | | | ...........................................
* || | | | a[i-1] = 00111101111000010100011110101110
* || | | | a[i] = 00111110010011001100110011001101
* || | | | xor_diff = 00000011101011011000101101100011
* || | | | .- 2-bit prefix : 0b11
* || | | | | .- lzb = 6 : 0b00110
* || | | | | | .- data length = (32 - 6) = 26 : 0b011010
* || | | | | | | .- data : 0b11101011011000101101100011
* || | | | | | | |
* || | | | | | | | ...........................................
* || | | | | | | | a[i-1] = 00111110010011001100110011001101
* || | | | | | | | a[i] = 00111101110011001100110011001101
* || | | | | | | | xor_diff = 00000011100000000000000000000000
* || | | | | | | | .- 2-bit prefix : 0b10
* || | | | | | | | | .- data : 0b11100000000000000000000000
* VV_v____ v_____v________________________V_v_____v______v____________________________V_v_____________________________
* 01101010 01011010 11011000 10110110 00111100 11001101 01110101 10110001 01101100 01110111 00000000 00000000 00000000
*
* Please also see unit tests for:
* * Examples on what output `BitWriter` produces on predefined input.
* * Compatibility tests solidifying encoded binary output on set of predefined sequences.
*/
class CompressionCodecGorilla : public ICompressionCodec
{
public:

4
dbms/src/Compression/tests/cached_compressed_read_buffer.cpp
View File

@ -32,7 +32,7 @@ int main(int argc, char ** argv)
{
Stopwatch watch;
CachedCompressedReadBuffer in(path, &cache, 0, 0);
CachedCompressedReadBuffer in(path, &cache, 0, 0, 0);
WriteBufferFromFile out("/dev/null");
copyData(in, out);
@ -44,7 +44,7 @@ int main(int argc, char ** argv)
{
Stopwatch watch;
CachedCompressedReadBuffer in(path, &cache, 0, 0);
CachedCompressedReadBuffer in(path, &cache, 0, 0, 0);
WriteBufferFromFile out("/dev/null");
copyData(in, out);

772
dbms/src/Compression/tests/gtest_compressionCodec.cpp
View File

File diff suppressed because it is too large Load Diff

5
dbms/src/Core/Settings.h
View File

@ -127,12 +127,11 @@ struct Settings : public SettingsCollection<Settings>
M(SettingUInt64, optimize_min_equality_disjunction_chain_length, 3, "The minimum length of the expression `expr = x1 OR ... expr = xN` for optimization ", 0) \
\
M(SettingUInt64, min_bytes_to_use_direct_io, 0, "The minimum number of bytes for reading the data with O_DIRECT option during SELECT queries execution. 0 - disabled.", 0) \
M(SettingUInt64, min_bytes_to_use_mmap_io, 0, "The minimum number of bytes for reading the data with mmap option during SELECT queries execution. 0 - disabled.", 0) \
\
M(SettingBool, force_index_by_date, 0, "Throw an exception if there is a partition key in a table, and it is not used.", 0) \
M(SettingBool, force_primary_key, 0, "Throw an exception if there is primary key in a table, and it is not used.", 0) \
\
M(SettingUInt64, mark_cache_min_lifetime, 10000, "If the maximum size of mark_cache is exceeded, delete only records older than mark_cache_min_lifetime seconds.", 0) \
\
M(SettingFloat, max_streams_to_max_threads_ratio, 1, "Allows you to use more sources than the number of threads - to more evenly distribute work across threads. It is assumed that this is a temporary solution, since it will be possible in the future to make the number of sources equal to the number of threads, but for each source to dynamically select available work for itself.", 0) \
M(SettingFloat, max_streams_multiplier_for_merge_tables, 5, "Ask more streams when reading from Merge table. Streams will be spread across tables that Merge table will use. This allows more even distribution of work across threads and especially helpful when merged tables differ in size.", 0) \
\
@ -383,6 +382,7 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, enable_scalar_subquery_optimization, true, "If it is set to true, prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once.", 0) \
M(SettingBool, optimize_trivial_count_query, true, "Process trivial 'SELECT count() FROM table' query from metadata.", 0) \
M(SettingUInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \
M(SettingBool, optimize_if_chain_to_miltiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \
\
/** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
\
@ -393,6 +393,7 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, allow_experimental_cross_to_join_conversion, true, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \
M(SettingBool, allow_experimental_data_skipping_indices, true, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \
M(SettingBool, merge_tree_uniform_read_distribution, true, "Obsolete setting, does nothing. Will be removed after 2020-05-20", 0) \
M(SettingUInt64, mark_cache_min_lifetime, 0, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \
DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS)

182
dbms/src/Core/SortCursor.h
View File

@ -22,8 +22,8 @@ namespace DB
*/
struct SortCursorImpl
{
ColumnRawPtrs all_columns;
ColumnRawPtrs sort_columns;
ColumnRawPtrs all_columns;
SortDescription desc;
size_t sort_columns_size = 0;
size_t pos = 0;
@ -110,21 +110,52 @@ using SortCursorImpls = std::vector<SortCursorImpl>;
/// For easy copying.
struct SortCursor
template <typename Derived>
struct SortCursorHelper
{
SortCursorImpl * impl;
SortCursor(SortCursorImpl * impl_) : impl(impl_) {}
const Derived & derived() const { return static_cast<const Derived &>(*this); }
SortCursorHelper(SortCursorImpl * impl_) : impl(impl_) {}
SortCursorImpl * operator-> () { return impl; }
const SortCursorImpl * operator-> () const { return impl; }
bool ALWAYS_INLINE greater(const SortCursorHelper & rhs) const
{
return derived().greaterAt(rhs.derived(), impl->pos, rhs.impl->pos);
}
/// Inverted so that the priority queue elements are removed in ascending order.
bool ALWAYS_INLINE operator< (const SortCursorHelper & rhs) const
{
return derived().greater(rhs.derived());
}
/// Checks that all rows in the current block of this cursor are less than or equal to all the rows of the current block of another cursor.
bool ALWAYS_INLINE totallyLessOrEquals(const SortCursorHelper & rhs) const
{
if (impl->rows == 0 || rhs.impl->rows == 0)
return false;
/// The last row of this cursor is no larger than the first row of the another cursor.
return !derived().greaterAt(rhs.derived(), impl->rows - 1, 0);
}
};
struct SortCursor : SortCursorHelper<SortCursor>
{
using SortCursorHelper<SortCursor>::SortCursorHelper;
/// The specified row of this cursor is greater than the specified row of another cursor.
bool greaterAt(const SortCursor & rhs, size_t lhs_pos, size_t rhs_pos) const
bool ALWAYS_INLINE greaterAt(const SortCursor & rhs, size_t lhs_pos, size_t rhs_pos) const
{
for (size_t i = 0; i < impl->sort_columns_size; ++i)
{
int direction = impl->desc[i].direction;
int nulls_direction = impl->desc[i].nulls_direction;
const auto & desc = impl->desc[i];
int direction = desc.direction;
int nulls_direction = desc.nulls_direction;
int res = direction * impl->sort_columns[i]->compareAt(lhs_pos, rhs_pos, *(rhs.impl->sort_columns[i]), nulls_direction);
if (res > 0)
return true;
@ -133,45 +164,37 @@ struct SortCursor
}
return impl->order > rhs.impl->order;
}
};
/// Checks that all rows in the current block of this cursor are less than or equal to all the rows of the current block of another cursor.
bool totallyLessOrEquals(const SortCursor & rhs) const
/// For the case with a single column and when there is no order between different cursors.
struct SimpleSortCursor : SortCursorHelper<SimpleSortCursor>
{
using SortCursorHelper<SimpleSortCursor>::SortCursorHelper;
bool ALWAYS_INLINE greaterAt(const SimpleSortCursor & rhs, size_t lhs_pos, size_t rhs_pos) const
{
if (impl->rows == 0 || rhs.impl->rows == 0)
return false;
/// The last row of this cursor is no larger than the first row of the another cursor.
return !greaterAt(rhs, impl->rows - 1, 0);
}
bool greater(const SortCursor & rhs) const
{
return greaterAt(rhs, impl->pos, rhs.impl->pos);
}
/// Inverted so that the priority queue elements are removed in ascending order.
bool operator< (const SortCursor & rhs) const
{
return greater(rhs);
const auto & desc = impl->desc[0];
int direction = desc.direction;
int nulls_direction = desc.nulls_direction;
int res = impl->sort_columns[0]->compareAt(lhs_pos, rhs_pos, *(rhs.impl->sort_columns[0]), nulls_direction);
return res != 0 && ((res > 0) == (direction > 0));
}
};
/// Separate comparator for locale-sensitive string comparisons
struct SortCursorWithCollation
struct SortCursorWithCollation : SortCursorHelper<SortCursorWithCollation>
{
SortCursorImpl * impl;
using SortCursorHelper<SortCursorWithCollation>::SortCursorHelper;
SortCursorWithCollation(SortCursorImpl * impl_) : impl(impl_) {}
SortCursorImpl * operator-> () { return impl; }
const SortCursorImpl * operator-> () const { return impl; }
bool greaterAt(const SortCursorWithCollation & rhs, size_t lhs_pos, size_t rhs_pos) const
bool ALWAYS_INLINE greaterAt(const SortCursorWithCollation & rhs, size_t lhs_pos, size_t rhs_pos) const
{
for (size_t i = 0; i < impl->sort_columns_size; ++i)
{
int direction = impl->desc[i].direction;
int nulls_direction = impl->desc[i].nulls_direction;
const auto & desc = impl->desc[i];
int direction = desc.direction;
int nulls_direction = desc.nulls_direction;
int res;
if (impl->need_collation[i])
{
@ -189,29 +212,11 @@ struct SortCursorWithCollation
}
return impl->order > rhs.impl->order;
}
bool totallyLessOrEquals(const SortCursorWithCollation & rhs) const
{
if (impl->rows == 0 || rhs.impl->rows == 0)
return false;
/// The last row of this cursor is no larger than the first row of the another cursor.
return !greaterAt(rhs, impl->rows - 1, 0);
}
bool greater(const SortCursorWithCollation & rhs) const
{
return greaterAt(rhs, impl->pos, rhs.impl->pos);
}
bool operator< (const SortCursorWithCollation & rhs) const
{
return greater(rhs);
}
};
/** Allows to fetch data from multiple sort cursors in sorted order (merging sorted data streams).
* TODO: Replace with "Loser Tree", see https://en.wikipedia.org/wiki/K-way_merge_algorithm
*/
template <typename Cursor>
class SortingHeap
@ -225,7 +230,8 @@ public:
size_t size = cursors.size();
queue.reserve(size);
for (size_t i = 0; i < size; ++i)
queue.emplace_back(&cursors[i]);
if (!cursors[i].empty())
queue.emplace_back(&cursors[i]);
std::make_heap(queue.begin(), queue.end());
}
@ -233,7 +239,11 @@ public:
Cursor & current() { return queue.front(); }
void next()
size_t size() { return queue.size(); }
Cursor & nextChild() { return queue[nextChildIndex()]; }
void ALWAYS_INLINE next()
{
assert(isValid());
@ -246,34 +256,67 @@ public:
removeTop();
}
void replaceTop(Cursor new_top)
{
current() = new_top;
updateTop();
}
void removeTop()
{
std::pop_heap(queue.begin(), queue.end());
queue.pop_back();
next_idx = 0;
}
void push(SortCursorImpl & cursor)
{
queue.emplace_back(&cursor);
std::push_heap(queue.begin(), queue.end());
next_idx = 0;
}
private:
using Container = std::vector<Cursor>;
Container queue;
/// Cache comparison between first and second child if the order in queue has not been changed.
size_t next_idx = 0;
size_t ALWAYS_INLINE nextChildIndex()
{
if (next_idx == 0)
{
next_idx = 1;
if (queue.size() > 2 && queue[1] < queue[2])
++next_idx;
}
return next_idx;
}
/// This is adapted version of the function __sift_down from libc++.
/// Why cannot simply use std::priority_queue?
/// - because it doesn't support updating the top element and requires pop and push instead.
void updateTop()
/// Also look at "Boost.Heap" library.
void ALWAYS_INLINE updateTop()
{
size_t size = queue.size();
if (size < 2)
return;
size_t child_idx = 1;
auto begin = queue.begin();
auto child_it = begin + 1;
/// Right child exists and is greater than left child.
if (size > 2 && *child_it < *(child_it + 1))
{
++child_it;
++child_idx;
}
size_t child_idx = nextChildIndex();
auto child_it = begin + child_idx;
/// Check if we are in order.
if (*child_it < *begin)
return;
next_idx = 0;
auto curr_it = begin;
auto top(std::move(*begin));
do
@ -282,11 +325,12 @@ private:
*curr_it = std::move(*child_it);
curr_it = child_it;
if ((size - 2) / 2 < child_idx)
break;
// recompute the child based off of the updated parent
child_idx = 2 * child_idx + 1;
if (child_idx >= size)
break;
child_it = begin + child_idx;
if ((child_idx + 1) < size && *child_it < *(child_it + 1))
@ -300,12 +344,6 @@ private:
} while (!(*child_it < top));
*curr_it = std::move(top);
}
void removeTop()
{
std::pop_heap(queue.begin(), queue.end());
queue.pop_back();
}
};
}

6
dbms/src/Core/tests/gtest_DecimalFunctions.cpp
View File

@ -120,7 +120,7 @@ TEST_P(DecimalUtilsSplitAndCombineTest, getFractionalPart_Decimal128)
}
// Intentionally small values that fit into 32-bit in order to cover Decimal32, Decimal64 and Decimal128 with single set of data.
INSTANTIATE_TEST_CASE_P(Basic,
INSTANTIATE_TEST_SUITE_P(Basic,
DecimalUtilsSplitAndCombineTest,
::testing::ValuesIn(std::initializer_list<DecimalUtilsSplitAndCombineTestParam>{
{
@ -168,5 +168,5 @@ INSTANTIATE_TEST_CASE_P(Basic,
89
}
}
}
),);
})
);

3
dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp
View File

@ -62,6 +62,9 @@ Block AddingDefaultsBlockInputStream::readImpl()
if (evaluate_block.has(column.first))
evaluate_block.erase(column.first);
if (!evaluate_block.columns())
evaluate_block.insert({ColumnConst::create(ColumnUInt8::create(1, 0), res.rows()), std::make_shared<DataTypeUInt8>(), "_dummy"});
evaluateMissingDefaults(evaluate_block, header.getNamesAndTypesList(), column_defaults, context, false);
std::unordered_map<size_t, MutableColumnPtr> mixed_columns;

11
dbms/src/DataStreams/AggregatingSortedBlockInputStream.cpp
View File

@ -138,14 +138,14 @@ Block AggregatingSortedBlockInputStream::readImpl()
}
void AggregatingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue)
void AggregatingSortedBlockInputStream::merge(MutableColumns & merged_columns, SortingHeap<SortCursor> & queue)
{
size_t merged_rows = 0;
/// We take the rows in the correct order and put them in `merged_block`, while the rows are no more than `max_block_size`
while (!queue.empty())
while (queue.isValid())
{
SortCursor current = queue.top();
SortCursor current = queue.current();
setPrimaryKeyRef(next_key, current);
@ -167,8 +167,6 @@ void AggregatingSortedBlockInputStream::merge(MutableColumns & merged_columns, s
return;
}
queue.pop();
if (key_differs)
{
current_key.swap(next_key);
@ -202,8 +200,7 @@ void AggregatingSortedBlockInputStream::merge(MutableColumns & merged_columns, s
if (!current->isLast())
{
current->next();
queue.push(current);
queue.next();
}
else
{

2
dbms/src/DataStreams/AggregatingSortedBlockInputStream.h
View File

@ -55,7 +55,7 @@ private:
/** We support two different cursors - with Collation and without.
* Templates are used instead of polymorphic SortCursor and calls to virtual functions.
*/
void merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue);
void merge(MutableColumns & merged_columns, SortingHeap<SortCursor> & queue);
/** Extract all states of aggregate functions and merge them with the current group.
*/

11
dbms/src/DataStreams/CollapsingSortedBlockInputStream.cpp
View File

@ -105,15 +105,15 @@ Block CollapsingSortedBlockInputStream::readImpl()
}
void CollapsingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue)
void CollapsingSortedBlockInputStream::merge(MutableColumns & merged_columns, SortingHeap<SortCursor> & queue)
{
MergeStopCondition stop_condition(average_block_sizes, max_block_size);
size_t current_block_granularity;
/// Take rows in correct order and put them into `merged_columns` until the rows no more than `max_block_size`
for (; !queue.empty(); ++current_pos)
for (; queue.isValid(); ++current_pos)
{
SortCursor current = queue.top();
SortCursor current = queue.current();
current_block_granularity = current->rows;
if (current_key.empty())
@ -131,8 +131,6 @@ void CollapsingSortedBlockInputStream::merge(MutableColumns & merged_columns, st
return;
}
queue.pop();
if (key_differs)
{
/// We write data for the previous primary key.
@ -185,8 +183,7 @@ void CollapsingSortedBlockInputStream::merge(MutableColumns & merged_columns, st
if (!current->isLast())
{
current->next();
queue.push(current);
queue.next();
}
else
{

2
dbms/src/DataStreams/CollapsingSortedBlockInputStream.h
View File

@ -73,7 +73,7 @@ private:
/** We support two different cursors - with Collation and without.
* Templates are used instead of polymorphic SortCursors and calls to virtual functions.
*/
void merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue);
void merge(MutableColumns & merged_columns, SortingHeap<SortCursor> & queue);
/// Output to result rows for the current primary key.
void insertRows(MutableColumns & merged_columns, size_t block_size, MergeStopCondition & condition);

11
dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.cpp
View File

@ -161,7 +161,7 @@ Block GraphiteRollupSortedBlockInputStream::readImpl()
}
void GraphiteRollupSortedBlockInputStream::merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue)
void GraphiteRollupSortedBlockInputStream::merge(MutableColumns & merged_columns, SortingHeap<SortCursor> & queue)
{
const DateLUTImpl & date_lut = DateLUT::instance();
@ -173,9 +173,9 @@ void GraphiteRollupSortedBlockInputStream::merge(MutableColumns & merged_columns
/// contribute towards current output row.
/// Variables starting with next_* refer to the row at the top of the queue.
while (!queue.empty())
while (queue.isValid())
{
SortCursor next_cursor = queue.top();
SortCursor next_cursor = queue.current();
StringRef next_path = next_cursor->all_columns[path_column_num]->getDataAt(next_cursor->pos);
bool new_path = is_first || next_path != current_group_path;
@ -253,12 +253,9 @@ void GraphiteRollupSortedBlockInputStream::merge(MutableColumns & merged_columns
current_group_path = next_path;
}
queue.pop();
if (!next_cursor->isLast())
{
next_cursor->next();
queue.push(next_cursor);
queue.next();
}
else
{

2
dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.h
View File

@ -225,7 +225,7 @@ private:
UInt32 selectPrecision(const Graphite::Retentions & retentions, time_t time) const;
void merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue);
void merge(MutableColumns & merged_columns, SortingHeap<SortCursor> & queue);
/// Insert the values into the resulting columns, which will not be changed in the future.
template <typename TSortCursor>

31
dbms/src/DataStreams/MergeSortingBlockInputStream.cpp
View File

@ -150,10 +150,12 @@ MergeSortingBlocksBlockInputStream::MergeSortingBlocksBlockInputStream(
blocks.swap(nonempty_blocks);
if (!has_collation)
if (has_collation)
queue_with_collation = SortingHeap<SortCursorWithCollation>(cursors);
else if (description.size() > 1)
queue_without_collation = SortingHeap<SortCursor>(cursors);
else
queue_with_collation = SortingHeap<SortCursorWithCollation>(cursors);
queue_simple = SortingHeap<SimpleSortCursor>(cursors);
}
@ -169,9 +171,12 @@ Block MergeSortingBlocksBlockInputStream::readImpl()
return res;
}
return !has_collation
? mergeImpl(queue_without_collation)
: mergeImpl(queue_with_collation);
if (has_collation)
return mergeImpl(queue_with_collation);
else if (description.size() > 1)
return mergeImpl(queue_without_collation);
else
return mergeImpl(queue_simple);
}
@ -179,9 +184,18 @@ template <typename TSortingHeap>
Block MergeSortingBlocksBlockInputStream::mergeImpl(TSortingHeap & queue)
{
size_t num_columns = header.columns();
MutableColumns merged_columns = header.cloneEmptyColumns();
/// TODO: reserve (in each column)
/// Reserve
if (queue.isValid() && !blocks.empty())
{
/// The expected size of output block is the same as input block
size_t size_to_reserve = blocks[0].rows();
for (auto & column : merged_columns)
column->reserve(size_to_reserve);
}
/// TODO: Optimization when a single block left.
/// Take rows from queue in right order and push to 'merged'.
size_t merged_rows = 0;
@ -210,6 +224,9 @@ Block MergeSortingBlocksBlockInputStream::mergeImpl(TSortingHeap & queue)
break;
}
if (!queue.isValid())
blocks.clear();
if (merged_rows == 0)
return {};

1
dbms/src/DataStreams/MergeSortingBlockInputStream.h
View File

@ -59,6 +59,7 @@ private:
bool has_collation = false;
SortingHeap<SortCursor> queue_without_collation;
SortingHeap<SimpleSortCursor> queue_simple;
SortingHeap<SortCursorWithCollation> queue_with_collation;
/** Two different cursors are supported - with and without Collation.

201
dbms/src/DataStreams/MergingSortedBlockInputStream.cpp
View File

@ -59,9 +59,9 @@ void MergingSortedBlockInputStream::init(MutableColumns & merged_columns)
}
if (has_collation)
initQueue(queue_with_collation);
queue_with_collation = SortingHeap<SortCursorWithCollation>(cursors);
else
initQueue(queue_without_collation);
queue_without_collation = SortingHeap<SortCursor>(cursors);
}
/// Let's check that all source blocks have the same structure.
@ -82,15 +82,6 @@ void MergingSortedBlockInputStream::init(MutableColumns & merged_columns)
}
template <typename TSortCursor>
void MergingSortedBlockInputStream::initQueue(std::priority_queue<TSortCursor> & queue)
{
for (size_t i = 0; i < cursors.size(); ++i)
if (!cursors[i].empty())
queue.push(TSortCursor(&cursors[i]));
}
Block MergingSortedBlockInputStream::readImpl()
{
if (finished)
@ -115,7 +106,7 @@ Block MergingSortedBlockInputStream::readImpl()
template <typename TSortCursor>
void MergingSortedBlockInputStream::fetchNextBlock(const TSortCursor & current, std::priority_queue<TSortCursor> & queue)
void MergingSortedBlockInputStream::fetchNextBlock(const TSortCursor & current, SortingHeap<TSortCursor> & queue)
{
size_t order = current->order;
size_t size = cursors.size();
@ -125,15 +116,19 @@ void MergingSortedBlockInputStream::fetchNextBlock(const TSortCursor & current,
while (true)
{
source_blocks[order] = new detail::SharedBlock(children[order]->read());
source_blocks[order] = new detail::SharedBlock(children[order]->read()); /// intrusive ptr
if (!*source_blocks[order])
{
queue.removeTop();
break;
}
if (source_blocks[order]->rows())
{
cursors[order].reset(*source_blocks[order]);
queue.push(TSortCursor(&cursors[order]));
queue.replaceTop(&cursors[order]);
source_blocks[order]->all_columns = cursors[order].all_columns;
source_blocks[order]->sort_columns = cursors[order].sort_columns;
break;
@ -154,19 +149,14 @@ bool MergingSortedBlockInputStream::MergeStopCondition::checkStop() const
return sum_rows_count >= average;
}
template
void MergingSortedBlockInputStream::fetchNextBlock<SortCursor>(const SortCursor & current, std::priority_queue<SortCursor> & queue);
template
void MergingSortedBlockInputStream::fetchNextBlock<SortCursorWithCollation>(const SortCursorWithCollation & current, std::priority_queue<SortCursorWithCollation> & queue);
template <typename TSortCursor>
void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::priority_queue<TSortCursor> & queue)
template <typename TSortingHeap>
void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, TSortingHeap & queue)
{
size_t merged_rows = 0;
MergeStopCondition stop_condition(average_block_sizes, max_block_size);
/** Increase row counters.
* Return true if it's time to finish generating the current data block.
*/
@ -186,123 +176,100 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::
return stop_condition.checkStop();
};
/// Take rows in required order and put them into `merged_columns`, while the rows are no more than `max_block_size`
while (!queue.empty())
/// Take rows in required order and put them into `merged_columns`, while the number of rows are no more than `max_block_size`
while (queue.isValid())
{
TSortCursor current = queue.top();
auto current = queue.current();
size_t current_block_granularity = current->rows;
queue.pop();
while (true)
/** And what if the block is totally less or equal than the rest for the current cursor?
* Or is there only one data source left in the queue? Then you can take the entire block on current cursor.
*/
if (current->isFirst()
&& (queue.size() == 1
|| (queue.size() >= 2 && current.totallyLessOrEquals(queue.nextChild()))))
{
/** And what if the block is totally less or equal than the rest for the current cursor?
* Or is there only one data source left in the queue? Then you can take the entire block on current cursor.
*/
if (current->isFirst() && (queue.empty() || current.totallyLessOrEquals(queue.top())))
// std::cerr << "current block is totally less or equals\n";
/// If there are already data in the current block, we first return it. We'll get here again the next time we call the merge function.
if (merged_rows != 0)
{
// std::cerr << "current block is totally less or equals\n";
/// If there are already data in the current block, we first return it. We'll get here again the next time we call the merge function.
if (merged_rows != 0)
{
//std::cerr << "merged rows is non-zero\n";
queue.push(current);
return;
}
/// Actually, current->order stores source number (i.e. cursors[current->order] == current)
size_t source_num = current->order;
if (source_num >= cursors.size())
throw Exception("Logical error in MergingSortedBlockInputStream", ErrorCodes::LOGICAL_ERROR);
for (size_t i = 0; i < num_columns; ++i)
merged_columns[i] = (*std::move(source_blocks[source_num]->getByPosition(i).column)).mutate();
// std::cerr << "copied columns\n";
merged_rows = merged_columns.at(0)->size();
/// Limit output
if (limit && total_merged_rows + merged_rows > limit)
{
merged_rows = limit - total_merged_rows;
for (size_t i = 0; i < num_columns; ++i)
{
auto & column = merged_columns[i];
column = (*column->cut(0, merged_rows)).mutate();
}
cancel(false);
finished = true;
}
/// Write order of rows for other columns
/// this data will be used in grather stream
if (out_row_sources_buf)
{
RowSourcePart row_source(source_num);
for (size_t i = 0; i < merged_rows; ++i)
out_row_sources_buf->write(row_source.data);
}
//std::cerr << "fetching next block\n";
total_merged_rows += merged_rows;
fetchNextBlock(current, queue);
//std::cerr << "merged rows is non-zero\n";
return;
}
// std::cerr << "total_merged_rows: " << total_merged_rows << ", merged_rows: " << merged_rows << "\n";
// std::cerr << "Inserting row\n";
for (size_t i = 0; i < num_columns; ++i)
merged_columns[i]->insertFrom(*current->all_columns[i], current->pos);
/// Actually, current->order stores source number (i.e. cursors[current->order] == current)
size_t source_num = current->order;
if (source_num >= cursors.size())
throw Exception("Logical error in MergingSortedBlockInputStream", ErrorCodes::LOGICAL_ERROR);
for (size_t i = 0; i < num_columns; ++i)
merged_columns[i] = (*std::move(source_blocks[source_num]->getByPosition(i).column)).mutate();
// std::cerr << "copied columns\n";
merged_rows = merged_columns.at(0)->size();
/// Limit output
if (limit && total_merged_rows + merged_rows > limit)
{
merged_rows = limit - total_merged_rows;
for (size_t i = 0; i < num_columns; ++i)
{
auto & column = merged_columns[i];
column = (*column->cut(0, merged_rows)).mutate();
}
cancel(false);
finished = true;
}
/// Write order of rows for other columns
/// this data will be used in grather stream
if (out_row_sources_buf)
{
/// Actually, current.impl->order stores source number (i.e. cursors[current.impl->order] == current.impl)
RowSourcePart row_source(current->order);
out_row_sources_buf->write(row_source.data);
RowSourcePart row_source(source_num);
for (size_t i = 0; i < merged_rows; ++i)
out_row_sources_buf->write(row_source.data);
}
if (!current->isLast())
{
// std::cerr << "moving to next row\n";
current->next();
//std::cerr << "fetching next block\n";
if (queue.empty() || !(current.greater(queue.top())))
{
if (count_row_and_check_limit(current_block_granularity))
{
// std::cerr << "pushing back to queue\n";
queue.push(current);
return;
}
total_merged_rows += merged_rows;
fetchNextBlock(current, queue);
return;
}
/// Do not put the cursor back in the queue, but continue to work with the current cursor.
// std::cerr << "current is still on top, using current row\n";
continue;
}
else
{
// std::cerr << "next row is not least, pushing back to queue\n";
queue.push(current);
}
}
else
{
/// We get the next block from the corresponding source, if there is one.
// std::cerr << "It was last row, fetching next block\n";
fetchNextBlock(current, queue);
}
// std::cerr << "total_merged_rows: " << total_merged_rows << ", merged_rows: " << merged_rows << "\n";
// std::cerr << "Inserting row\n";
for (size_t i = 0; i < num_columns; ++i)
merged_columns[i]->insertFrom(*current->all_columns[i], current->pos);
break;
if (out_row_sources_buf)
{
/// Actually, current.impl->order stores source number (i.e. cursors[current.impl->order] == current.impl)
RowSourcePart row_source(current->order);
out_row_sources_buf->write(row_source.data);
}
if (!current->isLast())
{
// std::cerr << "moving to next row\n";
queue.next();
}
else
{
/// We get the next block from the corresponding source, if there is one.
// std::cerr << "It was last row, fetching next block\n";
fetchNextBlock(current, queue);
}
if (count_row_and_check_limit(current_block_granularity))
return;
}
/// We have read all data. Ask childs to cancel providing more data.
cancel(false);
finished = true;
}

25
dbms/src/DataStreams/MergingSortedBlockInputStream.h
View File

@ -1,7 +1,5 @@
#pragma once
#include <queue>
#include <boost/smart_ptr/intrusive_ptr.hpp>
#include <common/logger_useful.h>
@ -87,7 +85,7 @@ protected:
/// Gets the next block from the source corresponding to the `current`.
template <typename TSortCursor>
void fetchNextBlock(const TSortCursor & current, std::priority_queue<TSortCursor> & queue);
void fetchNextBlock(const TSortCursor & current, SortingHeap<TSortCursor> & queue);
Block header;
@ -109,14 +107,10 @@ protected:
size_t num_columns = 0;
std::vector<SharedBlockPtr> source_blocks;
using CursorImpls = std::vector<SortCursorImpl>;
CursorImpls cursors;
SortCursorImpls cursors;
using Queue = std::priority_queue<SortCursor>;
Queue queue_without_collation;
using QueueWithCollation = std::priority_queue<SortCursorWithCollation>;
QueueWithCollation queue_with_collation;
SortingHeap<SortCursor> queue_without_collation;
SortingHeap<SortCursorWithCollation> queue_with_collation;
/// Used in Vertical merge algorithm to gather non-PK/non-index columns (on next step)
/// If it is not nullptr then it should be populated during execution
@ -177,13 +171,10 @@ protected:
private:
/** We support two different cursors - with Collation and without.
* Templates are used instead of polymorphic SortCursor and calls to virtual functions.
*/
template <typename TSortCursor>
void initQueue(std::priority_queue<TSortCursor> & queue);
template <typename TSortCursor>
void merge(MutableColumns & merged_columns, std::priority_queue<TSortCursor> & queue);
* Templates are used instead of polymorphic SortCursor and calls to virtual functions.
*/
template <typename TSortingHeap>
void merge(MutableColumns & merged_columns, TSortingHeap & queue);
Logger * log = &Logger::get("MergingSortedBlockInputStream");

2
dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
View File

@ -129,7 +129,7 @@ void PushingToViewsBlockOutputStream::write(const Block & block)
for (size_t view_num = 0; view_num < views.size(); ++view_num)
{
auto thread_group = CurrentThread::getGroup();
pool.scheduleOrThrowOnError([=]
pool.scheduleOrThrowOnError([=, this]
{
setThreadName("PushingToViews");
if (thread_group)

12
dbms/src/DataStreams/ReplacingSortedBlockInputStream.cpp
View File

@ -48,13 +48,14 @@ Block ReplacingSortedBlockInputStream::readImpl()
}
void ReplacingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue)
void ReplacingSortedBlockInputStream::merge(MutableColumns & merged_columns, SortingHeap<SortCursor> & queue)
{
MergeStopCondition stop_condition(average_block_sizes, max_block_size);
/// Take the rows in needed order and put them into `merged_columns` until rows no more than `max_block_size`
while (!queue.empty())
while (queue.isValid())
{
SortCursor current = queue.top();
SortCursor current = queue.current();
size_t current_block_granularity = current->rows;
if (current_key.empty())
@ -68,8 +69,6 @@ void ReplacingSortedBlockInputStream::merge(MutableColumns & merged_columns, std
if (key_differs && stop_condition.checkStop())
return;
queue.pop();
if (key_differs)
{
/// Write the data for the previous primary key.
@ -98,8 +97,7 @@ void ReplacingSortedBlockInputStream::merge(MutableColumns & merged_columns, std
if (!current->isLast())
{
current->next();
queue.push(current);
queue.next();
}
else
{

2
dbms/src/DataStreams/ReplacingSortedBlockInputStream.h
View File

@ -52,7 +52,7 @@ private:
/// Sources of rows with the current primary key.
PODArray<RowSourcePart> current_row_sources;
void merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue);
void merge(MutableColumns & merged_columns, SortingHeap<SortCursor> & queue);
/// Output into result the rows for current primary key.
void insertRow(MutableColumns & merged_columns);

11
dbms/src/DataStreams/SummingSortedBlockInputStream.cpp
View File

@ -314,14 +314,14 @@ Block SummingSortedBlockInputStream::readImpl()
}
void SummingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue)
void SummingSortedBlockInputStream::merge(MutableColumns & merged_columns, SortingHeap<SortCursor> & queue)
{
merged_rows = 0;
/// Take the rows in needed order and put them in `merged_columns` until rows no more than `max_block_size`
while (!queue.empty())
while (queue.isValid())
{
SortCursor current = queue.top();
SortCursor current = queue.current();
setPrimaryKeyRef(next_key, current);
@ -383,12 +383,9 @@ void SummingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::
current_row_is_zero = false;
}
queue.pop();
if (!current->isLast())
{
current->next();
queue.push(current);
queue.next();
}
else
{

4
dbms/src/DataStreams/SummingSortedBlockInputStream.h
View File

@ -1,5 +1,7 @@
#pragma once
#include <queue>
#include <Core/Row.h>
#include <Core/ColumnNumbers.h>
#include <Common/AlignedBuffer.h>
@ -140,7 +142,7 @@ private:
/** We support two different cursors - with Collation and without.
* Templates are used instead of polymorphic SortCursor and calls to virtual functions.
*/
void merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue);
void merge(MutableColumns & merged_columns, SortingHeap<SortCursor> & queue);
/// Insert the summed row for the current group into the result and updates some of per-block flags if the row is not "zero".
void insertCurrentRowIfNeeded(MutableColumns & merged_columns);

11
dbms/src/DataStreams/VersionedCollapsingSortedBlockInputStream.cpp
View File

@ -82,21 +82,18 @@ Block VersionedCollapsingSortedBlockInputStream::readImpl()
}
void VersionedCollapsingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue)
void VersionedCollapsingSortedBlockInputStream::merge(MutableColumns & merged_columns, SortingHeap<SortCursor> & queue)
{
MergeStopCondition stop_condition(average_block_sizes, max_block_size);
auto update_queue = [this, & queue](SortCursor & cursor)
{
queue.pop();
if (out_row_sources_buf)
current_row_sources.emplace(cursor->order, true);
if (!cursor->isLast())
{
cursor->next();
queue.push(cursor);
queue.next();
}
else
{
@ -106,9 +103,9 @@ void VersionedCollapsingSortedBlockInputStream::merge(MutableColumns & merged_co
};
/// Take rows in correct order and put them into `merged_columns` until the rows no more than `max_block_size`
while (!queue.empty())
while (queue.isValid())
{
SortCursor current = queue.top();
SortCursor current = queue.current();
size_t current_block_granularity = current->rows;
SharedBlockRowRef next_key;

4
dbms/src/DataStreams/VersionedCollapsingSortedBlockInputStream.h
View File

@ -5,7 +5,7 @@
#include <DataStreams/MergingSortedBlockInputStream.h>
#include <DataStreams/ColumnGathererStream.h>
#include <deque>
#include <queue>
namespace DB
@ -204,7 +204,7 @@ private:
/// Sources of rows for VERTICAL merge algorithm. Size equals to (size + number of gaps) in current_keys.
std::queue<RowSourcePart> current_row_sources;
void merge(MutableColumns & merged_columns, std::priority_queue<SortCursor> & queue);
void merge(MutableColumns & merged_columns, SortingHeap<SortCursor> & queue);
/// Output to result row for the current primary key.
void insertRow(size_t skip_rows, const SharedBlockRowRef & row, MutableColumns & merged_columns);

2
dbms/src/DataStreams/tests/union_stream2.cpp
View File

@ -57,6 +57,6 @@ catch (const Exception & e)
std::cerr << e.what() << ", " << e.displayText() << std::endl
<< std::endl
<< "Stack trace:" << std::endl
<< e.getStackTrace().toString();
<< e.getStackTraceString();
return 1;
}

2
dbms/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp
View File

@ -30,7 +30,7 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
}
static const std::vector<String> supported_functions{"any", "anyLast", "min", "max", "sum"};
static const std::vector<String> supported_functions{"any", "anyLast", "min", "max", "sum", "groupBitAnd", "groupBitOr", "groupBitXor"};
String DataTypeCustomSimpleAggregateFunction::getName() const

9
dbms/src/DataTypes/tests/gtest_data_type_get_common_type.cpp
View File

@ -104,7 +104,7 @@ TEST_P(MostSubtypeTest, getLeastSupertype)
}
}
INSTANTIATE_TEST_CASE_P(data_type,
INSTANTIATE_TEST_SUITE_P(data_type,
LeastSuperTypeTest,
::testing::ValuesIn(
std::initializer_list<TypesTestCase>{
@ -159,10 +159,10 @@ INSTANTIATE_TEST_CASE_P(data_type,
{"Tuple(Int64,Int8) Tuple(UInt64)", nullptr},
{"Array(Int64) Array(String)", nullptr},
}
),
)
);
INSTANTIATE_TEST_CASE_P(data_type,
INSTANTIATE_TEST_SUITE_P(data_type,
MostSubtypeTest,
::testing::ValuesIn(
std::initializer_list<TypesTestCase>{
@ -210,5 +210,6 @@ INSTANTIATE_TEST_CASE_P(data_type,
{"Int8 String", nullptr},
{"Nothing", nullptr},
{"FixedString(16) FixedString(8) String", nullptr},
}),
}
)
);

7
dbms/src/Databases/DatabaseLazy.cpp
View File

@ -23,7 +23,6 @@ namespace ErrorCodes
extern const int TABLE_ALREADY_EXISTS;
extern const int UNKNOWN_TABLE;
extern const int UNSUPPORTED_METHOD;
extern const int CANNOT_CREATE_TABLE_FROM_METADATA;
extern const int LOGICAL_ERROR;
}
@ -255,10 +254,10 @@ StoragePtr DatabaseLazy::loadTable(const Context & context, const String & table
return it->second.table = table;
}
}
catch (const Exception & e)
catch (Exception & e)
{
throw Exception("Cannot create table from metadata file " + table_metadata_path + ". Error: " + DB::getCurrentExceptionMessage(true),
e, DB::ErrorCodes::CANNOT_CREATE_TABLE_FROM_METADATA);
e.addMessage("Cannot create table from metadata file " + table_metadata_path);
throw;
}
}

33
dbms/src/Databases/DatabaseOrdinary.cpp
View File

@ -10,8 +10,6 @@
#include <IO/WriteHelpers.h>
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Interpreters/ExternalLoaderDatabaseConfigRepository.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Storages/StorageFactory.h>
@ -38,7 +36,6 @@ namespace DB
namespace ErrorCodes
{
extern const int CANNOT_CREATE_TABLE_FROM_METADATA;
extern const int CANNOT_CREATE_DICTIONARY_FROM_METADATA;
extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
extern const int CANNOT_PARSE_TEXT;
@ -68,13 +65,10 @@ namespace
= createTableFromAST(query, database_name, database.getTableDataPath(query), context, has_force_restore_data_flag);
database.attachTable(table_name, table);
}
catch (const Exception & e)
catch (Exception & e)
{
throw Exception(
"Cannot attach table '" + query.table + "' from query " + serializeAST(query)
+ ". Error: " + DB::getCurrentExceptionMessage(true),
e,
DB::ErrorCodes::CANNOT_CREATE_TABLE_FROM_METADATA);
e.addMessage("Cannot attach table '" + backQuote(query.table) + "' from query " + serializeAST(query));
throw;
}
}
@ -89,13 +83,10 @@ namespace
{
database.attachDictionary(query.table, context);
}
catch (const Exception & e)
catch (Exception & e)
{
throw Exception(
"Cannot create dictionary '" + query.table + "' from query " + serializeAST(query)
+ ". Error: " + DB::getCurrentExceptionMessage(true),
e,
DB::ErrorCodes::CANNOT_CREATE_DICTIONARY_FROM_METADATA);
e.addMessage("Cannot attach table '" + backQuote(query.table) + "' from query " + serializeAST(query));
throw;
}
}
@ -144,10 +135,10 @@ void DatabaseOrdinary::loadStoredObjects(
total_dictionaries += create_query->is_dictionary;
}
}
catch (const Exception & e)
catch (Exception & e)
{
throw Exception(
"Cannot parse definition from metadata file " + full_path + ". Error: " + DB::getCurrentExceptionMessage(true), e, ErrorCodes::CANNOT_PARSE_TEXT);
e.addMessage("Cannot parse definition from metadata file " + full_path);
throw;
}
});
@ -181,12 +172,8 @@ void DatabaseOrdinary::loadStoredObjects(
/// After all tables was basically initialized, startup them.
startupTables(pool);
/// Add database as repository
auto dictionaries_repository = std::make_unique<ExternalLoaderDatabaseConfigRepository>(shared_from_this(), context);
auto & external_loader = context.getExternalDictionariesLoader();
external_loader.addConfigRepository(getDatabaseName(), std::move(dictionaries_repository));
/// Attach dictionaries.
attachToExternalDictionariesLoader(context);
for (const auto & name_with_query : file_names)
{
auto create_query = name_with_query.second->as<const ASTCreateQuery &>();

37
dbms/src/Databases/DatabaseWithDictionaries.cpp
View File

@ -1,6 +1,7 @@
#include <Databases/DatabaseWithDictionaries.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/ExternalLoaderPresetConfigRepository.h>
#include <Interpreters/ExternalLoaderTempConfigRepository.h>
#include <Interpreters/ExternalLoaderDatabaseConfigRepository.h>
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
#include <Interpreters/Context.h>
#include <Storages/StorageDictionary.h>
@ -74,7 +75,7 @@ void DatabaseWithDictionaries::createDictionary(const Context & context, const S
/// A dictionary with the same full name could be defined in *.xml config files.
String full_name = getDatabaseName() + "." + dictionary_name;
auto & external_loader = const_cast<ExternalDictionariesLoader &>(context.getExternalDictionariesLoader());
const auto & external_loader = context.getExternalDictionariesLoader();
if (external_loader.getCurrentStatus(full_name) != ExternalLoader::Status::NOT_EXIST)
throw Exception(
"Dictionary " + backQuote(getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.",
@ -106,15 +107,10 @@ void DatabaseWithDictionaries::createDictionary(const Context & context, const S
/// Add a temporary repository containing the dictionary.
/// We need this temp repository to try loading the dictionary before actually attaching it to the database.
static std::atomic<size_t> counter = 0;
String temp_repository_name = String(IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX) + " creating " + full_name + " "
+ std::to_string(++counter);
external_loader.addConfigRepository(
temp_repository_name,
std::make_unique<ExternalLoaderPresetConfigRepository>(
std::vector{std::pair{dictionary_metadata_tmp_path,
getDictionaryConfigurationFromAST(query->as<const ASTCreateQuery &>(), getDatabaseName())}}));
SCOPE_EXIT({ external_loader.removeConfigRepository(temp_repository_name); });
auto temp_repository
= const_cast<ExternalDictionariesLoader &>(external_loader) /// the change of ExternalDictionariesLoader is temporary
.addConfigRepository(std::make_unique<ExternalLoaderTempConfigRepository>(
getDatabaseName(), dictionary_metadata_tmp_path, getDictionaryConfigurationFromAST(query->as<const ASTCreateQuery &>())));
bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true);
if (!lazy_load)
@ -253,4 +249,23 @@ ASTPtr DatabaseWithDictionaries::getCreateDictionaryQueryImpl(
return ast;
}
void DatabaseWithDictionaries::shutdown()
{
detachFromExternalDictionariesLoader();
DatabaseOnDisk::shutdown();
}
DatabaseWithDictionaries::~DatabaseWithDictionaries() = default;
void DatabaseWithDictionaries::attachToExternalDictionariesLoader(Context & context)
{
database_as_config_repo_for_external_loader = context.getExternalDictionariesLoader().addConfigRepository(
std::make_unique<ExternalLoaderDatabaseConfigRepository>(*this, context));
}
void DatabaseWithDictionaries::detachFromExternalDictionariesLoader()
{
database_as_config_repo_for_external_loader = {};
}
}

11
dbms/src/Databases/DatabaseWithDictionaries.h
View File

@ -1,4 +1,5 @@
#include <Databases/DatabaseOnDisk.h>
#include <ext/scope_guard.h>
namespace DB
{
@ -25,15 +26,25 @@ public:
bool isDictionaryExist(const Context & context, const String & dictionary_name) const override;
void shutdown() override;
~DatabaseWithDictionaries() override;
protected:
DatabaseWithDictionaries(const String & name, const String & metadata_path_, const String & logger)
: DatabaseOnDisk(name, metadata_path_, logger) {}
void attachToExternalDictionariesLoader(Context & context);
void detachFromExternalDictionariesLoader();
StoragePtr getDictionaryStorage(const Context & context, const String & table_name) const;
ASTPtr getCreateDictionaryQueryImpl(const Context & context,
const String & dictionary_name,
bool throw_on_error) const override;
private:
ext::scope_guard database_as_config_repo_for_external_loader;
};
}

29
dbms/src/Dictionaries/CacheDictionary.cpp
View File

@ -57,12 +57,15 @@ inline size_t CacheDictionary::getCellIdx(const Key id) const
CacheDictionary::CacheDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
const size_t size_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -73,7 +76,7 @@ CacheDictionary::CacheDictionary(
, rnd_engine(randomSeed())
{
if (!this->source_ptr->supportsSelectiveLoad())
throw Exception{name + ": source cannot be used with CacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
throw Exception{full_name + ": source cannot be used with CacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
createAttributes();
}
@ -204,7 +207,7 @@ void CacheDictionary::isInConstantVector(const Key child_id, const PaddedPODArra
void CacheDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
{
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto null_value = StringRef{std::get<String>(attribute.null_values)};
@ -215,7 +218,7 @@ void CacheDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
{
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, ids, out, [&](const size_t row) { return def->getDataAt(row); });
}
@ -224,7 +227,7 @@ void CacheDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
{
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, ids, out, [&](const size_t) { return StringRef{def}; });
}
@ -352,7 +355,7 @@ void CacheDictionary::createAttributes()
hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
}
}
}
@ -539,7 +542,7 @@ CacheDictionary::Attribute & CacheDictionary::getAttribute(const std::string & a
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -580,7 +583,7 @@ std::exception_ptr CacheDictionary::getLastException() const
void registerDictionaryCache(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -590,22 +593,24 @@ void registerDictionaryCache(DictionaryFactory & factory)
throw Exception{"'key' is not supported for dictionary of layout 'cache'", ErrorCodes::UNSUPPORTED_METHOD};
if (dict_struct.range_min || dict_struct.range_max)
throw Exception{name
throw Exception{full_name
+ ": elements .structure.range_min and .structure.range_max should be defined only "
"for a dictionary of layout 'range_hashed'",
ErrorCodes::BAD_ARGUMENTS};
const auto & layout_prefix = config_prefix + ".layout";
const auto size = config.getInt(layout_prefix + ".cache.size_in_cells");
if (size == 0)
throw Exception{name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
if (require_nonempty)
throw Exception{name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
return std::make_unique<CacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size);
return std::make_unique<CacheDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, size);
};
factory.registerLayout("cache", create_layout, false);
}

9
dbms/src/Dictionaries/CacheDictionary.h
View File

@ -25,13 +25,16 @@ class CacheDictionary final : public IDictionary
{
public:
CacheDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
const size_t size_);
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "Cache"; }
@ -52,7 +55,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<CacheDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, size);
return std::make_shared<CacheDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, size);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -254,7 +257,9 @@ private:
template <typename AncestorType>
void isInImpl(const PaddedPODArray<Key> & child_ids, const AncestorType & ancestor_ids, PaddedPODArray<UInt8> & out) const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
mutable DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

2
dbms/src/Dictionaries/CacheDictionary.inc.h
View File

@ -333,7 +333,7 @@ void CacheDictionary::update(
last_exception = std::current_exception();
backoff_end_time = now + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count));
tryLogException(last_exception, log, "Could not update cache dictionary '" + getName() +
tryLogException(last_exception, log, "Could not update cache dictionary '" + getFullName() +
"', next update is scheduled at " + ext::to_string(backoff_end_time));
}
}

27
dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
View File

@ -51,12 +51,15 @@ inline UInt64 ComplexKeyCacheDictionary::getCellIdx(const StringRef key) const
ComplexKeyCacheDictionary::ComplexKeyCacheDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
const size_t size_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -65,7 +68,7 @@ ComplexKeyCacheDictionary::ComplexKeyCacheDictionary(
, rnd_engine(randomSeed())
{
if (!this->source_ptr->supportsSelectiveLoad())
throw Exception{name + ": source cannot be used with ComplexKeyCacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
throw Exception{full_name + ": source cannot be used with ComplexKeyCacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
createAttributes();
}
@ -77,7 +80,7 @@ void ComplexKeyCacheDictionary::getString(
dict_struct.validateKeyTypes(key_types);
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto null_value = StringRef{std::get<String>(attribute.null_values)};
@ -94,7 +97,7 @@ void ComplexKeyCacheDictionary::getString(
dict_struct.validateKeyTypes(key_types);
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, key_columns, out, [&](const size_t row) { return def->getDataAt(row); });
}
@ -109,7 +112,7 @@ void ComplexKeyCacheDictionary::getString(
dict_struct.validateKeyTypes(key_types);
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, key_columns, out, [&](const size_t) { return StringRef{def}; });
}
@ -249,7 +252,7 @@ void ComplexKeyCacheDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical)
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
ErrorCodes::TYPE_MISMATCH};
}
}
@ -258,7 +261,7 @@ ComplexKeyCacheDictionary::Attribute & ComplexKeyCacheDictionary::getAttribute(c
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -394,7 +397,7 @@ BlockInputStreamPtr ComplexKeyCacheDictionary::getBlockInputStream(const Names &
void registerDictionaryComplexKeyCache(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -405,15 +408,17 @@ void registerDictionaryComplexKeyCache(DictionaryFactory & factory)
const auto & layout_prefix = config_prefix + ".layout";
const auto size = config.getInt(layout_prefix + ".complex_key_cache.size_in_cells");
if (size == 0)
throw Exception{name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
if (require_nonempty)
throw Exception{name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
return std::make_unique<ComplexKeyCacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size);
return std::make_unique<ComplexKeyCacheDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, size);
};
factory.registerLayout("complex_key_cache", create_layout, true);
}

9
dbms/src/Dictionaries/ComplexKeyCacheDictionary.h
View File

@ -42,6 +42,7 @@ class ComplexKeyCacheDictionary final : public IDictionaryBase
{
public:
ComplexKeyCacheDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -50,7 +51,9 @@ public:
std::string getKeyDescription() const { return key_description; }
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "ComplexKeyCache"; }
@ -75,7 +78,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<ComplexKeyCacheDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, size);
return std::make_shared<ComplexKeyCacheDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, size);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -668,7 +671,9 @@ private:
bool isEmptyCell(const UInt64 idx) const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

30
dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
View File

@ -15,13 +15,16 @@ namespace ErrorCodes
}
ComplexKeyHashedDictionary::ComplexKeyHashedDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
bool require_nonempty_,
BlockPtr saved_block_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -40,7 +43,7 @@ ComplexKeyHashedDictionary::ComplexKeyHashedDictionary(
dict_struct.validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
const auto null_value = std::get<TYPE>(attribute.null_values); \
\
@ -72,7 +75,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
@ -94,7 +97,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, \
@ -128,7 +131,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -148,7 +151,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -179,7 +182,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -256,7 +259,7 @@ void ComplexKeyHashedDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical)
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
ErrorCodes::TYPE_MISMATCH};
}
}
@ -397,7 +400,7 @@ void ComplexKeyHashedDictionary::loadData()
updateData();
if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
template <typename T>
@ -630,7 +633,7 @@ const ComplexKeyHashedDictionary::Attribute & ComplexKeyHashedDictionary::getAtt
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -742,7 +745,7 @@ BlockInputStreamPtr ComplexKeyHashedDictionary::getBlockInputStream(const Names
void registerDictionaryComplexKeyHashed(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string &,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -751,12 +754,13 @@ void registerDictionaryComplexKeyHashed(DictionaryFactory & factory)
if (!dict_struct.key)
throw Exception{"'key' is required for dictionary of layout 'complex_key_hashed'", ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
return std::make_unique<ComplexKeyHashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
return std::make_unique<ComplexKeyHashedDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
};
factory.registerLayout("complex_key_hashed", create_layout, true);
}
}

9
dbms/src/Dictionaries/ComplexKeyHashedDictionary.h
View File

@ -23,6 +23,7 @@ class ComplexKeyHashedDictionary final : public IDictionaryBase
{
public:
ComplexKeyHashedDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -32,7 +33,9 @@ public:
std::string getKeyDescription() const { return key_description; }
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "ComplexKeyHashed"; }
@ -48,7 +51,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<ComplexKeyHashedDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
return std::make_shared<ComplexKeyHashedDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -233,7 +236,9 @@ private:
template <typename T>
std::vector<StringRef> getKeys(const Attribute & attribute) const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

34
dbms/src/Dictionaries/FlatDictionary.cpp
View File

@ -21,13 +21,16 @@ static const auto max_array_size = 500000;
FlatDictionary::FlatDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
bool require_nonempty_,
BlockPtr saved_block_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -107,7 +110,7 @@ void FlatDictionary::isInConstantVector(const Key child_id, const PaddedPODArray
void FlatDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
const auto null_value = std::get<TYPE>(attribute.null_values); \
\
@ -133,7 +136,7 @@ DECLARE(Decimal128)
void FlatDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = std::get<StringRef>(attribute.null_values);
@ -152,7 +155,7 @@ void FlatDictionary::getString(const std::string & attribute_name, const PaddedP
ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \
@ -177,7 +180,7 @@ void FlatDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -191,7 +194,7 @@ void FlatDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -216,7 +219,7 @@ void FlatDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
FlatDictionary::getItemsImpl<StringRef, StringRef>(
attribute,
@ -297,7 +300,7 @@ void FlatDictionary::createAttributes()
hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
}
}
}
@ -404,7 +407,7 @@ void FlatDictionary::loadData()
updateData();
if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
@ -578,7 +581,7 @@ template <typename T>
void FlatDictionary::resize(Attribute & attribute, const Key id)
{
if (id >= max_array_size)
throw Exception{name + ": identifier should be less than " + toString(max_array_size), ErrorCodes::ARGUMENT_OUT_OF_BOUND};
throw Exception{full_name + ": identifier should be less than " + toString(max_array_size), ErrorCodes::ARGUMENT_OUT_OF_BOUND};
auto & array = std::get<ContainerType<T>>(attribute.arrays);
if (id >= array.size())
@ -666,7 +669,7 @@ const FlatDictionary::Attribute & FlatDictionary::getAttribute(const std::string
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -706,7 +709,7 @@ BlockInputStreamPtr FlatDictionary::getBlockInputStream(const Names & column_nam
void registerDictionaryFlat(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -716,13 +719,16 @@ void registerDictionaryFlat(DictionaryFactory & factory)
throw Exception{"'key' is not supported for dictionary of layout 'flat'", ErrorCodes::UNSUPPORTED_METHOD};
if (dict_struct.range_min || dict_struct.range_max)
throw Exception{name
throw Exception{full_name
+ ": elements .structure.range_min and .structure.range_max should be defined only "
"for a dictionary of layout 'range_hashed'",
ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
return std::make_unique<FlatDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
return std::make_unique<FlatDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
};
factory.registerLayout("flat", create_layout, false);
}

9
dbms/src/Dictionaries/FlatDictionary.h
View File

@ -22,6 +22,7 @@ class FlatDictionary final : public IDictionary
{
public:
FlatDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -29,7 +30,9 @@ public:
bool require_nonempty_,
BlockPtr saved_block_ = nullptr);
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "Flat"; }
@ -45,7 +48,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<FlatDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
return std::make_shared<FlatDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -222,7 +225,9 @@ private:
PaddedPODArray<Key> getIds() const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

41
dbms/src/Dictionaries/HashedDictionary.cpp
View File

@ -31,6 +31,7 @@ namespace ErrorCodes
HashedDictionary::HashedDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -38,7 +39,9 @@ HashedDictionary::HashedDictionary(
bool require_nonempty_,
bool sparse_,
BlockPtr saved_block_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -129,7 +132,7 @@ void HashedDictionary::isInConstantVector(const Key child_id, const PaddedPODArr
const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
const auto null_value = std::get<TYPE>(attribute.null_values); \
\
@ -155,7 +158,7 @@ DECLARE(Decimal128)
void HashedDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
@ -174,7 +177,7 @@ void HashedDictionary::getString(const std::string & attribute_name, const Padde
ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \
@ -199,7 +202,7 @@ void HashedDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -213,7 +216,7 @@ void HashedDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE & def, ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -238,7 +241,7 @@ void HashedDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -317,7 +320,7 @@ void HashedDictionary::createAttributes()
hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
}
}
}
@ -424,7 +427,7 @@ void HashedDictionary::loadData()
updateData();
if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
template <typename T>
@ -684,7 +687,7 @@ const HashedDictionary::Attribute & HashedDictionary::getAttribute(const std::st
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -768,27 +771,31 @@ BlockInputStreamPtr HashedDictionary::getBlockInputStream(const Names & column_n
void registerDictionaryHashed(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
DictionarySourcePtr source_ptr) -> DictionaryPtr
DictionarySourcePtr source_ptr,
bool sparse) -> DictionaryPtr
{
if (dict_struct.key)
throw Exception{"'key' is not supported for dictionary of layout 'hashed'", ErrorCodes::UNSUPPORTED_METHOD};
if (dict_struct.range_min || dict_struct.range_max)
throw Exception{name
throw Exception{full_name
+ ": elements .structure.range_min and .structure.range_max should be defined only "
"for a dictionary of layout 'range_hashed'",
ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
const bool sparse = name == "sparse_hashed";
return std::make_unique<HashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty, sparse);
return std::make_unique<HashedDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty, sparse);
};
factory.registerLayout("hashed", create_layout, false);
factory.registerLayout("sparse_hashed", create_layout, false);
using namespace std::placeholders;
factory.registerLayout("hashed", std::bind(create_layout, _1, _2, _3, _4, _5, /* sparse = */ false), false);
factory.registerLayout("sparse_hashed", std::bind(create_layout, _1, _2, _3, _4, _5, /* sparse = */ true), false);
}
}

9
dbms/src/Dictionaries/HashedDictionary.h
View File

@ -26,6 +26,7 @@ class HashedDictionary final : public IDictionary
{
public:
HashedDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -34,7 +35,9 @@ public:
bool sparse_,
BlockPtr saved_block_ = nullptr);
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return sparse ? "SparseHashed" : "Hashed"; }
@ -50,7 +53,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<HashedDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, sparse, saved_block);
return std::make_shared<HashedDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, sparse, saved_block);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -262,7 +265,9 @@ private:
template <typename ChildType, typename AncestorType>
void isInImpl(const ChildType & child_ids, const AncestorType & ancestor_ids, PaddedPODArray<UInt8> & out) const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

5
dbms/src/Dictionaries/IDictionary.h
View File

@ -25,6 +25,11 @@ struct IDictionaryBase : public IExternalLoadable
{
using Key = UInt64;
virtual const std::string & getDatabase() const = 0;
virtual const std::string & getName() const = 0;
virtual const std::string & getFullName() const = 0;
const std::string & getLoadableName() const override { return getFullName(); }
virtual std::string getTypeName() const = 0;
virtual size_t getBytesAllocated() const = 0;

Some files were not shown because too many files have changed in this diff Show More