thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-29 11:02:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merged with master

Browse Source
This commit is contained in:
Nikolai Kochetov 2020-01-23 18:44:26 +03:00
commit 4a4de82648
559 changed files with 11209 additions and 4180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/PULL_REQUEST_TEMPLATE.md vendored
View File

@ -7,16 +7,20 @@ Changelog category (leave one):
- Performance Improvement - Performance Improvement
- Backward Incompatible Change - Backward Incompatible Change
- Build/Testing/Packaging Improvement - Build/Testing/Packaging Improvement
- Documentation - Documentation (changelog entry is not required)
- Other - Other
- Non-significant (changelog entry is not needed) - Non-significant (changelog entry is not required)
Changelog entry (up to few sentences, required except for Non-significant/Documentation categories): Changelog entry (a user-readable short description of the changes that goes to CHANGELOG.md):
... ...
Detailed description (optional): Detailed description / Documentation draft:
... ...
By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder.
If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first.

3
.gitignore vendored
View File

@ -244,6 +244,9 @@ website/package-lock.json
# ccls cache # ccls cache
/.ccls-cache /.ccls-cache
# clangd cache
/.clangd
/compile_commands.json /compile_commands.json
# Toolchains # Toolchains

5
.gitmodules vendored
View File

@ -49,7 +49,7 @@
url = https://github.com/ClickHouse-Extras/boost.git url = https://github.com/ClickHouse-Extras/boost.git
[submodule "contrib/base64"] [submodule "contrib/base64"]
path = contrib/base64 path = contrib/base64
url = https://github.com/aklomp/base64.git url = https://github.com/powturbo/Turbo-Base64.git
[submodule "contrib/arrow"] [submodule "contrib/arrow"]
path = contrib/arrow path = contrib/arrow
url = https://github.com/apache/arrow url = https://github.com/apache/arrow
@ -134,6 +134,9 @@
[submodule "contrib/libc-headers"] [submodule "contrib/libc-headers"]
path = contrib/libc-headers path = contrib/libc-headers
url = https://github.com/ClickHouse-Extras/libc-headers.git url = https://github.com/ClickHouse-Extras/libc-headers.git
[submodule "contrib/replxx"]
path = contrib/replxx
url = https://github.com/AmokHuginnsson/replxx.git
[submodule "contrib/ryu"] [submodule "contrib/ryu"]
path = contrib/ryu path = contrib/ryu
url = https://github.com/ClickHouse-Extras/ryu.git url = https://github.com/ClickHouse-Extras/ryu.git

7
CMakeLists.txt
View File

@ -95,6 +95,8 @@ if (CMAKE_GENERATOR STREQUAL "Ninja")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always")
endif () endif ()
include (cmake/add_warning.cmake)
if (NOT MSVC) if (NOT MSVC)
set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror is also added inside directories with our own code. set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror is also added inside directories with our own code.
endif () endif ()
@ -224,8 +226,8 @@ else ()
set(NOT_UNBUNDLED 1) set(NOT_UNBUNDLED 1)
endif () endif ()
# Using system libs can cause lot of warnings in includes (on macro expansion). # Using system libs can cause a lot of warnings in includes (on macro expansion).
if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32) if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN) OR ARCH_32)
option (NO_WERROR "Disable -Werror compiler option" ON) option (NO_WERROR "Disable -Werror compiler option" ON)
endif () endif ()
@ -326,7 +328,6 @@ include (cmake/find/xxhash.cmake)
include (cmake/find/sparsehash.cmake) include (cmake/find/sparsehash.cmake)
include (cmake/find/rt.cmake) include (cmake/find/rt.cmake)
include (cmake/find/execinfo.cmake) include (cmake/find/execinfo.cmake)
include (cmake/find/readline_edit.cmake)
include (cmake/find/re2.cmake) include (cmake/find/re2.cmake)
include (cmake/find/libgsasl.cmake) include (cmake/find/libgsasl.cmake)
include (cmake/find/rdkafka.cmake) include (cmake/find/rdkafka.cmake)

14
CONTRIBUTING.md
View File

@ -1,10 +1,16 @@
# Contributing to ClickHouse # Contributing to ClickHouse
## Technical info ClickHouse is an open project, and you can contribute to it in many ways. You can help with ideas, code, or documentation. We appreciate any efforts that help us to make the project better.
Developer guide for writing code for ClickHouse is published on official website alongside the usage and operations documentation:
https://clickhouse.yandex/docs/en/development/architecture/
## Legal info Thank you.
## Technical Info
We have a [developer's guide](https://clickhouse.yandex/docs/en/development/developer_instruction/) for writing code for ClickHouse. Besides this guide, you can find [Overview of ClickHouse Architecture](https://clickhouse.yandex/docs/en/development/architecture/) and instructions on how to build ClickHouse in different environments.
If you want to contribute to documentation, read the [Contributing to ClickHouse Documentation](docs/README.md) guide.
## Legal Info
In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here: In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here:
1) https://yandex.ru/legal/cla/?lang=en (in English) and 1) https://yandex.ru/legal/cla/?lang=en (in English) and

18
cmake/add_warning.cmake Normal file
View File

@ -0,0 +1,18 @@
include (CheckCXXCompilerFlag)
# Try to add -Wflag if compiler supports it
macro (add_warning flag)
string (REPLACE "-" "_" underscored_flag ${flag})
string (REPLACE "+" "x" underscored_flag ${underscored_flag})
check_cxx_compiler_flag("-W${flag}" SUPPORTS_FLAG_${underscored_flag})
if (SUPPORTS_FLAG_${underscored_flag})
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W${flag}")
else ()
message (WARNING "Flag -W${flag} is unsupported")
endif ()
endmacro ()
# Try to add -Wno flag if compiler supports it
macro (no_warning flag)
add_warning(no-${flag})
endmacro ()

3
cmake/find/base64.cmake
View File

@ -1,4 +1,4 @@
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64/lib/lib.c") if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64/LICENSE")
set (MISSING_INTERNAL_BASE64_LIBRARY 1) set (MISSING_INTERNAL_BASE64_LIBRARY 1)
message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive") message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
endif () endif ()
@ -11,7 +11,6 @@ if (ENABLE_BASE64)
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64") if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64")
message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive") message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
else() else()
set (BASE64_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64/include)
set (BASE64_LIBRARY base64) set (BASE64_LIBRARY base64)
set (USE_BASE64 1) set (USE_BASE64 1)
endif() endif()

9
cmake/find/cxx.cmake
View File

@ -1,5 +1,12 @@
set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT ${NOT_UNBUNDLED})
if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/CMakeLists.txt")
message(WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive")
set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT 0)
endif()
option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${NOT_UNBUNDLED}) option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${NOT_UNBUNDLED})
option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${NOT_UNBUNDLED}) option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT})
if (USE_LIBCXX) if (USE_LIBCXX)
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build. set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build.

60
cmake/find/readline_edit.cmake
View File

@ -1,60 +0,0 @@
include (CMakePushCheckState)
cmake_push_check_state ()
option (ENABLE_READLINE "Enable readline" ${ENABLE_LIBRARIES})
if (ENABLE_READLINE)
set (READLINE_PATHS "/usr/local/opt/readline/lib")
# First try find custom lib for macos users (default lib without history support)
find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS} NO_DEFAULT_PATH)
if (NOT READLINE_LIB)
find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS})
endif ()
list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.2)
find_library (EDIT_LIB NAMES edit)
set(READLINE_INCLUDE_PATHS "/usr/local/opt/readline/include")
if (READLINE_LIB AND TERMCAP_LIBRARY)
find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS} NO_DEFAULT_PATH)
if (NOT READLINE_INCLUDE_DIR)
find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS})
endif ()
if (READLINE_INCLUDE_DIR AND READLINE_LIB)
set (USE_READLINE 1)
set (LINE_EDITING_LIBS ${READLINE_LIB} ${TERMCAP_LIBRARY})
message (STATUS "Using line editing libraries (readline): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}")
endif ()
elseif (EDIT_LIB AND TERMCAP_LIBRARY)
find_library (CURSES_LIB NAMES curses)
find_path (READLINE_INCLUDE_DIR NAMES editline/readline.h PATHS ${READLINE_INCLUDE_PATHS})
if (CURSES_LIB AND READLINE_INCLUDE_DIR)
set (USE_LIBEDIT 1)
set (LINE_EDITING_LIBS ${EDIT_LIB} ${CURSES_LIB} ${TERMCAP_LIBRARY})
message (STATUS "Using line editing libraries (edit): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}")
endif ()
endif ()
endif ()
if (LINE_EDITING_LIBS AND READLINE_INCLUDE_DIR)
include (CheckCXXSourceRuns)
set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${LINE_EDITING_LIBS})
set (CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES} ${READLINE_INCLUDE_DIR})
check_cxx_source_runs ("
#include <stdio.h>
#include <readline/readline.h>
#include <readline/history.h>
int main() {
add_history(NULL);
append_history(1,NULL);
return 0;
}
" HAVE_READLINE_HISTORY)
else ()
message (STATUS "Not using any library for line editing.")
endif ()
cmake_pop_check_state ()

1
cmake/sanitize.cmake
View File

@ -48,7 +48,6 @@ if (SANITIZE)
set (ENABLE_EMBEDDED_COMPILER 0 CACHE BOOL "") set (ENABLE_EMBEDDED_COMPILER 0 CACHE BOOL "")
set (USE_INTERNAL_CAPNP_LIBRARY 0 CACHE BOOL "") set (USE_INTERNAL_CAPNP_LIBRARY 0 CACHE BOOL "")
set (USE_SIMDJSON 0 CACHE BOOL "") set (USE_SIMDJSON 0 CACHE BOOL "")
set (ENABLE_READLINE 0 CACHE BOOL "")
set (ENABLE_ORC 0 CACHE BOOL "") set (ENABLE_ORC 0 CACHE BOOL "")
set (ENABLE_PARQUET 0 CACHE BOOL "") set (ENABLE_PARQUET 0 CACHE BOOL "")
set (USE_CAPNP 0 CACHE BOOL "") set (USE_CAPNP 0 CACHE BOOL "")

1
cmake/target.cmake
View File

@ -15,7 +15,6 @@ if (CMAKE_CROSSCOMPILING)
set (USE_SNAPPY OFF CACHE INTERNAL "") set (USE_SNAPPY OFF CACHE INTERNAL "")
set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") set (ENABLE_PROTOBUF OFF CACHE INTERNAL "")
set (ENABLE_PARQUET OFF CACHE INTERNAL "") set (ENABLE_PARQUET OFF CACHE INTERNAL "")
set (ENABLE_READLINE OFF CACHE INTERNAL "")
set (ENABLE_ICU OFF CACHE INTERNAL "") set (ENABLE_ICU OFF CACHE INTERNAL "")
set (ENABLE_FASTOPS OFF CACHE INTERNAL "") set (ENABLE_FASTOPS OFF CACHE INTERNAL "")
elseif (OS_LINUX) elseif (OS_LINUX)

2
contrib/CMakeLists.txt vendored
View File

@ -331,3 +331,5 @@ endif()
if (USE_FASTOPS) if (USE_FASTOPS)
add_subdirectory (fastops-cmake) add_subdirectory (fastops-cmake)
endif() endif()
add_subdirectory(replxx-cmake)

3
contrib/aws-s3-cmake/CMakeLists.txt
View File

@ -74,7 +74,6 @@ file(GLOB S3_UNIFIED_SRC
) )
set(S3_INCLUDES set(S3_INCLUDES
"${CMAKE_CURRENT_SOURCE_DIR}/include/"
"${AWS_COMMON_LIBRARY_DIR}/include/" "${AWS_COMMON_LIBRARY_DIR}/include/"
"${AWS_EVENT_STREAM_LIBRARY_DIR}/include/" "${AWS_EVENT_STREAM_LIBRARY_DIR}/include/"
"${AWS_S3_LIBRARY_DIR}/include/" "${AWS_S3_LIBRARY_DIR}/include/"
@ -96,7 +95,7 @@ target_compile_definitions(aws_s3 PUBLIC -DENABLE_CURL_CLIENT)
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MAJOR=1") target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MAJOR=1")
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MINOR=7") target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MINOR=7")
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_PATCH=231") target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_PATCH=231")
target_include_directories(aws_s3 PUBLIC ${S3_INCLUDES} "${CMAKE_BINARY_DIR}/install") target_include_directories(aws_s3 PUBLIC ${S3_INCLUDES})
if (OPENSSL_FOUND) if (OPENSSL_FOUND)
target_compile_definitions(aws_s3 PUBLIC -DENABLE_OPENSSL_ENCRYPTION) target_compile_definitions(aws_s3 PUBLIC -DENABLE_OPENSSL_ENCRYPTION)

2
contrib/base64 vendored

@ -1 +1 @@
Subproject commit 32a87227aa254e2f02332c0ed8bc375522414fa3 Subproject commit 5257626d2be17a3eb23f79be17fe55ebba394ad2

1
contrib/base64-cmake/.gitignore vendored
View File

@ -1 +0,0 @@
config.h

76
contrib/base64-cmake/CMakeLists.txt
View File

@ -1,59 +1,33 @@
SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64) SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64)
macro(cast_to_bool var instruction) add_library(base64_scalar OBJECT ${LIBRARY_DIR}/turbob64c.c ${LIBRARY_DIR}/turbob64d.c)
if (HAVE_${var}) add_library(base64_ssse3 OBJECT ${LIBRARY_DIR}/turbob64sse.c) # This file also contains code for ARM NEON
set(base64_${var} 1)
set(base64_${var}_opt ${instruction})
else()
set(base64_${var} 0)
endif()
endmacro()
cast_to_bool(NEON32 "") # TODO flags if (ARCH_AMD64)
cast_to_bool(NEON64 "") # TODO flags add_library(base64_avx OBJECT ${LIBRARY_DIR}/turbob64sse.c) # This is not a mistake. One file is compiled twice.
cast_to_bool(SSSE3 "-mssse3") add_library(base64_avx2 OBJECT ${LIBRARY_DIR}/turbob64avx2.c)
cast_to_bool(SSE41 "-msse4.1")
cast_to_bool(SSE42 "-msse4.2")
cast_to_bool(AVX "-mavx")
cast_to_bool(AVX2 "-mavx2")
set(HAVE_FAST_UNALIGNED_ACCESS 0)
if(HAVE_SSSE3 OR HAVE_SSE41 OR HAVE_SSE42 OR HAVE_AVX OR HAVE_AVX2)
set(HAVE_FAST_UNALIGNED_ACCESS 1)
endif () endif ()
configure_file(config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h) target_compile_options(base64_scalar PRIVATE -falign-loops)
add_library(base64 if (ARCH_AMD64)
${LIBRARY_DIR}/lib/lib.c target_compile_options(base64_ssse3 PRIVATE -mssse3 -falign-loops)
${LIBRARY_DIR}/lib/codec_choose.c target_compile_options(base64_avx PRIVATE -falign-loops -mavx)
${LIBRARY_DIR}/lib/arch/avx/codec.c target_compile_options(base64_avx2 PRIVATE -falign-loops -mavx2)
${LIBRARY_DIR}/lib/arch/avx2/codec.c else ()
${LIBRARY_DIR}/lib/arch/generic/codec.c target_compile_options(base64_ssse3 PRIVATE -falign-loops)
${LIBRARY_DIR}/lib/arch/neon32/codec.c endif ()
${LIBRARY_DIR}/lib/arch/neon64/codec.c
${LIBRARY_DIR}/lib/arch/sse41/codec.c
${LIBRARY_DIR}/lib/arch/sse42/codec.c
${LIBRARY_DIR}/lib/arch/ssse3/codec.c
${LIBRARY_DIR}/lib/codecs.h if (ARCH_AMD64)
${CMAKE_CURRENT_BINARY_DIR}/config.h add_library(base64
${CMAKE_CURRENT_SOURCE_DIR}/tables.h) $<TARGET_OBJECTS:base64_scalar>
$<TARGET_OBJECTS:base64_ssse3>
$<TARGET_OBJECTS:base64_avx>
$<TARGET_OBJECTS:base64_avx2>)
else ()
add_library(base64
$<TARGET_OBJECTS:base64_scalar>
$<TARGET_OBJECTS:base64_ssse3>)
endif ()
if(HAVE_AVX) target_include_directories(base64 SYSTEM PUBLIC ${LIBRARY_DIR})
set_source_files_properties(${LIBRARY_DIR}/lib/arch/avx/codec.c PROPERTIES COMPILE_FLAGS -mavx)
endif()
if(HAVE_AVX2)
set_source_files_properties(${LIBRARY_DIR}/lib/arch/avx2/codec.c PROPERTIES COMPILE_FLAGS -mavx2)
endif()
if(HAVE_SSE41)
set_source_files_properties(${LIBRARY_DIR}/lib/arch/sse41/codec.c PROPERTIES COMPILE_FLAGS -msse4.1)
endif()
if(HAVE_SSE42)
set_source_files_properties(${LIBRARY_DIR}/lib/arch/sse42/codec.c PROPERTIES COMPILE_FLAGS -msse4.2)
endif()
if(HAVE_SSSE3)
set_source_files_properties(${LIBRARY_DIR}/lib/arch/ssse3/codec.c PROPERTIES COMPILE_FLAGS -mssse3)
endif()
target_include_directories(base64 PRIVATE ${LIBRARY_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})

5
contrib/base64-cmake/README.md
View File

@ -1,5 +0,0 @@
`tables.h` file was generated by the base64 library with it's makefile:
```
cc -std=c99 -O3 -Wall -Wextra -pedantic -o lib/table_generator lib/table_generator.c
./lib/table_generator > lib/tables.h
```

8
contrib/base64-cmake/config.h.in
View File

@ -1,8 +0,0 @@
#define HAVE_NEON32 @base64_NEON32@
#define HAVE_NEON64 @base64_NEON64@
#cmakedefine HAVE_SSSE3 @base64_SSSE3@
#cmakedefine HAVE_SSE41 @base64_SSE41@
#cmakedefine HAVE_SSE42 @base64_SSE42@
#cmakedefine HAVE_AVX @base64_AVX@
#cmakedefine HAVE_AVX2 @base64_AVX2@
#cmakedefine HAVE_FAST_UNALIGNED_ACCESS @HAVE_FAST_UNALIGNED_ACCESS@

393
contrib/base64-cmake/tables.h
View File

@ -1,393 +0,0 @@
#include <stdint.h>
#define CHAR62 '+'
#define CHAR63 '/'
#define CHARPAD '='
#if BASE64_LITTLE_ENDIAN
/* SPECIAL DECODE TABLES FOR LITTLE ENDIAN (INTEL) CPUS */
const uint32_t base64_table_dec_d0[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x000000f8, 0xffffffff, 0xffffffff, 0xffffffff, 0x000000fc,
0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4,
0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018,
0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030,
0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048,
0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060,
0x00000064, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078,
0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090,
0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8,
0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0,
0x000000c4, 0x000000c8, 0x000000cc, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d1[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x0000e003, 0xffffffff, 0xffffffff, 0xffffffff, 0x0000f003,
0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003,
0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000,
0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000,
0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001,
0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001,
0x00009001, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001,
0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002,
0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002,
0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003,
0x00001003, 0x00002003, 0x00003003, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d2[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00800f00, 0xffffffff, 0xffffffff, 0xffffffff, 0x00c00f00,
0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00,
0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100,
0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300,
0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400,
0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600,
0x00400600, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700,
0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900,
0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00,
0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00,
0x00400c00, 0x00800c00, 0x00c00c00, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d3[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x003e0000, 0xffffffff, 0xffffffff, 0xffffffff, 0x003f0000,
0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000,
0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000,
0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000,
0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000,
0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000,
0x00190000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000,
0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000,
0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000,
0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000,
0x00310000, 0x00320000, 0x00330000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
#else
/* SPECIAL DECODE TABLES FOR BIG ENDIAN (IBM/MOTOROLA/SUN) CPUS */
const uint32_t base64_table_dec_d0[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xf8000000, 0xffffffff, 0xffffffff, 0xffffffff, 0xfc000000,
0xd0000000, 0xd4000000, 0xd8000000, 0xdc000000, 0xe0000000, 0xe4000000,
0xe8000000, 0xec000000, 0xf0000000, 0xf4000000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x04000000, 0x08000000, 0x0c000000, 0x10000000, 0x14000000, 0x18000000,
0x1c000000, 0x20000000, 0x24000000, 0x28000000, 0x2c000000, 0x30000000,
0x34000000, 0x38000000, 0x3c000000, 0x40000000, 0x44000000, 0x48000000,
0x4c000000, 0x50000000, 0x54000000, 0x58000000, 0x5c000000, 0x60000000,
0x64000000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x68000000, 0x6c000000, 0x70000000, 0x74000000, 0x78000000,
0x7c000000, 0x80000000, 0x84000000, 0x88000000, 0x8c000000, 0x90000000,
0x94000000, 0x98000000, 0x9c000000, 0xa0000000, 0xa4000000, 0xa8000000,
0xac000000, 0xb0000000, 0xb4000000, 0xb8000000, 0xbc000000, 0xc0000000,
0xc4000000, 0xc8000000, 0xcc000000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d1[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x03e00000, 0xffffffff, 0xffffffff, 0xffffffff, 0x03f00000,
0x03400000, 0x03500000, 0x03600000, 0x03700000, 0x03800000, 0x03900000,
0x03a00000, 0x03b00000, 0x03c00000, 0x03d00000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00100000, 0x00200000, 0x00300000, 0x00400000, 0x00500000, 0x00600000,
0x00700000, 0x00800000, 0x00900000, 0x00a00000, 0x00b00000, 0x00c00000,
0x00d00000, 0x00e00000, 0x00f00000, 0x01000000, 0x01100000, 0x01200000,
0x01300000, 0x01400000, 0x01500000, 0x01600000, 0x01700000, 0x01800000,
0x01900000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x01a00000, 0x01b00000, 0x01c00000, 0x01d00000, 0x01e00000,
0x01f00000, 0x02000000, 0x02100000, 0x02200000, 0x02300000, 0x02400000,
0x02500000, 0x02600000, 0x02700000, 0x02800000, 0x02900000, 0x02a00000,
0x02b00000, 0x02c00000, 0x02d00000, 0x02e00000, 0x02f00000, 0x03000000,
0x03100000, 0x03200000, 0x03300000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d2[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x000f8000, 0xffffffff, 0xffffffff, 0xffffffff, 0x000fc000,
0x000d0000, 0x000d4000, 0x000d8000, 0x000dc000, 0x000e0000, 0x000e4000,
0x000e8000, 0x000ec000, 0x000f0000, 0x000f4000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00004000, 0x00008000, 0x0000c000, 0x00010000, 0x00014000, 0x00018000,
0x0001c000, 0x00020000, 0x00024000, 0x00028000, 0x0002c000, 0x00030000,
0x00034000, 0x00038000, 0x0003c000, 0x00040000, 0x00044000, 0x00048000,
0x0004c000, 0x00050000, 0x00054000, 0x00058000, 0x0005c000, 0x00060000,
0x00064000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00068000, 0x0006c000, 0x00070000, 0x00074000, 0x00078000,
0x0007c000, 0x00080000, 0x00084000, 0x00088000, 0x0008c000, 0x00090000,
0x00094000, 0x00098000, 0x0009c000, 0x000a0000, 0x000a4000, 0x000a8000,
0x000ac000, 0x000b0000, 0x000b4000, 0x000b8000, 0x000bc000, 0x000c0000,
0x000c4000, 0x000c8000, 0x000cc000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d3[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00003e00, 0xffffffff, 0xffffffff, 0xffffffff, 0x00003f00,
0x00003400, 0x00003500, 0x00003600, 0x00003700, 0x00003800, 0x00003900,
0x00003a00, 0x00003b00, 0x00003c00, 0x00003d00, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00000100, 0x00000200, 0x00000300, 0x00000400, 0x00000500, 0x00000600,
0x00000700, 0x00000800, 0x00000900, 0x00000a00, 0x00000b00, 0x00000c00,
0x00000d00, 0x00000e00, 0x00000f00, 0x00001000, 0x00001100, 0x00001200,
0x00001300, 0x00001400, 0x00001500, 0x00001600, 0x00001700, 0x00001800,
0x00001900, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00001a00, 0x00001b00, 0x00001c00, 0x00001d00, 0x00001e00,
0x00001f00, 0x00002000, 0x00002100, 0x00002200, 0x00002300, 0x00002400,
0x00002500, 0x00002600, 0x00002700, 0x00002800, 0x00002900, 0x00002a00,
0x00002b00, 0x00002c00, 0x00002d00, 0x00002e00, 0x00002f00, 0x00003000,
0x00003100, 0x00003200, 0x00003300, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
#endif

10
contrib/icu-cmake/CMakeLists.txt
View File

@ -437,8 +437,12 @@ ${ICU_SOURCE_DIR}/i18n/formattedval_iterimpl.cpp
${ICU_SOURCE_DIR}/i18n/formattedval_sbimpl.cpp ${ICU_SOURCE_DIR}/i18n/formattedval_sbimpl.cpp
${ICU_SOURCE_DIR}/i18n/formatted_string_builder.cpp) ${ICU_SOURCE_DIR}/i18n/formatted_string_builder.cpp)
file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/empty.cpp CONTENT " ")
enable_language(ASM) enable_language(ASM)
set(ICUDATA_SOURCES ${ICUDATA_SOURCE_DIR}/icudt66l_dat.S) set(ICUDATA_SOURCES
${ICUDATA_SOURCE_DIR}/icudt66l_dat.S
${CMAKE_CURRENT_BINARY_DIR}/empty.cpp # Without this cmake can incorrectly detects library type (OBJECT) instead of SHARED/STATIC
)
# Note that we don't like any kind of binary plugins (because of runtime dependencies, vulnerabilities, ABI incompatibilities). # Note that we don't like any kind of binary plugins (because of runtime dependencies, vulnerabilities, ABI incompatibilities).
add_definitions(-D_REENTRANT -DU_HAVE_ELF_H=1 -DU_HAVE_STRTOD_L=1 -DU_HAVE_XLOCALE_H=0 -DDEFAULT_ICU_PLUGINS="/dev/null") add_definitions(-D_REENTRANT -DU_HAVE_ELF_H=1 -DU_HAVE_STRTOD_L=1 -DU_HAVE_XLOCALE_H=0 -DDEFAULT_ICU_PLUGINS="/dev/null")
@ -447,8 +451,8 @@ add_library(icuuc ${ICUUC_SOURCES})
add_library(icui18n ${ICUI18N_SOURCES}) add_library(icui18n ${ICUI18N_SOURCES})
add_library(icudata ${ICUDATA_SOURCES}) add_library(icudata ${ICUDATA_SOURCES})
target_link_libraries(icuuc icudata) target_link_libraries(icuuc PRIVATE icudata)
target_link_libraries(icui18n icuuc) target_link_libraries(icui18n PRIVATE icuuc)
target_include_directories(icuuc SYSTEM PUBLIC ${ICU_SOURCE_DIR}/common/) target_include_directories(icuuc SYSTEM PUBLIC ${ICU_SOURCE_DIR}/common/)
target_include_directories(icui18n SYSTEM PUBLIC ${ICU_SOURCE_DIR}/i18n/) target_include_directories(icui18n SYSTEM PUBLIC ${ICU_SOURCE_DIR}/i18n/)

7
contrib/libcxx-cmake/CMakeLists.txt
View File

@ -54,13 +54,12 @@ endif ()
target_compile_options(cxx PUBLIC $<$<COMPILE_LANGUAGE:CXX>:-nostdinc++>) target_compile_options(cxx PUBLIC $<$<COMPILE_LANGUAGE:CXX>:-nostdinc++>)
check_cxx_compiler_flag(-Wreserved-id-macro HAVE_WARNING_RESERVED_ID_MACRO)
if (HAVE_WARNING_RESERVED_ID_MACRO) if (SUPPORTS_FLAG_no_reserved_id_macro)
target_compile_options(cxx PUBLIC -Wno-reserved-id-macro) target_compile_options(cxx PUBLIC -Wno-reserved-id-macro)
endif () endif ()
check_cxx_compiler_flag(-Wctad-maybe-unsupported HAVE_WARNING_CTAD_MAYBE_UNSUPPORTED) if (SUPPORTS_FLAG_no_ctad_maybe_unsupported)
if (HAVE_WARNING_CTAD_MAYBE_UNSUPPORTED)
target_compile_options(cxx PUBLIC -Wno-ctad-maybe-unsupported) target_compile_options(cxx PUBLIC -Wno-ctad-maybe-unsupported)
endif () endif ()

1
contrib/replxx vendored Submodule

@ -0,0 +1 @@
Subproject commit 37582f0bb8c52513c6c6b76797c02d852d701dad

57
contrib/replxx-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,57 @@
option (ENABLE_REPLXX "Enable replxx support" ${ENABLE_LIBRARIES})
if (ENABLE_REPLXX)
option (USE_INTERNAL_REPLXX "Use internal replxx library" ${NOT_UNBUNDLED})
if (USE_INTERNAL_REPLXX)
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx")
set(SRCS
${LIBRARY_DIR}/src/conversion.cxx
${LIBRARY_DIR}/src/ConvertUTF.cpp
${LIBRARY_DIR}/src/escape.cxx
${LIBRARY_DIR}/src/history.cxx
${LIBRARY_DIR}/src/io.cxx
${LIBRARY_DIR}/src/prompt.cxx
${LIBRARY_DIR}/src/replxx_impl.cxx
${LIBRARY_DIR}/src/replxx.cxx
${LIBRARY_DIR}/src/util.cxx
${LIBRARY_DIR}/src/wcwidth.cpp
)
add_library (replxx ${SRCS})
target_include_directories(replxx PUBLIC ${LIBRARY_DIR}/include)
else ()
find_library(LIBRARY_REPLXX NAMES replxx replxx-static)
find_path(INCLUDE_REPLXX replxx.hxx)
add_library(replxx UNKNOWN IMPORTED)
set_property(TARGET replxx PROPERTY IMPORTED_LOCATION ${LIBRARY_REPLXX})
target_include_directories(replxx PUBLIC ${INCLUDE_REPLXX})
set(CMAKE_REQUIRED_LIBRARIES replxx)
check_cxx_source_compiles(
"
#include <replxx.hxx>
int main() {
replxx::Replxx rx;
}
"
EXTERNAL_REPLXX_WORKS
)
if (NOT EXTERNAL_REPLXX_WORKS)
message (FATAL_ERROR "replxx is unusable: ${LIBRARY_REPLXX} ${INCLUDE_REPLXX}")
endif ()
endif ()
target_compile_options(replxx PUBLIC -Wno-documentation)
target_compile_definitions(replxx PUBLIC USE_REPLXX=1)
message (STATUS "Using replxx")
else ()
add_library(replxx INTERFACE)
target_compile_definitions(replxx INTERFACE USE_REPLXX=0)
message (STATUS "Not using replxx (Beware! Runtime fallback to readline is possible!)")
endif ()

106
dbms/CMakeLists.txt
View File

@ -45,36 +45,75 @@ endif ()
option (WEVERYTHING "Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang." ON) option (WEVERYTHING "Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang." ON)
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") if (COMPILER_CLANG)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpedantic -Wno-vla-extension -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types") add_warning(pedantic)
no_warning(gnu-anonymous-struct)
no_warning(nested-anon-types)
no_warning(vla-extension)
no_warning(zero-length-array)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow -Wshadow-uncaptured-local -Wextra-semi -Wcomma -Winconsistent-missing-destructor-override -Wunused-exception-parameter -Wcovered-switch-default -Wold-style-cast -Wrange-loop-analysis -Wunused-member-function -Wunreachable-code -Wunreachable-code-return -Wnewline-eof -Wembedded-directive -Wgnu-case-range -Wunused-macros -Wconditional-uninitialized -Wdeprecated -Wundef -Wreserved-id-macro -Wredundant-parens -Wzero-as-null-pointer-constant") add_warning(comma)
add_warning(conditional-uninitialized)
add_warning(covered-switch-default)
add_warning(deprecated)
add_warning(embedded-directive)
add_warning(empty-init-stmt) # linux-only
add_warning(extra-semi-stmt) # linux-only
add_warning(extra-semi)
add_warning(gnu-case-range)
add_warning(inconsistent-missing-destructor-override)
add_warning(newline-eof)
add_warning(old-style-cast)
add_warning(range-loop-analysis)
add_warning(redundant-parens)
add_warning(reserved-id-macro)
add_warning(shadow-field) # clang 8+
add_warning(shadow-uncaptured-local)
add_warning(shadow)
add_warning(string-plus-int) # clang 8+
add_warning(undef)
add_warning(unreachable-code-return)
add_warning(unreachable-code)
add_warning(unused-exception-parameter)
add_warning(unused-macros)
add_warning(unused-member-function)
add_warning(zero-as-null-pointer-constant)
if (WEVERYTHING) if (WEVERYTHING)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-padded -Wno-switch-enum -Wno-deprecated-dynamic-exception-spec -Wno-float-equal -Wno-weak-vtables -Wno-shift-sign-overflow -Wno-sign-conversion -Wno-conversion -Wno-exit-time-destructors -Wno-undefined-func-template -Wno-documentation-unknown-command -Wno-missing-variable-declarations -Wno-unused-template -Wno-global-constructors -Wno-c99-extensions -Wno-missing-prototypes -Wno-weak-template-vtables -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types -Wno-double-promotion -Wno-disabled-macro-expansion -Wno-vla-extension -Wno-vla -Wno-packed") add_warning(everything)
no_warning(c++98-compat-pedantic)
no_warning(c++98-compat)
no_warning(c99-extensions)
no_warning(conversion)
no_warning(ctad-maybe-unsupported) # clang 9+, linux-only
no_warning(deprecated-dynamic-exception-spec)
no_warning(disabled-macro-expansion)
no_warning(documentation-unknown-command)
no_warning(double-promotion)
no_warning(exit-time-destructors)
no_warning(float-equal)
no_warning(global-constructors)
no_warning(gnu-anonymous-struct)
no_warning(missing-prototypes)
no_warning(missing-variable-declarations)
no_warning(nested-anon-types)
no_warning(packed)
no_warning(padded)
no_warning(return-std-move-in-c++11) # clang 7+
no_warning(shift-sign-overflow)
no_warning(sign-conversion)
no_warning(switch-enum)
no_warning(undefined-func-template)
no_warning(unused-template)
no_warning(vla-extension)
no_warning(vla)
no_warning(weak-template-vtables)
no_warning(weak-vtables)
no_warning(zero-length-array)
# TODO Enable conversion, sign-conversion, double-promotion warnings. # TODO Enable conversion, sign-conversion, double-promotion warnings.
endif () endif ()
elseif (COMPILER_GCC)
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7)
if (WEVERYTHING)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-std-move-in-c++11")
endif ()
endif ()
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow-field -Wstring-plus-int")
if(NOT APPLE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra-semi-stmt -Wempty-init-stmt")
endif()
endif ()
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9)
if (WEVERYTHING AND NOT APPLE)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-ctad-maybe-unsupported")
endif ()
endif ()
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
# Add compiler options only to c++ compiler # Add compiler options only to c++ compiler
function(add_cxx_compile_options option) function(add_cxx_compile_options option)
add_compile_options("$<$<STREQUAL:$<TARGET_PROPERTY:LINKER_LANGUAGE>,CXX>:${option}>") add_compile_options("$<$<STREQUAL:$<TARGET_PROPERTY:LINKER_LANGUAGE>,CXX>:${option}>")
@ -103,10 +142,10 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
add_cxx_compile_options(-Wmaybe-uninitialized) add_cxx_compile_options(-Wmaybe-uninitialized)
# Warn when the indentation of the code does not reflect the block structure # Warn when the indentation of the code does not reflect the block structure
add_cxx_compile_options(-Wmisleading-indentation) add_cxx_compile_options(-Wmisleading-indentation)
# Warn if a global function is defined without a previous declaration # Warn if a global function is defined without a previous declaration - disabled because of build times
# add_cxx_compile_options(-Wmissing-declarations) # add_cxx_compile_options(-Wmissing-declarations)
# Warn if a user-supplied include directory does not exist # Warn if a user-supplied include directory does not exist
# add_cxx_compile_options(-Wmissing-include-dirs) add_cxx_compile_options(-Wmissing-include-dirs)
# Obvious # Obvious
add_cxx_compile_options(-Wnon-virtual-dtor) add_cxx_compile_options(-Wnon-virtual-dtor)
# Obvious # Obvious
@ -138,7 +177,7 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
# Warn for suspicious length parameters to certain string and memory built-in functions if the argument uses sizeof # Warn for suspicious length parameters to certain string and memory built-in functions if the argument uses sizeof
add_cxx_compile_options(-Wsizeof-pointer-memaccess) add_cxx_compile_options(-Wsizeof-pointer-memaccess)
# Warn about overriding virtual functions that are not marked with the override keyword # Warn about overriding virtual functions that are not marked with the override keyword
# add_cxx_compile_options(-Wsuggest-override) add_cxx_compile_options(-Wsuggest-override)
# Warn whenever a switch statement has an index of boolean type and the case values are outside the range of a boolean type # Warn whenever a switch statement has an index of boolean type and the case values are outside the range of a boolean type
add_cxx_compile_options(-Wswitch-bool) add_cxx_compile_options(-Wswitch-bool)
# Warn if a self-comparison always evaluates to true or false # Warn if a self-comparison always evaluates to true or false
@ -156,7 +195,7 @@ if (USE_DEBUG_HELPERS)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${INCLUDE_DEBUG_HELPERS}") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${INCLUDE_DEBUG_HELPERS}")
endif () endif ()
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") if (COMPILER_GCC)
# If we leave this optimization enabled, gcc-7 replaces a pair of SSE intrinsics (16 byte load, store) with a call to memcpy. # If we leave this optimization enabled, gcc-7 replaces a pair of SSE intrinsics (16 byte load, store) with a call to memcpy.
# It leads to slow code. This is compiler bug. It looks like this: # It leads to slow code. This is compiler bug. It looks like this:
# #
@ -524,7 +563,7 @@ if (USE_JEMALLOC)
endif() endif()
endif () endif ()
dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/Formats/include) dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR})
target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR}) target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR})
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR}) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR})
@ -546,7 +585,14 @@ if (ENABLE_TESTS AND USE_GTEST)
add_executable(unit_tests_dbms ${dbms_gtest_sources}) add_executable(unit_tests_dbms ${dbms_gtest_sources})
# gtest framework has substandard code # gtest framework has substandard code
target_compile_options(unit_tests_dbms PRIVATE -Wno-zero-as-null-pointer-constant -Wno-undef -Wno-sign-compare -Wno-used-but-marked-unused -Wno-missing-noreturn) target_compile_options(unit_tests_dbms PRIVATE
-Wno-zero-as-null-pointer-constant
-Wno-undef
-Wno-sign-compare
-Wno-used-but-marked-unused
-Wno-missing-noreturn
-Wno-gnu-zero-variadic-macro-arguments
)
target_link_libraries(unit_tests_dbms PRIVATE ${GTEST_BOTH_LIBRARIES} clickhouse_functions clickhouse_parsers dbms clickhouse_common_zookeeper string_utils) target_link_libraries(unit_tests_dbms PRIVATE ${GTEST_BOTH_LIBRARIES} clickhouse_functions clickhouse_parsers dbms clickhouse_common_zookeeper string_utils)
add_check(unit_tests_dbms) add_check(unit_tests_dbms)

10
dbms/cmake/version.cmake
View File

@ -1,11 +1,11 @@
# This strings autochanged from release_lib.sh: # This strings autochanged from release_lib.sh:
set(VERSION_REVISION 54431) set(VERSION_REVISION 54432)
set(VERSION_MAJOR 20) set(VERSION_MAJOR 20)
set(VERSION_MINOR 1) set(VERSION_MINOR 2)
set(VERSION_PATCH 1) set(VERSION_PATCH 1)
set(VERSION_GITHASH 51d4c8a53be94504e3607b2232e12e5ef7a8ec28) set(VERSION_GITHASH 4b9acaaa9099e71c36e5c818031149c5cba2bbdb)
set(VERSION_DESCRIBE v20.1.1.1-prestable) set(VERSION_DESCRIBE v20.2.1.1-prestable)
set(VERSION_STRING 20.1.1.1) set(VERSION_STRING 20.2.1.1)
# end of autochange # end of autochange
set(VERSION_EXTRA "" CACHE STRING "") set(VERSION_EXTRA "" CACHE STRING "")

12
dbms/programs/benchmark/Benchmark.cpp
View File

@ -101,7 +101,7 @@ public:
} }
void initialize(Poco::Util::Application & self [[maybe_unused]]) void initialize(Poco::Util::Application & self [[maybe_unused]]) override
{ {
std::string home_path; std::string home_path;
const char * home_path_cstr = getenv("HOME"); const char * home_path_cstr = getenv("HOME");
@ -111,7 +111,7 @@ public:
configReadClient(config(), home_path); configReadClient(config(), home_path);
} }
int main(const std::vector<std::string> &) int main(const std::vector<std::string> &) override
{ {
if (!json_path.empty() && Poco::File(json_path).exists()) /// Clear file with previous results if (!json_path.empty() && Poco::File(json_path).exists()) /// Clear file with previous results
Poco::File(json_path).remove(); Poco::File(json_path).remove();
@ -254,7 +254,7 @@ private:
if (interrupt_listener.check()) if (interrupt_listener.check())
{ {
std::cout << "Stopping launch of queries. SIGINT recieved.\n"; std::cout << "Stopping launch of queries. SIGINT received.\n";
return false; return false;
} }
@ -418,7 +418,7 @@ private:
std::cerr << percent << "%\t\t"; std::cerr << percent << "%\t\t";
for (const auto & info : infos) for (const auto & info : infos)
{ {
std::cerr << info->sampler.quantileInterpolated(percent / 100.0) << " sec." << "\t"; std::cerr << info->sampler.quantileNearest(percent / 100.0) << " sec." << "\t";
} }
std::cerr << "\n"; std::cerr << "\n";
}; };
@ -453,7 +453,7 @@ private:
auto print_percentile = [&json_out](Stats & info, auto percent, bool with_comma = true) auto print_percentile = [&json_out](Stats & info, auto percent, bool with_comma = true)
{ {
json_out << "\"" << percent << "\"" << ": " << info.sampler.quantileInterpolated(percent / 100.0) << (with_comma ? ",\n" : "\n"); json_out << "\"" << percent << "\"" << ": " << info.sampler.quantileNearest(percent / 100.0) << (with_comma ? ",\n" : "\n");
}; };
json_out << "{\n"; json_out << "{\n";
@ -492,7 +492,7 @@ private:
public: public:
~Benchmark() ~Benchmark() override
{ {
shutdown = true; shutdown = true;
} }

8
dbms/programs/client/CMakeLists.txt
View File

@ -1,14 +1,10 @@
set(CLICKHOUSE_CLIENT_SOURCES set(CLICKHOUSE_CLIENT_SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/Client.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Client.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ConnectionParameters.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ConnectionParameters.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Suggest.cpp
) )
set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY}) set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${Boost_PROGRAM_OPTIONS_LIBRARY})
set(CLICKHOUSE_CLIENT_INCLUDE PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include)
if (READLINE_INCLUDE_DIR)
set(CLICKHOUSE_CLIENT_INCLUDE ${CLICKHOUSE_CLIENT_INCLUDE} SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
endif ()
include(CheckSymbolExists) include(CheckSymbolExists)
check_symbol_exists(readpassphrase readpassphrase.h HAVE_READPASSPHRASE) check_symbol_exists(readpassphrase readpassphrase.h HAVE_READPASSPHRASE)

292
dbms/programs/client/Client.cpp
View File

@ -1,7 +1,13 @@
#include "TestHint.h" #include "TestHint.h"
#include "ConnectionParameters.h" #include "ConnectionParameters.h"
#include "Suggest.h"
#if USE_REPLXX
# include <common/ReplxxLineReader.h>
#else
# include <common/LineReader.h>
#endif
#include <port/unistd.h>
#include <stdlib.h> #include <stdlib.h>
#include <fcntl.h> #include <fcntl.h>
#include <signal.h> #include <signal.h>
@ -18,8 +24,8 @@
#include <Poco/String.h> #include <Poco/String.h>
#include <Poco/File.h> #include <Poco/File.h>
#include <Poco/Util/Application.h> #include <Poco/Util/Application.h>
#include <common/readline_use.h>
#include <common/find_symbols.h> #include <common/find_symbols.h>
#include <common/LineReader.h>
#include <Common/ClickHouseRevision.h> #include <Common/ClickHouseRevision.h>
#include <Common/Stopwatch.h> #include <Common/Stopwatch.h>
#include <Common/Exception.h> #include <Common/Exception.h>
@ -69,10 +75,6 @@
#include <common/argsToConfig.h> #include <common/argsToConfig.h>
#include <Common/TerminalSize.h> #include <Common/TerminalSize.h>
#if USE_READLINE
#include "Suggest.h"
#endif
#ifndef __clang__ #ifndef __clang__
#pragma GCC optimize("-fno-var-tracking-assignments") #pragma GCC optimize("-fno-var-tracking-assignments")
#endif #endif
@ -89,39 +91,6 @@
#define DISABLE_LINE_WRAPPING "\033[?7l" #define DISABLE_LINE_WRAPPING "\033[?7l"
#define ENABLE_LINE_WRAPPING "\033[?7h" #define ENABLE_LINE_WRAPPING "\033[?7h"
#if USE_READLINE && RL_VERSION_MAJOR >= 7
#define BRACK_PASTE_PREF "\033[200~"
#define BRACK_PASTE_SUFF "\033[201~"
#define BRACK_PASTE_LAST '~'
#define BRACK_PASTE_SLEN 6
/// This handler bypasses some unused macro/event checkings.
static int clickhouse_rl_bracketed_paste_begin(int /* count */, int /* key */)
{
std::string buf;
buf.reserve(128);
RL_SETSTATE(RL_STATE_MOREINPUT);
SCOPE_EXIT(RL_UNSETSTATE(RL_STATE_MOREINPUT));
int c;
while ((c = rl_read_key()) >= 0)
{
if (c == '\r')
c = '\n';
buf.push_back(c);
if (buf.size() >= BRACK_PASTE_SLEN && c == BRACK_PASTE_LAST && buf.substr(buf.size() - BRACK_PASTE_SLEN) == BRACK_PASTE_SUFF)
{
buf.resize(buf.size() - BRACK_PASTE_SLEN);
break;
}
}
return static_cast<size_t>(rl_insert_text(buf.c_str())) == buf.size() ? 0 : 1;
}
#endif
namespace DB namespace DB
{ {
@ -135,9 +104,7 @@ namespace ErrorCodes
extern const int UNKNOWN_PACKET_FROM_SERVER; extern const int UNKNOWN_PACKET_FROM_SERVER;
extern const int UNEXPECTED_PACKET_FROM_SERVER; extern const int UNEXPECTED_PACKET_FROM_SERVER;
extern const int CLIENT_OUTPUT_FORMAT_SPECIFIED; extern const int CLIENT_OUTPUT_FORMAT_SPECIFIED;
extern const int LOGICAL_ERROR;
extern const int CANNOT_SET_SIGNAL_HANDLER; extern const int CANNOT_SET_SIGNAL_HANDLER;
extern const int CANNOT_READLINE;
extern const int SYSTEM_ERROR; extern const int SYSTEM_ERROR;
extern const int INVALID_USAGE_OF_INPUT; extern const int INVALID_USAGE_OF_INPUT;
} }
@ -158,7 +125,7 @@ private:
"учшеж", "йгшеж", "дщпщгеж", "учшеж", "йгшеж", "дщпщгеж",
"q", "й", "\\q", "\\Q", "\\й", "\\Й", ":q", "Жй" "q", "й", "\\q", "\\Q", "\\й", "\\Й", ":q", "Жй"
}; };
bool is_interactive = true; /// Use either readline interface or batch mode. bool is_interactive = true; /// Use either interactive line editing interface or batch mode.
bool need_render_progress = true; /// Render query execution progress. bool need_render_progress = true; /// Render query execution progress.
bool echo_queries = false; /// Print queries before execution in batch mode. bool echo_queries = false; /// Print queries before execution in batch mode.
bool ignore_error = false; /// In case of errors, don't print error message, continue to next query. Only applicable for non-interactive mode. bool ignore_error = false; /// In case of errors, don't print error message, continue to next query. Only applicable for non-interactive mode.
@ -243,7 +210,7 @@ private:
ConnectionParameters connection_parameters; ConnectionParameters connection_parameters;
void initialize(Poco::Util::Application & self) void initialize(Poco::Util::Application & self) override
{ {
Poco::Util::Application::initialize(self); Poco::Util::Application::initialize(self);
@ -271,7 +238,7 @@ private:
} }
int main(const std::vector<std::string> & /*args*/) int main(const std::vector<std::string> & /*args*/) override
{ {
try try
{ {
@ -515,26 +482,10 @@ private:
if (print_time_to_stderr) if (print_time_to_stderr)
throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS); throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS);
#if USE_READLINE if (server_revision >= Suggest::MIN_SERVER_REVISION && !config().getBool("disable_suggestion", false))
SCOPE_EXIT({ Suggest::instance().finalize(); });
if (server_revision >= Suggest::MIN_SERVER_REVISION
&& !config().getBool("disable_suggestion", false))
{
/// Load suggestion data from the server. /// Load suggestion data from the server.
Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit")); Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit"));
/// Added '.' to the default list. Because it is used to separate database and table.
rl_basic_word_break_characters = " \t\n\r\"\\'`@$><=;|&{(.";
/// Not append whitespace after single suggestion. Because whitespace after function name is meaningless.
rl_completion_append_character = '\0';
rl_completion_entry_function = Suggest::generator;
}
else
/// Turn tab completion off.
rl_bind_key('\t', rl_insert);
#endif
/// Load command history if present. /// Load command history if present.
if (config().has("history_file")) if (config().has("history_file"))
history_file = config().getString("history_file"); history_file = config().getString("history_file");
@ -547,70 +498,49 @@ private:
history_file = home_path + "/.clickhouse-client-history"; history_file = home_path + "/.clickhouse-client-history";
} }
if (!history_file.empty()) if (!history_file.empty() && !Poco::File(history_file).exists())
{ Poco::File(history_file).createFile();
if (Poco::File(history_file).exists())
{ #if USE_REPLXX
#if USE_READLINE ReplxxLineReader lr(Suggest::instance(), history_file, '\\', config().has("multiline") ? ';' : 0);
int res = read_history(history_file.c_str()); #else
if (res) LineReader lr(history_file, '\\', config().has("multiline") ? ';' : 0);
std::cerr << "Cannot read history from file " + history_file + ": "+ errnoToString(ErrorCodes::CANNOT_READ_HISTORY);
#endif #endif
do
{
auto input = lr.readLine(prompt(), ":-] ");
if (input.empty())
break;
try
{
if (!process(input))
break;
}
catch (const Exception & e)
{
actual_client_error = e.code();
if (!actual_client_error || actual_client_error != expected_client_error)
{
std::cerr << std::endl
<< "Exception on client:" << std::endl
<< "Code: " << e.code() << ". " << e.displayText() << std::endl;
if (config().getBool("stacktrace", false))
std::cerr << "Stack trace:" << std::endl << e.getStackTraceString() << std::endl;
std::cerr << std::endl;
}
/// Client-side exception during query execution can result in the loss of
/// sync in the connection protocol.
/// So we reconnect and allow to enter the next query.
connect();
} }
else /// Create history file.
Poco::File(history_file).createFile();
} }
while (true);
#if USE_READLINE
/// Install Ctrl+C signal handler that will be used in interactive mode.
if (rl_initialize())
throw Exception("Cannot initialize readline", ErrorCodes::CANNOT_READLINE);
#if RL_VERSION_MAJOR >= 7
/// Enable bracketed-paste-mode only when multiquery is enabled and multiline is
/// disabled, so that we are able to paste and execute multiline queries in a whole
/// instead of erroring out, while be less intrusive.
if (config().has("multiquery") && !config().has("multiline"))
{
/// When bracketed paste mode is set, pasted text is bracketed with control sequences so
/// that the program can differentiate pasted text from typed-in text. This helps
/// clickhouse-client so that without -m flag, one can still paste multiline queries, and
/// possibly get better pasting performance. See https://cirw.in/blog/bracketed-paste for
/// more details.
rl_variable_bind("enable-bracketed-paste", "on");
/// Use our bracketed paste handler to get better user experience. See comments above.
rl_bind_keyseq(BRACK_PASTE_PREF, clickhouse_rl_bracketed_paste_begin);
}
#endif
auto clear_prompt_or_exit = [](int)
{
/// This is signal safe.
ssize_t res = write(STDOUT_FILENO, "\n", 1);
/// Allow to quit client while query is in progress by pressing Ctrl+C twice.
/// (First press to Ctrl+C will try to cancel query by InterruptListener).
if (res == 1 && rl_line_buffer[0] && !RL_ISSTATE(RL_STATE_DONE))
{
rl_replace_line("", 0);
if (rl_forced_update_display())
_exit(0);
}
else
{
/// A little dirty, but we struggle to find better way to correctly
/// force readline to exit after returning from the signal handler.
_exit(0);
}
};
if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR)
throwFromErrno("Cannot set signal handler.", ErrorCodes::CANNOT_SET_SIGNAL_HANDLER);
#endif
loop();
if (isNewYearMode()) if (isNewYearMode())
std::cout << "Happy new year." << std::endl; std::cout << "Happy new year." << std::endl;
@ -622,17 +552,6 @@ private:
} }
else else
{ {
/// This is intended for testing purposes.
if (config().getBool("always_load_suggestion_data", false))
{
#if USE_READLINE
SCOPE_EXIT({ Suggest::instance().finalize(); });
Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit"));
#else
throw Exception("Command line suggestions cannot work without readline", ErrorCodes::BAD_ARGUMENTS);
#endif
}
query_id = config().getString("query_id", ""); query_id = config().getString("query_id", "");
nonInteractive(); nonInteractive();
@ -707,111 +626,11 @@ private:
} }
/// Check if multi-line query is inserted from the paste buffer.
/// Allows delaying the start of query execution until the entirety of query is inserted.
static bool hasDataInSTDIN()
{
timeval timeout = { 0, 0 };
fd_set fds;
FD_ZERO(&fds);
FD_SET(STDIN_FILENO, &fds);
return select(1, &fds, nullptr, nullptr, &timeout) == 1;
}
inline const String prompt() const inline const String prompt() const
{ {
return boost::replace_all_copy(prompt_by_server_display_name, "{database}", config().getString("database", "default")); return boost::replace_all_copy(prompt_by_server_display_name, "{database}", config().getString("database", "default"));
} }
void loop()
{
String input;
String prev_input;
while (char * line_ = readline(input.empty() ? prompt().c_str() : ":-] "))
{
String line = line_;
free(line_);
size_t ws = line.size();
while (ws > 0 && isWhitespaceASCII(line[ws - 1]))
--ws;
if (ws == 0 || line.empty())
continue;
bool ends_with_semicolon = line[ws - 1] == ';';
bool ends_with_backslash = line[ws - 1] == '\\';
has_vertical_output_suffix = (ws >= 2) && (line[ws - 2] == '\\') && (line[ws - 1] == 'G');
if (ends_with_backslash)
line = line.substr(0, ws - 1);
input += line;
if (!ends_with_backslash && (ends_with_semicolon || has_vertical_output_suffix || (!config().has("multiline") && !hasDataInSTDIN())))
{
// TODO: should we do sensitive data masking on client too? History file can be source of secret leaks.
if (input != prev_input)
{
/// Replace line breaks with spaces to prevent the following problem.
/// Every line of multi-line query is saved to history file as a separate line.
/// If the user restarts the client then after pressing the "up" button
/// every line of the query will be displayed separately.
std::string logged_query = input;
if (config().has("multiline"))
std::replace(logged_query.begin(), logged_query.end(), '\n', ' ');
add_history(logged_query.c_str());
#if USE_READLINE && HAVE_READLINE_HISTORY
if (!history_file.empty() && append_history(1, history_file.c_str()))
std::cerr << "Cannot append history to file " + history_file + ": " + errnoToString(ErrorCodes::CANNOT_APPEND_HISTORY);
#endif
prev_input = input;
}
if (has_vertical_output_suffix)
input = input.substr(0, input.length() - 2);
try
{
if (!process(input))
break;
}
catch (const Exception & e)
{
actual_client_error = e.code();
if (!actual_client_error || actual_client_error != expected_client_error)
{
std::cerr << std::endl
<< "Exception on client:" << std::endl
<< "Code: " << e.code() << ". " << e.displayText() << std::endl;
if (config().getBool("stacktrace", false))
std::cerr << "Stack trace:" << std::endl
<< e.getStackTraceString() << std::endl;
std::cerr << std::endl;
}
/// Client-side exception during query execution can result in the loss of
/// sync in the connection protocol.
/// So we reconnect and allow to enter the next query.
connect();
}
input = "";
}
else
{
input += '\n';
}
}
}
void nonInteractive() void nonInteractive()
{ {
@ -2002,13 +1821,6 @@ public:
server_logs_file = options["server_logs_file"].as<std::string>(); server_logs_file = options["server_logs_file"].as<std::string>();
if (options.count("disable_suggestion")) if (options.count("disable_suggestion"))
config().setBool("disable_suggestion", true); config().setBool("disable_suggestion", true);
if (options.count("always_load_suggestion_data"))
{
if (options.count("disable_suggestion"))
throw Exception("Command line parameters disable_suggestion (-A) and always_load_suggestion_data cannot be specified simultaneously",
ErrorCodes::BAD_ARGUMENTS);
config().setBool("always_load_suggestion_data", true);
}
if (options.count("suggestion_limit")) if (options.count("suggestion_limit"))
config().setInt("suggestion_limit", options["suggestion_limit"].as<int>()); config().setInt("suggestion_limit", options["suggestion_limit"].as<int>());

144
dbms/programs/client/Suggest.cpp Normal file
View File

@ -0,0 +1,144 @@
#include "Suggest.h"
#include <Columns/ColumnString.h>
#include <Common/typeid_cast.h>
namespace DB
{
void Suggest::load(const ConnectionParameters & connection_parameters, size_t suggestion_limit)
{
loading_thread = std::thread([connection_parameters, suggestion_limit, this]
{
try
{
Connection connection(
connection_parameters.host,
connection_parameters.port,
connection_parameters.default_database,
connection_parameters.user,
connection_parameters.password,
"client",
connection_parameters.compression,
connection_parameters.security);
loadImpl(connection, connection_parameters.timeouts, suggestion_limit);
}
catch (...)
{
std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n";
}
/// Note that keyword suggestions are available even if we cannot load data from server.
std::sort(words.begin(), words.end());
ready = true;
});
}
Suggest::Suggest()
{
/// Keywords may be not up to date with ClickHouse parser.
words = {"CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT",
"MATERIALIZED", "ALIAS", "ENGINE", "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP",
"RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", "COPY", "PROJECT",
"PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO",
"OUTFILE", "FORMAT", "TABLES", "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE",
"END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", "INSERT", "VALUES",
"SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER",
"LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY",
"WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC",
"IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE"};
}
void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit)
{
std::stringstream query;
query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM ("
"SELECT name FROM system.functions"
" UNION ALL "
"SELECT name FROM system.table_engines"
" UNION ALL "
"SELECT name FROM system.formats"
" UNION ALL "
"SELECT name FROM system.table_functions"
" UNION ALL "
"SELECT name FROM system.data_type_families"
" UNION ALL "
"SELECT name FROM system.settings"
" UNION ALL "
"SELECT cluster FROM system.clusters"
" UNION ALL "
"SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate";
/// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero.
if (suggestion_limit > 0)
{
String limit_str = toString(suggestion_limit);
query <<
" UNION ALL "
"SELECT name FROM system.databases LIMIT " << limit_str
<< " UNION ALL "
"SELECT DISTINCT name FROM system.tables LIMIT " << limit_str
<< " UNION ALL "
"SELECT DISTINCT name FROM system.columns LIMIT " << limit_str;
}
query << ") WHERE notEmpty(res)";
fetch(connection, timeouts, query.str());
}
void Suggest::fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query)
{
connection.sendQuery(timeouts, query);
while (true)
{
Packet packet = connection.receivePacket();
switch (packet.type)
{
case Protocol::Server::Data:
fillWordsFromBlock(packet.block);
continue;
case Protocol::Server::Progress:
continue;
case Protocol::Server::ProfileInfo:
continue;
case Protocol::Server::Totals:
continue;
case Protocol::Server::Extremes:
continue;
case Protocol::Server::Log:
continue;
case Protocol::Server::Exception:
packet.exception->rethrow();
return;
case Protocol::Server::EndOfStream:
return;
default:
throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
}
}
}
void Suggest::fillWordsFromBlock(const Block & block)
{
if (!block)
return;
if (block.columns() != 1)
throw Exception("Wrong number of columns received for query to read words for suggestion", ErrorCodes::LOGICAL_ERROR);
const ColumnString & column = typeid_cast<const ColumnString &>(*block.getByPosition(0).column);
size_t rows = block.rows();
for (size_t i = 0; i < rows; ++i)
words.emplace_back(column.getDataAt(i).toString());
}
}

207
dbms/programs/client/Suggest.h
View File

@ -2,18 +2,9 @@
#include "ConnectionParameters.h" #include "ConnectionParameters.h"
#include <string>
#include <sstream>
#include <string.h>
#include <vector>
#include <algorithm>
#include <common/readline_use.h>
#include <Common/typeid_cast.h>
#include <Columns/ColumnString.h>
#include <Client/Connection.h> #include <Client/Connection.h>
#include <IO/ConnectionTimeouts.h> #include <IO/ConnectionTimeouts.h>
#include <common/LineReader.h>
namespace DB namespace DB
@ -24,141 +15,8 @@ namespace ErrorCodes
extern const int UNKNOWN_PACKET_FROM_SERVER; extern const int UNKNOWN_PACKET_FROM_SERVER;
} }
class Suggest : private boost::noncopyable class Suggest : public LineReader::Suggest, boost::noncopyable
{ {
private:
/// The vector will be filled with completion words from the server and sorted.
using Words = std::vector<std::string>;
/// Keywords may be not up to date with ClickHouse parser.
Words words
{
"CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", "MATERIALIZED", "ALIAS", "ENGINE",
"AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER",
"COPY", "PROJECT", "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", "OUTFILE", "FORMAT", "TABLES",
"DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE",
"INSERT", "VALUES", "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", "LEFT", "RIGHT", "FULL", "OUTER",
"CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", "IN",
"KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE"
};
/// Words are fetched asynchronously.
std::thread loading_thread;
std::atomic<bool> ready{false};
/// Points to current word to suggest.
Words::const_iterator pos;
/// Points after the last possible match.
Words::const_iterator end;
/// Set iterators to the matched range of words if any.
void findRange(const char * prefix, size_t prefix_length)
{
std::string prefix_str(prefix);
std::tie(pos, end) = std::equal_range(words.begin(), words.end(), prefix_str,
[prefix_length](const std::string & s, const std::string & prefix_searched) { return strncmp(s.c_str(), prefix_searched.c_str(), prefix_length) < 0; });
}
/// Iterates through matched range.
char * nextMatch()
{
if (pos >= end)
return nullptr;
/// readline will free memory by itself.
char * word = strdup(pos->c_str());
++pos;
return word;
}
void loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit)
{
std::stringstream query;
query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM ("
"SELECT name FROM system.functions"
" UNION ALL "
"SELECT name FROM system.table_engines"
" UNION ALL "
"SELECT name FROM system.formats"
" UNION ALL "
"SELECT name FROM system.table_functions"
" UNION ALL "
"SELECT name FROM system.data_type_families"
" UNION ALL "
"SELECT name FROM system.settings"
" UNION ALL "
"SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate";
/// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero.
if (suggestion_limit > 0)
{
String limit_str = toString(suggestion_limit);
query <<
" UNION ALL "
"SELECT name FROM system.databases LIMIT " << limit_str
<< " UNION ALL "
"SELECT DISTINCT name FROM system.tables LIMIT " << limit_str
<< " UNION ALL "
"SELECT DISTINCT name FROM system.columns LIMIT " << limit_str;
}
query << ") WHERE notEmpty(res)";
fetch(connection, timeouts, query.str());
}
void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query)
{
connection.sendQuery(timeouts, query);
while (true)
{
Packet packet = connection.receivePacket();
switch (packet.type)
{
case Protocol::Server::Data:
fillWordsFromBlock(packet.block);
continue;
case Protocol::Server::Progress:
continue;
case Protocol::Server::ProfileInfo:
continue;
case Protocol::Server::Totals:
continue;
case Protocol::Server::Extremes:
continue;
case Protocol::Server::Log:
continue;
case Protocol::Server::Exception:
packet.exception->rethrow();
return;
case Protocol::Server::EndOfStream:
return;
default:
throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
}
}
}
void fillWordsFromBlock(const Block & block)
{
if (!block)
return;
if (block.columns() != 1)
throw Exception("Wrong number of columns received for query to read words for suggestion", ErrorCodes::LOGICAL_ERROR);
const ColumnString & column = typeid_cast<const ColumnString &>(*block.getByPosition(0).column);
size_t rows = block.rows();
for (size_t i = 0; i < rows; ++i)
words.emplace_back(column.getDataAt(i).toString());
}
public: public:
static Suggest & instance() static Suggest & instance()
{ {
@ -166,64 +24,25 @@ public:
return instance; return instance;
} }
/// More old server versions cannot execute the query above. void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit);
/// Older server versions cannot execute the query above.
static constexpr int MIN_SERVER_REVISION = 54406; static constexpr int MIN_SERVER_REVISION = 54406;
void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit) private:
{ Suggest();
loading_thread = std::thread([connection_parameters, suggestion_limit, this] ~Suggest()
{
try
{
Connection connection(
connection_parameters.host,
connection_parameters.port,
connection_parameters.default_database,
connection_parameters.user,
connection_parameters.password,
"client",
connection_parameters.compression,
connection_parameters.security);
loadImpl(connection, connection_parameters.timeouts, suggestion_limit);
}
catch (...)
{
std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n";
}
/// Note that keyword suggestions are available even if we cannot load data from server.
std::sort(words.begin(), words.end());
ready = true;
});
}
void finalize()
{ {
if (loading_thread.joinable()) if (loading_thread.joinable())
loading_thread.join(); loading_thread.join();
} }
/// A function for readline. void loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit);
static char * generator(const char * text, int state) void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query);
{ void fillWordsFromBlock(const Block & block);
Suggest & suggest = Suggest::instance();
if (!suggest.ready)
return nullptr;
if (state == 0)
suggest.findRange(text, strlen(text));
/// Do not append whitespace after word. For unknown reason, rl_completion_append_character = '\0' does not work. /// Words are fetched asynchronously.
rl_completion_suppress_append = 1; std::thread loading_thread;
return suggest.nextMatch();
}
~Suggest()
{
finalize();
}
}; };
} }

4
dbms/programs/client/TestHint.h
View File

@ -13,7 +13,7 @@ namespace DB
namespace ErrorCodes namespace ErrorCodes
{ {
extern const int LOGICAL_ERROR; extern const int UNEXPECTED_ERROR_CODE;
} }
@ -74,7 +74,7 @@ public:
std::cerr << "Success when error expected. It expects server error " std::cerr << "Success when error expected. It expects server error "
<< server_error << ", client error " << client_error << "." << std::endl; << server_error << ", client error " << client_error << "." << std::endl;
got_exception = true; got_exception = true;
last_exception = std::make_unique<Exception>("Success when error expected", ErrorCodes::LOGICAL_ERROR); /// return error to OS last_exception = std::make_unique<Exception>("Success when error expected", ErrorCodes::UNEXPECTED_ERROR_CODE); /// return error to OS
return false; return false;
} }

2
dbms/programs/client/readpassphrase/CMakeLists.txt
View File

@ -10,4 +10,4 @@ set_target_properties(readpassphrase
PROPERTIES LINKER_LANGUAGE C PROPERTIES LINKER_LANGUAGE C
) )
# . to allow #include <readpassphrase.h> # . to allow #include <readpassphrase.h>
target_include_directories(readpassphrase PUBLIC . ${CMAKE_CURRENT_BINARY_DIR}/include ${CMAKE_CURRENT_BINARY_DIR}/../include) target_include_directories(readpassphrase PUBLIC . ${CMAKE_CURRENT_BINARY_DIR}/include)

2
dbms/programs/compressor/Compressor.cpp
View File

@ -70,7 +70,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
("hc", "use LZ4HC instead of LZ4") ("hc", "use LZ4HC instead of LZ4")
("zstd", "use ZSTD instead of LZ4") ("zstd", "use ZSTD instead of LZ4")
("codec", boost::program_options::value<std::vector<std::string>>()->multitoken(), "use codecs combination instead of LZ4") ("codec", boost::program_options::value<std::vector<std::string>>()->multitoken(), "use codecs combination instead of LZ4")
("level", boost::program_options::value<int>(), "compression level for codecs spicified via flags") ("level", boost::program_options::value<int>(), "compression level for codecs specified via flags")
("none", "use no compression instead of LZ4") ("none", "use no compression instead of LZ4")
("stat", "print block statistics of compressed data") ("stat", "print block statistics of compressed data")
; ;

2
dbms/programs/copier/ClusterCopier.cpp
View File

@ -2430,7 +2430,7 @@ void ClusterCopierApp::defineOptions(Poco::Util::OptionSet & options)
.argument("copy-fault-probability").binding("copy-fault-probability")); .argument("copy-fault-probability").binding("copy-fault-probability"));
options.addOption(Poco::Util::Option("log-level", "", "sets log level") options.addOption(Poco::Util::Option("log-level", "", "sets log level")
.argument("log-level").binding("log-level")); .argument("log-level").binding("log-level"));
options.addOption(Poco::Util::Option("base-dir", "", "base directory for copiers, consequitive copier launches will populate /base-dir/launch_id/* directories") options.addOption(Poco::Util::Option("base-dir", "", "base directory for copiers, consecutive copier launches will populate /base-dir/launch_id/* directories")
.argument("base-dir").binding("base-dir")); .argument("base-dir").binding("base-dir"));
using Me = std::decay_t<decltype(*this)>; using Me = std::decay_t<decltype(*this)>;

4
dbms/programs/local/LocalServer.cpp
View File

@ -164,7 +164,7 @@ try
setupUsers(); setupUsers();
/// Limit on total number of concurrently executing queries. /// Limit on total number of concurrently executing queries.
/// Threre are no need for concurrent threads, override max_concurrent_queries. /// There is no need for concurrent threads, override max_concurrent_queries.
context->getProcessList().setMaxSize(0); context->getProcessList().setMaxSize(0);
/// Size of cache for uncompressed blocks. Zero means disabled. /// Size of cache for uncompressed blocks. Zero means disabled.
@ -182,7 +182,7 @@ try
context->setDefaultProfiles(config()); context->setDefaultProfiles(config());
/** Init dummy default DB /** Init dummy default DB
* NOTE: We force using isolated default database to avoid conflicts with default database from server enviroment * NOTE: We force using isolated default database to avoid conflicts with default database from server environment
* Otherwise, metadata of temporary File(format, EXPLICIT_PATH) tables will pollute metadata/ directory; * Otherwise, metadata of temporary File(format, EXPLICIT_PATH) tables will pollute metadata/ directory;
* if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons. * if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons.
*/ */

4
dbms/programs/obfuscator/Obfuscator.cpp
View File

@ -40,7 +40,7 @@
#include <Common/TerminalSize.h> #include <Common/TerminalSize.h>
static const char * documantation = R"( static const char * documentation = R"(
Simple tool for table data obfuscation. Simple tool for table data obfuscation.
It reads input table and produces output table, that retain some properties of input, but contains different data. It reads input table and produces output table, that retain some properties of input, but contains different data.
@ -979,7 +979,7 @@ try
|| !options.count("input-format") || !options.count("input-format")
|| !options.count("output-format")) || !options.count("output-format"))
{ {
std::cout << documantation << "\n" std::cout << documentation << "\n"
<< "\nUsage: " << argv[0] << " [options] < in > out\n" << "\nUsage: " << argv[0] << " [options] < in > out\n"
<< "\nInput must be seekable file (it will be read twice).\n" << "\nInput must be seekable file (it will be read twice).\n"
<< "\n" << description << "\n" << "\n" << description << "\n"

12
dbms/programs/odbc-bridge/CMakeLists.txt
View File

@ -15,20 +15,24 @@ set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemo
if (USE_POCO_SQLODBC) if (USE_POCO_SQLODBC)
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQLODBC_LIBRARY}) set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQLODBC_LIBRARY})
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQLODBC_INCLUDE_DIR}) # Wouldnt work anyway because of the way list variable got expanded in `target_include_directories`
# set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQLODBC_INCLUDE_DIR})
endif () endif ()
if (Poco_SQL_FOUND) if (Poco_SQL_FOUND)
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQL_LIBRARY}) set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQL_LIBRARY})
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR}) # Wouldnt work anyway because of the way list variable got expanded in `target_include_directories`
# set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR})
endif () endif ()
if (USE_POCO_DATAODBC) if (USE_POCO_DATAODBC)
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_DataODBC_LIBRARY}) set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_DataODBC_LIBRARY})
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_DataODBC_INCLUDE_DIR}) # Wouldnt work anyway because of the way list variable got expanded in `target_include_directories`
# set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_DataODBC_INCLUDE_DIR})
endif() endif()
if (Poco_Data_FOUND) if (Poco_Data_FOUND)
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_Data_LIBRARY}) set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_Data_LIBRARY})
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR}) # Wouldnt work anyway because of the way list variable got expanded in `target_include_directories`
# set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR})
endif () endif ()
clickhouse_program_add_library(odbc-bridge) clickhouse_program_add_library(odbc-bridge)

2
dbms/programs/odbc-bridge/MainHandler.cpp
View File

@ -138,7 +138,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne
{ {
auto message = getCurrentExceptionMessage(true); auto message = getCurrentExceptionMessage(true);
response.setStatusAndReason( response.setStatusAndReason(
Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); // can't call process_error, bacause of too soon response sending Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); // can't call process_error, because of too soon response sending
writeStringBinary(message, out); writeStringBinary(message, out);
tryLogCurrentException(log); tryLogCurrentException(log);
} }

2
dbms/programs/odbc-bridge/ODBCBridge.cpp
View File

@ -88,7 +88,7 @@ void ODBCBridge::defineOptions(Poco::Util::OptionSet & options)
options.addOption( options.addOption(
Poco::Util::Option("listen-host", "", "hostname to listen, default localhost").argument("listen-host").binding("listen-host")); Poco::Util::Option("listen-host", "", "hostname to listen, default localhost").argument("listen-host").binding("listen-host"));
options.addOption( options.addOption(
Poco::Util::Option("http-timeout", "", "http timout for socket, default 1800").argument("http-timeout").binding("http-timeout")); Poco::Util::Option("http-timeout", "", "http timeout for socket, default 1800").argument("http-timeout").binding("http-timeout"));
options.addOption(Poco::Util::Option("max-server-connections", "", "max connections to server, default 1024") options.addOption(Poco::Util::Option("max-server-connections", "", "max connections to server, default 1024")
.argument("max-server-connections") .argument("max-server-connections")

2
dbms/programs/performance-test/PerformanceTest.cpp
View File

@ -315,7 +315,7 @@ void PerformanceTest::runQueries(
stop_conditions.reportIterations(iteration); stop_conditions.reportIterations(iteration);
if (stop_conditions.areFulfilled()) if (stop_conditions.areFulfilled())
{ {
LOG_INFO(log, "Stop conditions fullfilled"); LOG_INFO(log, "Stop conditions fulfilled");
break; break;
} }

8
dbms/programs/performance-test/PerformanceTestSuite.cpp
View File

@ -200,7 +200,7 @@ private:
if (current.checkPreconditions()) if (current.checkPreconditions())
{ {
LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are fullfilled"); LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are fulfilled");
LOG_INFO( LOG_INFO(
log, log,
"Preparing for run, have " << info.create_and_fill_queries.size() << " create and fill queries"); "Preparing for run, have " << info.create_and_fill_queries.size() << " create and fill queries");
@ -219,7 +219,7 @@ private:
return {report_builder->buildFullReport(info, result, query_indexes[info.path]), current.checkSIGINT()}; return {report_builder->buildFullReport(info, result, query_indexes[info.path]), current.checkSIGINT()};
} }
else else
LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are not fullfilled, skip run"); LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are not fulfilled, skip run");
return {"", current.checkSIGINT()}; return {"", current.checkSIGINT()};
} }
@ -361,8 +361,8 @@ try
po::notify(options); po::notify(options);
Poco::AutoPtr<Poco::PatternFormatter> formatter(new Poco::PatternFormatter("%Y.%m.%d %H:%M:%S.%F <%p> %s: %t")); Poco::AutoPtr<Poco::PatternFormatter> formatter(new Poco::PatternFormatter("%Y.%m.%d %H:%M:%S.%F <%p> %s: %t"));
Poco::AutoPtr<Poco::ConsoleChannel> console_chanel(new Poco::ConsoleChannel); Poco::AutoPtr<Poco::ConsoleChannel> console_channel(new Poco::ConsoleChannel);
Poco::AutoPtr<Poco::FormattingChannel> channel(new Poco::FormattingChannel(formatter, console_chanel)); Poco::AutoPtr<Poco::FormattingChannel> channel(new Poco::FormattingChannel(formatter, console_channel));
Poco::Logger::root().setLevel(options["log-level"].as<std::string>()); Poco::Logger::root().setLevel(options["log-level"].as<std::string>());
Poco::Logger::root().setChannel(channel); Poco::Logger::root().setChannel(channel);

2
dbms/programs/performance-test/ReportBuilder.cpp
View File

@ -117,7 +117,7 @@ std::string ReportBuilder::buildFullReport(
if (isASCIIString(statistics.exception)) if (isASCIIString(statistics.exception))
runJSON.set("exception", jsonString(statistics.exception, settings), false); runJSON.set("exception", jsonString(statistics.exception, settings), false);
else else
runJSON.set("exception", "Some exception occured with non ASCII message. This may produce invalid JSON. Try reproduce locally."); runJSON.set("exception", "Some exception occurred with non ASCII message. This may produce invalid JSON. Try reproduce locally.");
} }
if (test_info.exec_type == ExecutionType::Loop) if (test_info.exec_type == ExecutionType::Loop)

14
dbms/programs/server/InterserverIOHTTPHandler.cpp
View File

@ -28,23 +28,23 @@ std::pair<String, bool> InterserverIOHTTPHandler::checkAuthentication(Poco::Net:
if (config.has("interserver_http_credentials.user")) if (config.has("interserver_http_credentials.user"))
{ {
if (!request.hasCredentials()) if (!request.hasCredentials())
return {"Server requires HTTP Basic authentification, but client doesn't provide it", false}; return {"Server requires HTTP Basic authentication, but client doesn't provide it", false};
String scheme, info; String scheme, info;
request.getCredentials(scheme, info); request.getCredentials(scheme, info);
if (scheme != "Basic") if (scheme != "Basic")
return {"Server requires HTTP Basic authentification but client provides another method", false}; return {"Server requires HTTP Basic authentication but client provides another method", false};
String user = config.getString("interserver_http_credentials.user"); String user = config.getString("interserver_http_credentials.user");
String password = config.getString("interserver_http_credentials.password", ""); String password = config.getString("interserver_http_credentials.password", "");
Poco::Net::HTTPBasicCredentials credentials(info); Poco::Net::HTTPBasicCredentials credentials(info);
if (std::make_pair(user, password) != std::make_pair(credentials.getUsername(), credentials.getPassword())) if (std::make_pair(user, password) != std::make_pair(credentials.getUsername(), credentials.getPassword()))
return {"Incorrect user or password in HTTP Basic authentification", false}; return {"Incorrect user or password in HTTP Basic authentication", false};
} }
else if (request.hasCredentials()) else if (request.hasCredentials())
{ {
return {"Client requires HTTP Basic authentification, but server doesn't provide it", false}; return {"Client requires HTTP Basic authentication, but server doesn't provide it", false};
} }
return {"", true}; return {"", true};
} }
@ -61,6 +61,10 @@ void InterserverIOHTTPHandler::processQuery(Poco::Net::HTTPServerRequest & reque
ReadBufferFromIStream body(request.stream()); ReadBufferFromIStream body(request.stream());
auto endpoint = server.context().getInterserverIOHandler().getEndpoint(endpoint_name); auto endpoint = server.context().getInterserverIOHandler().getEndpoint(endpoint_name);
/// Locked for read while query processing
std::shared_lock lock(endpoint->rwlock);
if (endpoint->blocker.isCancelled())
throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED);
if (compress) if (compress)
{ {
@ -99,7 +103,7 @@ void InterserverIOHTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & requ
response.setStatusAndReason(Poco::Net::HTTPServerResponse::HTTP_UNAUTHORIZED); response.setStatusAndReason(Poco::Net::HTTPServerResponse::HTTP_UNAUTHORIZED);
if (!response.sent()) if (!response.sent())
writeString(message, *used_output.out); writeString(message, *used_output.out);
LOG_WARNING(log, "Query processing failed request: '" << request.getURI() << "' authentification failed"); LOG_WARNING(log, "Query processing failed request: '" << request.getURI() << "' authentication failed");
} }
} }
catch (Exception & e) catch (Exception & e)

20
dbms/programs/server/MySQLHandler.cpp
View File

@ -79,21 +79,19 @@ void MySQLHandler::run()
if (!connection_context.mysql.max_packet_size) if (!connection_context.mysql.max_packet_size)
connection_context.mysql.max_packet_size = MAX_PACKET_LENGTH; connection_context.mysql.max_packet_size = MAX_PACKET_LENGTH;
/* LOG_TRACE(log, "Capabilities: " << handshake_response.capability_flags LOG_TRACE(log, "Capabilities: " << handshake_response.capability_flags
<< "\nmax_packet_size: " << ", max_packet_size: "
<< handshake_response.max_packet_size << handshake_response.max_packet_size
<< "\ncharacter_set: " << ", character_set: "
<< handshake_response.character_set << static_cast<int>(handshake_response.character_set)
<< "\nuser: " << ", user: "
<< handshake_response.username << handshake_response.username
<< "\nauth_response length: " << ", auth_response length: "
<< handshake_response.auth_response.length() << handshake_response.auth_response.length()
<< "\nauth_response: " << ", database: "
<< handshake_response.auth_response
<< "\ndatabase: "
<< handshake_response.database << handshake_response.database
<< "\nauth_plugin_name: " << ", auth_plugin_name: "
<< handshake_response.auth_plugin_name);*/ << handshake_response.auth_plugin_name);
client_capability_flags = handshake_response.capability_flags; client_capability_flags = handshake_response.capability_flags;
if (!(client_capability_flags & CLIENT_PROTOCOL_41)) if (!(client_capability_flags & CLIENT_PROTOCOL_41))

6
dbms/programs/server/MySQLHandlerFactory.cpp
View File

@ -34,7 +34,7 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_)
} }
catch (...) catch (...)
{ {
LOG_INFO(log, "Failed to create SSL context. SSL will be disabled. Error: " << getCurrentExceptionMessage(false)); LOG_TRACE(log, "Failed to create SSL context. SSL will be disabled. Error: " << getCurrentExceptionMessage(false));
ssl_enabled = false; ssl_enabled = false;
} }
#endif #endif
@ -47,7 +47,7 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_)
} }
catch (...) catch (...)
{ {
LOG_WARNING(log, "Failed to read RSA keys. Error: " << getCurrentExceptionMessage(false)); LOG_TRACE(log, "Failed to read RSA key pair from server certificate. Error: " << getCurrentExceptionMessage(false));
generateRSAKeys(); generateRSAKeys();
} }
#endif #endif
@ -104,7 +104,7 @@ void MySQLHandlerFactory::readRSAKeys()
void MySQLHandlerFactory::generateRSAKeys() void MySQLHandlerFactory::generateRSAKeys()
{ {
LOG_INFO(log, "Generating new RSA key."); LOG_TRACE(log, "Generating new RSA key pair.");
public_key.reset(RSA_new()); public_key.reset(RSA_new());
if (!public_key) if (!public_key)
throw Exception("Failed to allocate RSA key. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR); throw Exception("Failed to allocate RSA key. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);

6
dbms/programs/server/PrometheusRequestHandler.h
View File

@ -31,7 +31,7 @@ public:
template <typename HandlerType> template <typename HandlerType>
class PrometeusRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory class PrometheusRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory
{ {
private: private:
IServer & server; IServer & server;
@ -39,7 +39,7 @@ private:
PrometheusMetricsWriter metrics_writer; PrometheusMetricsWriter metrics_writer;
public: public:
PrometeusRequestHandlerFactory(IServer & server_, const AsynchronousMetrics & async_metrics_) PrometheusRequestHandlerFactory(IServer & server_, const AsynchronousMetrics & async_metrics_)
: server(server_) : server(server_)
, endpoint_path(server_.config().getString("prometheus.endpoint", "/metrics")) , endpoint_path(server_.config().getString("prometheus.endpoint", "/metrics"))
, metrics_writer(server_.config(), "prometheus", async_metrics_) , metrics_writer(server_.config(), "prometheus", async_metrics_)
@ -56,6 +56,6 @@ public:
} }
}; };
using PrometeusHandlerFactory = PrometeusRequestHandlerFactory<PrometheusRequestHandler>; using PrometheusHandlerFactory = PrometheusRequestHandlerFactory<PrometheusRequestHandler>;
} }

26
dbms/programs/server/Server.cpp
View File

@ -277,15 +277,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
Poco::File(path + "data/" + default_database).createDirectories(); Poco::File(path + "data/" + default_database).createDirectories();
Poco::File(path + "metadata/" + default_database).createDirectories(); Poco::File(path + "metadata/" + default_database).createDirectories();
/// Check that we have read and write access to all data paths
auto disk_selector = global_context->getDiskSelector();
for (const auto & [name, disk] : disk_selector.getDisksMap())
{
Poco::File disk_path(disk->getPath());
if (!disk_path.canRead() || !disk_path.canWrite())
throw Exception("There is no RW access to disk " + name + " (" + disk->getPath() + ")", ErrorCodes::PATH_ACCESS_DENIED);
}
StatusFile status{path + "status"}; StatusFile status{path + "status"};
SCOPE_EXIT({ SCOPE_EXIT({
@ -445,8 +436,10 @@ int Server::main(const std::vector<std::string> & /*args*/)
main_config_zk_changed_event, main_config_zk_changed_event,
[&](ConfigurationPtr config) [&](ConfigurationPtr config)
{ {
setTextLog(global_context->getTextLog()); // FIXME logging-related things need synchronization -- see the 'Logger * log' saved
buildLoggers(*config, logger()); // in a lot of places. For now, disable updating log configuration without server restart.
//setTextLog(global_context->getTextLog());
//buildLoggers(*config, logger());
global_context->setClustersConfig(config); global_context->setClustersConfig(config);
global_context->setMacros(std::make_unique<Macros>(*config, "macros")); global_context->setMacros(std::make_unique<Macros>(*config, "macros"));
@ -554,8 +547,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
/// ///
/// It also cannot work with sanitizers. /// It also cannot work with sanitizers.
/// Sanitizers are using quick "frame walking" stack unwinding (this implies -fno-omit-frame-pointer) /// Sanitizers are using quick "frame walking" stack unwinding (this implies -fno-omit-frame-pointer)
/// And they do unwiding frequently (on every malloc/free, thread/mutex operations, etc). /// And they do unwinding frequently (on every malloc/free, thread/mutex operations, etc).
/// They change %rbp during unwinding and it confuses libunwind if signal comes during sanitizer unwiding /// They change %rbp during unwinding and it confuses libunwind if signal comes during sanitizer unwinding
/// and query profiler decide to unwind stack with libunwind at this moment. /// and query profiler decide to unwind stack with libunwind at this moment.
/// ///
/// Symptoms: you'll get silent Segmentation Fault - without sanitizer message and without usual ClickHouse diagnostics. /// Symptoms: you'll get silent Segmentation Fault - without sanitizer message and without usual ClickHouse diagnostics.
@ -724,7 +717,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
socket.setSendTimeout(settings.http_send_timeout); socket.setSendTimeout(settings.http_send_timeout);
auto handler_factory = createDefaultHandlerFatory<HTTPHandler>(*this, "HTTPHandler-factory"); auto handler_factory = createDefaultHandlerFatory<HTTPHandler>(*this, "HTTPHandler-factory");
if (config().has("prometheus") && config().getInt("prometheus.port", 0) == 0) if (config().has("prometheus") && config().getInt("prometheus.port", 0) == 0)
handler_factory->addHandler<PrometeusHandlerFactory>(async_metrics); handler_factory->addHandler<PrometheusHandlerFactory>(async_metrics);
servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>( servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>(
handler_factory, handler_factory,
@ -854,7 +847,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
socket.setReceiveTimeout(settings.http_receive_timeout); socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout); socket.setSendTimeout(settings.http_send_timeout);
auto handler_factory = new HTTPRequestHandlerFactoryMain(*this, "PrometheusHandler-factory"); auto handler_factory = new HTTPRequestHandlerFactoryMain(*this, "PrometheusHandler-factory");
handler_factory->addHandler<PrometeusHandlerFactory>(async_metrics); handler_factory->addHandler<PrometheusHandlerFactory>(async_metrics);
servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>( servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>(
handler_factory, handler_factory,
server_pool, server_pool,
@ -871,6 +864,9 @@ int Server::main(const std::vector<std::string> & /*args*/)
for (auto & server : servers) for (auto & server : servers)
server->start(); server->start();
setTextLog(global_context->getTextLog());
buildLoggers(config(), logger());
main_config_reloader->start(); main_config_reloader->start();
users_config_reloader->start(); users_config_reloader->start();
if (dns_cache_updater) if (dns_cache_updater)

4
dbms/programs/server/TCPHandler.cpp
View File

@ -606,7 +606,7 @@ void TCPHandler::processOrdinaryQueryWithProcessors(size_t num_threads)
/// If exception was thrown during pipeline execution, skip it while processing other exception. /// If exception was thrown during pipeline execution, skip it while processing other exception.
} }
pipeline = QueryPipeline() /// pipeline = QueryPipeline()
); );
while (true) while (true)
@ -993,7 +993,7 @@ bool TCPHandler::receiveData(bool scalar)
if (!(storage = query_context->tryGetExternalTable(name))) if (!(storage = query_context->tryGetExternalTable(name)))
{ {
NamesAndTypesList columns = block.getNamesAndTypesList(); NamesAndTypesList columns = block.getNamesAndTypesList();
storage = StorageMemory::create("_external", name, ColumnsDescription{columns}, ConstraintsDescription{}); storage = StorageMemory::create(StorageID("_external", name), ColumnsDescription{columns}, ConstraintsDescription{});
storage->startup(); storage->startup();
query_context->addExternalTable(name, storage); query_context->addExternalTable(name, storage);
} }

2
dbms/programs/server/TCPHandler.h
View File

@ -111,7 +111,7 @@ public:
server_display_name = server.config().getString("display_name", getFQDNOrHostName()); server_display_name = server.config().getString("display_name", getFQDNOrHostName());
} }
void run(); void run() override;
/// This method is called right before the query execution. /// This method is called right before the query execution.
virtual void customizeContext(DB::Context & /*context*/) {} virtual void customizeContext(DB::Context & /*context*/) {}

7
dbms/programs/server/config.d/part_log.xml Normal file
View File

@ -0,0 +1,7 @@
<yandex>
<part_log>
<database>system</database>
<table>part_log</table>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</part_log>
</yandex>

1
dbms/programs/server/config.xml
View File

@ -34,6 +34,7 @@
<!--display_name>production</display_name--> <!-- It is the name that will be shown in the client --> <!--display_name>production</display_name--> <!-- It is the name that will be shown in the client -->
<http_port>8123</http_port> <http_port>8123</http_port>
<tcp_port>9000</tcp_port> <tcp_port>9000</tcp_port>
<mysql_port>9004</mysql_port>
<!-- For HTTPS and SSL over native protocol. --> <!-- For HTTPS and SSL over native protocol. -->
<!-- <!--
<https_port>8443</https_port> <https_port>8443</https_port>

25
dbms/programs/server/users.xml
View File

@ -49,7 +49,7 @@
In first line will be password and in second - corresponding SHA256. In first line will be password and in second - corresponding SHA256.
How to generate double SHA1: How to generate double SHA1:
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | openssl dgst -sha1 -binary | openssl dgst -sha1 Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-'
In first line will be password and in second - corresponding double SHA1. In first line will be password and in second - corresponding double SHA1.
--> -->
<password></password> <password></password>
@ -83,30 +83,7 @@
<!-- Quota for user. --> <!-- Quota for user. -->
<quota>default</quota> <quota>default</quota>
<!-- Example of row level security policy. -->
<!-- <databases>
<test>
<filtered_table1>
<filter>a = 1</filter>
</filtered_table1>
<filtered_table2>
<filter>a + b &lt; 1 or c - d &gt; 5</filter>
</filtered_table2>
</test>
</databases> -->
</default> </default>
<!-- Example of user with readonly access. -->
<!-- <readonly>
<password></password>
<networks incl="networks" replace="replace">
<ip>::1</ip>
<ip>127.0.0.1</ip>
</networks>
<profile>readonly</profile>
<quota>default</quota>
</readonly> -->
</users> </users>
<!-- Quotas. --> <!-- Quotas. -->

20
dbms/src/Access/UsersConfigAccessStorage.cpp
View File

@ -135,13 +135,25 @@ namespace
for (const String & database : databases) for (const String & database : databases)
{ {
const String database_config = databases_config + "." + database; const String database_config = databases_config + "." + database;
Poco::Util::AbstractConfiguration::Keys table_names; Poco::Util::AbstractConfiguration::Keys keys_in_database_config;
config.keys(database_config, table_names); config.keys(database_config, keys_in_database_config);
/// Read table properties /// Read table properties
for (const String & table_name : table_names) for (const String & key_in_database_config : keys_in_database_config)
{ {
const auto filter_config = database_config + "." + table_name + ".filter"; String table_name = key_in_database_config;
String filter_config = database_config + "." + table_name + ".filter";
if (key_in_database_config.starts_with("table["))
{
const auto table_name_config = database_config + "." + table_name + "[@name]";
if (config.has(table_name_config))
{
table_name = config.getString(table_name_config);
filter_config = database_config + ".table[@name='" + table_name + "']";
}
}
if (config.has(filter_config)) if (config.has(filter_config))
{ {
try try

2
dbms/src/AggregateFunctions/AggregateFunctionForEach.h
View File

@ -54,7 +54,7 @@ private:
{ {
AggregateFunctionForEachData & state = data(place); AggregateFunctionForEachData & state = data(place);
/// Ensure we have aggreate states for new_size elements, allocate /// Ensure we have aggregate states for new_size elements, allocate
/// from arena if needed. When reallocating, we can't copy the /// from arena if needed. When reallocating, we can't copy the
/// states to new buffer with memcpy, because they may contain pointers /// states to new buffer with memcpy, because they may contain pointers
/// to themselves. In particular, this happens when a state contains /// to themselves. In particular, this happens when a state contains

48
dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp
View File

@ -29,17 +29,23 @@ static IAggregateFunction * createWithNumericOrTimeType(const IDataType & argume
} }
template <typename has_limit, typename ... TArgs> template <typename Trait, typename ... TArgs>
inline AggregateFunctionPtr createAggregateFunctionGroupArrayImpl(const DataTypePtr & argument_type, TArgs ... args) inline AggregateFunctionPtr createAggregateFunctionGroupArrayImpl(const DataTypePtr & argument_type, TArgs ... args)
{ {
if (auto res = createWithNumericOrTimeType<GroupArrayNumericImpl, has_limit>(*argument_type, argument_type, std::forward<TArgs>(args)...)) if (auto res = createWithNumericOrTimeType<GroupArrayNumericImpl, Trait>(*argument_type, argument_type, std::forward<TArgs>(args)...))
return AggregateFunctionPtr(res); return AggregateFunctionPtr(res);
WhichDataType which(argument_type); WhichDataType which(argument_type);
if (which.idx == TypeIndex::String) if (which.idx == TypeIndex::String)
return std::make_shared<GroupArrayGeneralListImpl<GroupArrayListNodeString, has_limit::value>>(argument_type, std::forward<TArgs>(args)...); return std::make_shared<GroupArrayGeneralImpl<GroupArrayNodeString, Trait>>(argument_type, std::forward<TArgs>(args)...);
return std::make_shared<GroupArrayGeneralListImpl<GroupArrayListNodeGeneral, has_limit::value>>(argument_type, std::forward<TArgs>(args)...); return std::make_shared<GroupArrayGeneralImpl<GroupArrayNodeGeneral, Trait>>(argument_type, std::forward<TArgs>(args)...);
// Link list implementation doesn't show noticeable performance improvement
// if (which.idx == TypeIndex::String)
// return std::make_shared<GroupArrayGeneralListImpl<GroupArrayListNodeString, Trait>>(argument_type, std::forward<TArgs>(args)...);
// return std::make_shared<GroupArrayGeneralListImpl<GroupArrayListNodeGeneral, Trait>>(argument_type, std::forward<TArgs>(args)...);
} }
@ -72,9 +78,38 @@ static AggregateFunctionPtr createAggregateFunctionGroupArray(const std::string
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
if (!limit_size) if (!limit_size)
return createAggregateFunctionGroupArrayImpl<std::false_type>(argument_types[0]); return createAggregateFunctionGroupArrayImpl<GroupArrayTrait<false, Sampler::NONE>>(argument_types[0]);
else else
return createAggregateFunctionGroupArrayImpl<std::true_type>(argument_types[0], max_elems); return createAggregateFunctionGroupArrayImpl<GroupArrayTrait<true, Sampler::NONE>>(argument_types[0], max_elems);
}
static AggregateFunctionPtr
createAggregateFunctionGroupArraySample(const std::string & name, const DataTypes & argument_types, const Array & parameters)
{
assertUnary(name, argument_types);
UInt64 max_elems = std::numeric_limits<UInt64>::max();
UInt64 seed = 123456;
UInt64 * params[2] = {&max_elems, &seed};
if (parameters.size() != 1 && parameters.size() != 2)
throw Exception("Incorrect number of parameters for aggregate function " + name + ", should be 1 or 2",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
for (auto i = 0ul; i < parameters.size(); ++i)
{
auto type = parameters[i].getType();
if (type != Field::Types::Int64 && type != Field::Types::UInt64)
throw Exception("Parameter for aggregate function " + name + " should be positive number", ErrorCodes::BAD_ARGUMENTS);
if ((type == Field::Types::Int64 && parameters[i].get<Int64>() < 0) ||
(type == Field::Types::UInt64 && parameters[i].get<UInt64>() == 0))
throw Exception("Parameter for aggregate function " + name + " should be positive number", ErrorCodes::BAD_ARGUMENTS);
*params[i] = parameters[i].get<UInt64>();
}
return createAggregateFunctionGroupArrayImpl<GroupArrayTrait<true, Sampler::RNG>>(argument_types[0], max_elems, seed);
} }
} }
@ -83,6 +118,7 @@ static AggregateFunctionPtr createAggregateFunctionGroupArray(const std::string
void registerAggregateFunctionGroupArray(AggregateFunctionFactory & factory) void registerAggregateFunctionGroupArray(AggregateFunctionFactory & factory)
{ {
factory.registerFunction("groupArray", createAggregateFunctionGroupArray); factory.registerFunction("groupArray", createAggregateFunctionGroupArray);
factory.registerFunction("groupArraySample", createAggregateFunctionGroupArraySample);
} }
} }

575
dbms/src/AggregateFunctions/AggregateFunctionGroupArray.h
View File

@ -1,40 +1,97 @@
#pragma once #pragma once
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h> #include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <DataTypes/DataTypeArray.h> #include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeString.h> #include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypesNumber.h>
#include <Columns/ColumnVector.h>
#include <Columns/ColumnArray.h> #include <Columns/ColumnArray.h>
#include <Columns/ColumnString.h> #include <Columns/ColumnString.h>
#include <Columns/ColumnVector.h>
#include <Common/ArenaAllocator.h> #include <Common/ArenaAllocator.h>
#include <Common/assert_cast.h> #include <Common/assert_cast.h>
#include <AggregateFunctions/IAggregateFunction.h> #include <AggregateFunctions/IAggregateFunction.h>
#include <common/likely.h>
#include <type_traits> #include <type_traits>
#include <common/likely.h>
#define AGGREGATE_FUNCTION_GROUP_ARRAY_MAX_ARRAY_SIZE 0xFFFFFF #define AGGREGATE_FUNCTION_GROUP_ARRAY_MAX_ARRAY_SIZE 0xFFFFFF
namespace DB namespace DB
{ {
namespace ErrorCodes namespace ErrorCodes
{ {
extern const int TOO_LARGE_ARRAY_SIZE; extern const int TOO_LARGE_ARRAY_SIZE;
extern const int LOGICAL_ERROR; extern const int LOGICAL_ERROR;
} }
enum class Sampler
{
NONE,
RNG,
DETERMINATOR // TODO
};
template <bool Thas_limit, Sampler Tsampler>
struct GroupArrayTrait
{
static constexpr bool has_limit = Thas_limit;
static constexpr Sampler sampler = Tsampler;
};
template <typename Trait>
static constexpr const char * getNameByTrait()
{
if (Trait::sampler == Sampler::NONE)
return "groupArray";
else if (Trait::sampler == Sampler::RNG)
return "groupArraySample";
// else if (Trait::sampler == Sampler::DETERMINATOR) // TODO
__builtin_unreachable();
}
template <typename T>
struct GroupArraySamplerData
{
// Switch to ordinary Allocator after 4096 bytes to avoid fragmentation and trash in Arena
using Allocator = MixedAlignedArenaAllocator<alignof(T), 4096>;
using Array = PODArray<T, 32, Allocator>;
Array value;
size_t total_values = 0;
pcg32_fast rng;
UInt64 genRandom(size_t lim)
{
/// With a large number of values, we will generate random numbers several times slower.
if (lim <= static_cast<UInt64>(rng.max()))
return static_cast<UInt32>(rng()) % static_cast<UInt32>(lim);
else
return (static_cast<UInt64>(rng()) * (static_cast<UInt64>(rng.max()) + 1ULL) + static_cast<UInt64>(rng())) % lim;
}
void randomShuffle()
{
for (size_t i = 1; i < value.size(); ++i)
{
size_t j = genRandom(i + 1);
std::swap(value[i], value[j]);
}
}
};
/// A particular case is an implementation for numeric types. /// A particular case is an implementation for numeric types.
template <typename T, bool has_sampler>
struct GroupArrayNumericData;
template <typename T> template <typename T>
struct GroupArrayNumericData struct GroupArrayNumericData<T, false>
{ {
// Switch to ordinary Allocator after 4096 bytes to avoid fragmentation and trash in Arena // Switch to ordinary Allocator after 4096 bytes to avoid fragmentation and trash in Arena
using Allocator = MixedAlignedArenaAllocator<alignof(T), 4096>; using Allocator = MixedAlignedArenaAllocator<alignof(T), 4096>;
@ -43,51 +100,140 @@ struct GroupArrayNumericData
Array value; Array value;
}; };
template <typename T>
template <typename T, typename Tlimit_num_elems> struct GroupArrayNumericData<T, true> : public GroupArraySamplerData<T>
class GroupArrayNumericImpl final
: public IAggregateFunctionDataHelper<GroupArrayNumericData<T>, GroupArrayNumericImpl<T, Tlimit_num_elems>>
{ {
static constexpr bool limit_num_elems = Tlimit_num_elems::value; };
template <typename T, typename Trait>
class GroupArrayNumericImpl final
: public IAggregateFunctionDataHelper<GroupArrayNumericData<T, Trait::sampler != Sampler::NONE>, GroupArrayNumericImpl<T, Trait>>
{
using Data = GroupArrayNumericData<T, Trait::sampler != Sampler::NONE>;
static constexpr bool limit_num_elems = Trait::has_limit;
DataTypePtr & data_type; DataTypePtr & data_type;
UInt64 max_elems; UInt64 max_elems;
UInt64 seed;
public: public:
explicit GroupArrayNumericImpl(const DataTypePtr & data_type_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max()) explicit GroupArrayNumericImpl(
: IAggregateFunctionDataHelper<GroupArrayNumericData<T>, GroupArrayNumericImpl<T, Tlimit_num_elems>>({data_type_}, {}) const DataTypePtr & data_type_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max(), UInt64 seed_ = 123456)
, data_type(this->argument_types[0]), max_elems(max_elems_) {} : IAggregateFunctionDataHelper<GroupArrayNumericData<T, Trait::sampler != Sampler::NONE>, GroupArrayNumericImpl<T, Trait>>(
{data_type_}, {})
String getName() const override { return "groupArray"; } , data_type(this->argument_types[0])
, max_elems(max_elems_)
DataTypePtr getReturnType() const override , seed(seed_)
{ {
return std::make_shared<DataTypeArray>(data_type); }
String getName() const override { return getNameByTrait<Trait>(); }
DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(data_type); }
void insert(Data & a, const T & v, Arena * arena) const
{
++a.total_values;
if (a.value.size() < max_elems)
a.value.push_back(v, arena);
else
{
UInt64 rnd = a.genRandom(a.total_values);
if (rnd < max_elems)
a.value[rnd] = v;
}
}
void create(AggregateDataPtr place) const override
{
[[maybe_unused]] auto a = new (place) Data;
if constexpr (Trait::sampler == Sampler::RNG)
a->rng.seed(seed);
} }
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena * arena) const override void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena * arena) const override
{ {
if (limit_num_elems && this->data(place).value.size() >= max_elems) if constexpr (Trait::sampler == Sampler::NONE)
return; {
if (limit_num_elems && this->data(place).value.size() >= max_elems)
return;
this->data(place).value.push_back(assert_cast<const ColumnVector<T> &>(*columns[0]).getData()[row_num], arena); this->data(place).value.push_back(assert_cast<const ColumnVector<T> &>(*columns[0]).getData()[row_num], arena);
}
if constexpr (Trait::sampler == Sampler::RNG)
{
auto & a = this->data(place);
++a.total_values;
if (a.value.size() < max_elems)
a.value.push_back(assert_cast<const ColumnVector<T> &>(*columns[0]).getData()[row_num], arena);
else
{
UInt64 rnd = a.genRandom(a.total_values);
if (rnd < max_elems)
a.value[rnd] = assert_cast<const ColumnVector<T> &>(*columns[0]).getData()[row_num];
}
}
// TODO
// if constexpr (Trait::sampler == Sampler::DETERMINATOR)
} }
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena * arena) const override void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena * arena) const override
{ {
auto & cur_elems = this->data(place); if constexpr (Trait::sampler == Sampler::NONE)
auto & rhs_elems = this->data(rhs); {
auto & cur_elems = this->data(place);
auto & rhs_elems = this->data(rhs);
if (!limit_num_elems) if (!limit_num_elems)
{ {
if (rhs_elems.value.size()) if (rhs_elems.value.size())
cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.end(), arena); cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.end(), arena);
}
else
{
UInt64 elems_to_insert = std::min(static_cast<size_t>(max_elems) - cur_elems.value.size(), rhs_elems.value.size());
if (elems_to_insert)
cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.begin() + elems_to_insert, arena);
}
} }
else
if constexpr (Trait::sampler == Sampler::RNG)
{ {
UInt64 elems_to_insert = std::min(static_cast<size_t>(max_elems) - cur_elems.value.size(), rhs_elems.value.size()); if (this->data(rhs).value.empty()) /// rhs state is empty
if (elems_to_insert) return;
cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.begin() + elems_to_insert, arena);
auto & a = this->data(place);
auto & b = this->data(rhs);
if (b.total_values <= max_elems)
{
for (size_t i = 0; i < b.value.size(); ++i)
insert(a, b.value[i], arena);
}
else if (a.total_values <= max_elems)
{
decltype(a.value) from;
from.swap(a.value, arena);
a.value.assign(b.value.begin(), b.value.end(), arena);
a.total_values = b.total_values;
for (size_t i = 0; i < from.size(); ++i)
insert(a, from[i], arena);
}
else
{
a.randomShuffle();
a.total_values += b.total_values;
for (size_t i = 0; i < max_elems; ++i)
{
UInt64 rnd = a.genRandom(a.total_values);
if (rnd < b.total_values)
a.value[i] = b.value[i];
}
}
} }
// TODO
// if constexpr (Trait::sampler == Sampler::DETERMINATOR)
} }
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
@ -96,6 +242,17 @@ public:
size_t size = value.size(); size_t size = value.size();
writeVarUInt(size, buf); writeVarUInt(size, buf);
buf.write(reinterpret_cast<const char *>(value.data()), size * sizeof(value[0])); buf.write(reinterpret_cast<const char *>(value.data()), size * sizeof(value[0]));
if constexpr (Trait::sampler == Sampler::RNG)
{
DB::writeIntBinary<size_t>(this->data(place).total_values, buf);
std::ostringstream rng_stream;
rng_stream << this->data(place).rng;
DB::writeStringBinary(rng_stream.str(), buf);
}
// TODO
// if constexpr (Trait::sampler == Sampler::DETERMINATOR)
} }
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena * arena) const override void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena * arena) const override
@ -113,6 +270,18 @@ public:
value.resize(size, arena); value.resize(size, arena);
buf.read(reinterpret_cast<char *>(value.data()), size * sizeof(value[0])); buf.read(reinterpret_cast<char *>(value.data()), size * sizeof(value[0]));
if constexpr (Trait::sampler == Sampler::RNG)
{
DB::readIntBinary<size_t>(this->data(place).total_values, buf);
std::string rng_string;
DB::readStringBinary(rng_string, buf);
std::istringstream rng_stream(rng_string);
rng_stream >> this->data(place).rng;
}
// TODO
// if constexpr (Trait::sampler == Sampler::DETERMINATOR)
} }
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
@ -132,10 +301,7 @@ public:
} }
} }
bool allocatesMemoryInArena() const override bool allocatesMemoryInArena() const override { return true; }
{
return true;
}
}; };
@ -145,26 +311,24 @@ public:
/// Nodes used to implement a linked list for storage of groupArray states /// Nodes used to implement a linked list for storage of groupArray states
template <typename Node> template <typename Node>
struct GroupArrayListNodeBase struct GroupArrayNodeBase
{ {
Node * next;
UInt64 size; // size of payload UInt64 size; // size of payload
/// Returns pointer to actual payload /// Returns pointer to actual payload
char * data() char * data() { return reinterpret_cast<char *>(this) + sizeof(Node); }
{
static_assert(sizeof(GroupArrayListNodeBase) == sizeof(Node)); const char * data() const { return reinterpret_cast<const char *>(this) + sizeof(Node); }
return reinterpret_cast<char *>(this) + sizeof(Node);
}
/// Clones existing node (does not modify next field) /// Clones existing node (does not modify next field)
Node * clone(Arena * arena) Node * clone(Arena * arena) const
{ {
return reinterpret_cast<Node *>(const_cast<char *>(arena->alignedInsert(reinterpret_cast<char *>(this), sizeof(Node) + size, alignof(Node)))); return reinterpret_cast<Node *>(
const_cast<char *>(arena->alignedInsert(reinterpret_cast<const char *>(this), sizeof(Node) + size, alignof(Node))));
} }
/// Write node to buffer /// Write node to buffer
void write(WriteBuffer & buf) void write(WriteBuffer & buf) const
{ {
writeVarUInt(size, buf); writeVarUInt(size, buf);
buf.write(data(), size); buf.write(data(), size);
@ -183,6 +347,290 @@ struct GroupArrayListNodeBase
} }
}; };
struct GroupArrayNodeString : public GroupArrayNodeBase<GroupArrayNodeString>
{
using Node = GroupArrayNodeString;
/// Create node from string
static Node * allocate(const IColumn & column, size_t row_num, Arena * arena)
{
StringRef string = assert_cast<const ColumnString &>(column).getDataAt(row_num);
Node * node = reinterpret_cast<Node *>(arena->alignedAlloc(sizeof(Node) + string.size, alignof(Node)));
node->size = string.size;
memcpy(node->data(), string.data, string.size);
return node;
}
void insertInto(IColumn & column)
{
assert_cast<ColumnString &>(column).insertData(data(), size);
}
};
struct GroupArrayNodeGeneral : public GroupArrayNodeBase<GroupArrayNodeGeneral>
{
using Node = GroupArrayNodeGeneral;
static Node * allocate(const IColumn & column, size_t row_num, Arena * arena)
{
const char * begin = arena->alignedAlloc(sizeof(Node), alignof(Node));
StringRef value = column.serializeValueIntoArena(row_num, *arena, begin);
Node * node = reinterpret_cast<Node *>(const_cast<char *>(begin));
node->size = value.size;
return node;
}
void insertInto(IColumn & column) { column.deserializeAndInsertFromArena(data()); }
};
template <typename Node, bool has_sampler>
struct GroupArrayGeneralData;
template <typename Node>
struct GroupArrayGeneralData<Node, false>
{
// Switch to ordinary Allocator after 4096 bytes to avoid fragmentation and trash in Arena
using Allocator = MixedAlignedArenaAllocator<alignof(Node *), 4096>;
using Array = PODArray<Node *, 32, Allocator>;
Array value;
};
template <typename Node>
struct GroupArrayGeneralData<Node, true> : public GroupArraySamplerData<Node *>
{
};
/// Implementation of groupArray for String or any ComplexObject via Array
template <typename Node, typename Trait>
class GroupArrayGeneralImpl final
: public IAggregateFunctionDataHelper<GroupArrayGeneralData<Node, Trait::sampler != Sampler::NONE>, GroupArrayGeneralImpl<Node, Trait>>
{
static constexpr bool limit_num_elems = Trait::has_limit;
using Data = GroupArrayGeneralData<Node, Trait::sampler != Sampler::NONE>;
static Data & data(AggregateDataPtr place) { return *reinterpret_cast<Data *>(place); }
static const Data & data(ConstAggregateDataPtr place) { return *reinterpret_cast<const Data *>(place); }
DataTypePtr & data_type;
UInt64 max_elems;
UInt64 seed;
public:
GroupArrayGeneralImpl(const DataTypePtr & data_type_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max(), UInt64 seed_ = 123456)
: IAggregateFunctionDataHelper<GroupArrayGeneralData<Node, Trait::sampler != Sampler::NONE>, GroupArrayGeneralImpl<Node, Trait>>(
{data_type_}, {})
, data_type(this->argument_types[0])
, max_elems(max_elems_)
, seed(seed_)
{
}
String getName() const override { return getNameByTrait<Trait>(); }
DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(data_type); }
void insert(Data & a, const Node * v, Arena * arena) const
{
++a.total_values;
if (a.value.size() < max_elems)
a.value.push_back(v->clone(arena), arena);
else
{
UInt64 rnd = a.genRandom(a.total_values);
if (rnd < max_elems)
a.value[rnd] = v->clone(arena);
}
}
void create(AggregateDataPtr place) const override
{
[[maybe_unused]] auto a = new (place) Data;
if constexpr (Trait::sampler == Sampler::RNG)
a->rng.seed(seed);
}
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena * arena) const override
{
if constexpr (Trait::sampler == Sampler::NONE)
{
if (limit_num_elems && data(place).value.size() >= max_elems)
return;
Node * node = Node::allocate(*columns[0], row_num, arena);
data(place).value.push_back(node, arena);
}
if constexpr (Trait::sampler == Sampler::RNG)
{
auto & a = data(place);
++a.total_values;
if (a.value.size() < max_elems)
a.value.push_back(Node::allocate(*columns[0], row_num, arena), arena);
else
{
UInt64 rnd = a.genRandom(a.total_values);
if (rnd < max_elems)
a.value[rnd] = Node::allocate(*columns[0], row_num, arena);
}
}
// TODO
// if constexpr (Trait::sampler == Sampler::DETERMINATOR)
}
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena * arena) const override
{
if constexpr (Trait::sampler == Sampler::NONE)
mergeNoSampler(place, rhs, arena);
else if constexpr (Trait::sampler == Sampler::RNG)
mergeWithRNGSampler(place, rhs, arena);
// TODO
// else if constexpr (Trait::sampler == Sampler::DETERMINATOR)
}
void ALWAYS_INLINE mergeNoSampler(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena * arena) const
{
if (data(rhs).value.empty()) /// rhs state is empty
return;
UInt64 new_elems;
if (limit_num_elems)
{
if (data(place).value.size() >= max_elems)
return;
new_elems = std::min(data(rhs).value.size(), static_cast<size_t>(max_elems) - data(place).value.size());
}
else
new_elems = data(rhs).value.size();
auto & a = data(place).value;
auto & b = data(rhs).value;
for (UInt64 i = 0; i < new_elems; ++i)
a.push_back(b[i]->clone(arena), arena);
}
void ALWAYS_INLINE mergeWithRNGSampler(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena * arena) const
{
if (data(rhs).value.empty()) /// rhs state is empty
return;
auto & a = data(place);
auto & b = data(rhs);
if (b.total_values <= max_elems)
{
for (size_t i = 0; i < b.value.size(); ++i)
insert(a, b.value[i], arena);
}
else if (a.total_values <= max_elems)
{
decltype(a.value) from;
from.swap(a.value, arena);
for (auto & node : b.value)
a.value.push_back(node->clone(arena), arena);
a.total_values = b.total_values;
for (size_t i = 0; i < from.size(); ++i)
insert(a, from[i], arena);
}
else
{
a.randomShuffle();
a.total_values += b.total_values;
for (size_t i = 0; i < max_elems; ++i)
{
UInt64 rnd = a.genRandom(a.total_values);
if (rnd < b.total_values)
a.value[i] = b.value[i]->clone(arena);
}
}
}
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
{
writeVarUInt(data(place).value.size(), buf);
auto & value = data(place).value;
for (auto & node : value)
node->write(buf);
if constexpr (Trait::sampler == Sampler::RNG)
{
DB::writeIntBinary<size_t>(data(place).total_values, buf);
std::ostringstream rng_stream;
rng_stream << data(place).rng;
DB::writeStringBinary(rng_stream.str(), buf);
}
// TODO
// if constexpr (Trait::sampler == Sampler::DETERMINATOR)
}
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena * arena) const override
{
UInt64 elems;
readVarUInt(elems, buf);
if (unlikely(elems == 0))
return;
if (unlikely(elems > AGGREGATE_FUNCTION_GROUP_ARRAY_MAX_ARRAY_SIZE))
throw Exception("Too large array size", ErrorCodes::TOO_LARGE_ARRAY_SIZE);
if (limit_num_elems && unlikely(elems > max_elems))
throw Exception("Too large array size, it should not exceed " + toString(max_elems), ErrorCodes::TOO_LARGE_ARRAY_SIZE);
auto & value = data(place).value;
value.resize(elems, arena);
for (UInt64 i = 0; i < elems; ++i)
value[i] = Node::read(buf, arena);
if constexpr (Trait::sampler == Sampler::RNG)
{
DB::readIntBinary<size_t>(data(place).total_values, buf);
std::string rng_string;
DB::readStringBinary(rng_string, buf);
std::istringstream rng_stream(rng_string);
rng_stream >> data(place).rng;
}
// TODO
// if constexpr (Trait::sampler == Sampler::DETERMINATOR)
}
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
{
auto & column_array = assert_cast<ColumnArray &>(to);
auto & offsets = column_array.getOffsets();
offsets.push_back(offsets.back() + data(place).value.size());
auto & column_data = column_array.getData();
if (std::is_same_v<Node, GroupArrayNodeString>)
{
auto & string_offsets = assert_cast<ColumnString &>(column_data).getOffsets();
string_offsets.reserve(string_offsets.size() + data(place).value.size());
}
auto & value = data(place).value;
for (auto & node : value)
node->insertInto(column_data);
}
bool allocatesMemoryInArena() const override { return true; }
};
template <typename Node>
struct GroupArrayListNodeBase : public GroupArrayNodeBase<Node>
{
Node * next;
};
struct GroupArrayListNodeString : public GroupArrayListNodeBase<GroupArrayListNodeString> struct GroupArrayListNodeString : public GroupArrayListNodeBase<GroupArrayListNodeString>
{ {
using Node = GroupArrayListNodeString; using Node = GroupArrayListNodeString;
@ -200,10 +648,7 @@ struct GroupArrayListNodeString : public GroupArrayListNodeBase<GroupArrayListNo
return node; return node;
} }
void insertInto(IColumn & column) void insertInto(IColumn & column) { assert_cast<ColumnString &>(column).insertData(data(), size); }
{
assert_cast<ColumnString &>(column).insertData(data(), size);
}
}; };
struct GroupArrayListNodeGeneral : public GroupArrayListNodeBase<GroupArrayListNodeGeneral> struct GroupArrayListNodeGeneral : public GroupArrayListNodeBase<GroupArrayListNodeGeneral>
@ -222,10 +667,7 @@ struct GroupArrayListNodeGeneral : public GroupArrayListNodeBase<GroupArrayListN
return node; return node;
} }
void insertInto(IColumn & column) void insertInto(IColumn & column) { column.deserializeAndInsertFromArena(data()); }
{
column.deserializeAndInsertFromArena(data());
}
}; };
@ -240,23 +682,27 @@ struct GroupArrayGeneralListData
/// Implementation of groupArray for String or any ComplexObject via linked list /// Implementation of groupArray for String or any ComplexObject via linked list
/// It has poor performance in case of many small objects /// It has poor performance in case of many small objects
template <typename Node, bool limit_num_elems> template <typename Node, typename Trait>
class GroupArrayGeneralListImpl final class GroupArrayGeneralListImpl final
: public IAggregateFunctionDataHelper<GroupArrayGeneralListData<Node>, GroupArrayGeneralListImpl<Node, limit_num_elems>> : public IAggregateFunctionDataHelper<GroupArrayGeneralListData<Node>, GroupArrayGeneralListImpl<Node, Trait>>
{ {
static constexpr bool limit_num_elems = Trait::has_limit;
using Data = GroupArrayGeneralListData<Node>; using Data = GroupArrayGeneralListData<Node>;
static Data & data(AggregateDataPtr place) { return *reinterpret_cast<Data*>(place); } static Data & data(AggregateDataPtr place) { return *reinterpret_cast<Data *>(place); }
static const Data & data(ConstAggregateDataPtr place) { return *reinterpret_cast<const Data*>(place); } static const Data & data(ConstAggregateDataPtr place) { return *reinterpret_cast<const Data *>(place); }
DataTypePtr & data_type; DataTypePtr & data_type;
UInt64 max_elems; UInt64 max_elems;
public: public:
GroupArrayGeneralListImpl(const DataTypePtr & data_type_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max()) GroupArrayGeneralListImpl(const DataTypePtr & data_type_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
: IAggregateFunctionDataHelper<GroupArrayGeneralListData<Node>, GroupArrayGeneralListImpl<Node, limit_num_elems>>({data_type_}, {}) : IAggregateFunctionDataHelper<GroupArrayGeneralListData<Node>, GroupArrayGeneralListImpl<Node, Trait>>({data_type_}, {})
, data_type(this->argument_types[0]), max_elems(max_elems_) {} , data_type(this->argument_types[0])
, max_elems(max_elems_)
{
}
String getName() const override { return "groupArray"; } String getName() const override { return getNameByTrait<Trait>(); }
DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(data_type); } DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(data_type); }
@ -295,7 +741,7 @@ public:
if (data(place).elems >= max_elems) if (data(place).elems >= max_elems)
return; return;
new_elems = std::min(data(place).elems + data(rhs).elems, max_elems); new_elems = std::min(data(place).elems + data(rhs).elems, static_cast<size_t>(max_elems));
} }
else else
{ {
@ -394,10 +840,7 @@ public:
} }
} }
bool allocatesMemoryInArena() const override bool allocatesMemoryInArena() const override { return true; }
{
return true;
}
}; };
#undef AGGREGATE_FUNCTION_GROUP_ARRAY_MAX_ARRAY_SIZE #undef AGGREGATE_FUNCTION_GROUP_ARRAY_MAX_ARRAY_SIZE

29
dbms/src/AggregateFunctions/AggregateFunctionMLMethod.cpp
View File

@ -487,10 +487,17 @@ void LogisticRegression::compute(
size_t row_num) size_t row_num)
{ {
Float64 derivative = bias; Float64 derivative = bias;
std::vector<Float64> values(weights.size());
for (size_t i = 0; i < weights.size(); ++i) for (size_t i = 0; i < weights.size(); ++i)
{ {
auto value = (*columns[i]).getFloat64(row_num); values[i] = (*columns[i]).getFloat64(row_num);
derivative += weights[i] * value; }
for (size_t i = 0; i < weights.size(); ++i)
{
derivative += weights[i] * values[i];
} }
derivative *= target; derivative *= target;
derivative = exp(derivative); derivative = exp(derivative);
@ -498,8 +505,7 @@ void LogisticRegression::compute(
batch_gradient[weights.size()] += target / (derivative + 1); batch_gradient[weights.size()] += target / (derivative + 1);
for (size_t i = 0; i < weights.size(); ++i) for (size_t i = 0; i < weights.size(); ++i)
{ {
auto value = (*columns[i]).getFloat64(row_num); batch_gradient[i] += target * values[i] / (derivative + 1) - 2 * l2_reg_coef * weights[i];
batch_gradient[i] += target * value / (derivative + 1) - 2 * l2_reg_coef * weights[i];
} }
} }
@ -558,18 +564,25 @@ void LinearRegression::compute(
size_t row_num) size_t row_num)
{ {
Float64 derivative = (target - bias); Float64 derivative = (target - bias);
std::vector<Float64> values(weights.size());
for (size_t i = 0; i < weights.size(); ++i) for (size_t i = 0; i < weights.size(); ++i)
{ {
auto value = (*columns[i]).getFloat64(row_num); values[i] = (*columns[i]).getFloat64(row_num);
derivative -= weights[i] * value; }
for (size_t i = 0; i < weights.size(); ++i)
{
derivative -= weights[i] * values[i];
} }
derivative *= 2; derivative *= 2;
batch_gradient[weights.size()] += derivative; batch_gradient[weights.size()] += derivative;
for (size_t i = 0; i < weights.size(); ++i) for (size_t i = 0; i < weights.size(); ++i)
{ {
auto value = (*columns[i]).getFloat64(row_num); batch_gradient[i] += derivative * values[i] - 2 * l2_reg_coef * weights[i];
batch_gradient[i] += derivative * value - 2 * l2_reg_coef * weights[i];
} }
} }

4
dbms/src/Client/Connection.cpp
View File

@ -774,9 +774,7 @@ std::unique_ptr<Exception> Connection::receiveException()
{ {
//LOG_TRACE(log_wrapper.get(), "Receiving exception"); //LOG_TRACE(log_wrapper.get(), "Receiving exception");
Exception e; return std::make_unique<Exception>(readException(*in, "Received from " + getDescription()));
readException(e, *in, "Received from " + getDescription());
return std::unique_ptr<Exception>{ e.clone() };
} }

2
dbms/src/Columns/ColumnVector.cpp
View File

@ -217,7 +217,7 @@ UInt64 ColumnVector<T>::get64(size_t n) const
} }
template <typename T> template <typename T>
Float64 ColumnVector<T>::getFloat64(size_t n) const inline Float64 ColumnVector<T>::getFloat64(size_t n) const
{ {
return static_cast<Float64>(data[n]); return static_cast<Float64>(data[n]);
} }

24
dbms/src/Common/ArenaAllocator.h
View File

@ -35,6 +35,12 @@ public:
{ {
// Do nothing, trash in arena remains. // Do nothing, trash in arena remains.
} }
protected:
static constexpr size_t getStackThreshold()
{
return 0;
}
}; };
@ -66,6 +72,12 @@ public:
static void free(void * /*buf*/, size_t /*size*/) static void free(void * /*buf*/, size_t /*size*/)
{ {
} }
protected:
static constexpr size_t getStackThreshold()
{
return 0;
}
}; };
@ -100,6 +112,12 @@ public:
if (size >= REAL_ALLOCATION_TRESHOLD) if (size >= REAL_ALLOCATION_TRESHOLD)
TRealAllocator::free(buf, size); TRealAllocator::free(buf, size);
} }
protected:
static constexpr size_t getStackThreshold()
{
return 0;
}
}; };
@ -136,6 +154,12 @@ public:
} }
void free(void * /*buf*/, size_t /*size*/) {} void free(void * /*buf*/, size_t /*size*/) {}
protected:
static constexpr size_t getStackThreshold()
{
return N;
}
}; };
} }

2
dbms/src/Common/COW.h
View File

@ -15,7 +15,7 @@
private: private:
friend class COW<Column>; friend class COW<Column>;
/// Leave all constructors in private section. They will be avaliable through 'create' method. /// Leave all constructors in private section. They will be available through 'create' method.
Column(); Column();
/// Provide 'clone' method. It can be virtual if you want polymorphic behaviour. /// Provide 'clone' method. It can be virtual if you want polymorphic behaviour.

4
dbms/src/Common/ErrorCodes.cpp
View File

@ -387,7 +387,6 @@ namespace ErrorCodes
extern const int PTHREAD_ERROR = 411; extern const int PTHREAD_ERROR = 411;
extern const int NETLINK_ERROR = 412; extern const int NETLINK_ERROR = 412;
extern const int CANNOT_SET_SIGNAL_HANDLER = 413; extern const int CANNOT_SET_SIGNAL_HANDLER = 413;
extern const int CANNOT_READLINE = 414;
extern const int ALL_REPLICAS_LOST = 415; extern const int ALL_REPLICAS_LOST = 415;
extern const int REPLICA_STATUS_CHANGED = 416; extern const int REPLICA_STATUS_CHANGED = 416;
extern const int EXPECTED_ALL_OR_ANY = 417; extern const int EXPECTED_ALL_OR_ANY = 417;
@ -476,6 +475,9 @@ namespace ErrorCodes
extern const int CANNOT_CREATE_DATABASE = 501; extern const int CANNOT_CREATE_DATABASE = 501;
extern const int CANNOT_SIGQUEUE = 502; extern const int CANNOT_SIGQUEUE = 502;
extern const int AGGREGATE_FUNCTION_THROW = 503; extern const int AGGREGATE_FUNCTION_THROW = 503;
extern const int FILE_ALREADY_EXISTS = 504;
extern const int CANNOT_DELETE_DIRECTORY = 505;
extern const int UNEXPECTED_ERROR_CODE = 506;
extern const int KEEPER_EXCEPTION = 999; extern const int KEEPER_EXCEPTION = 999;
extern const int POCO_EXCEPTION = 1000; extern const int POCO_EXCEPTION = 1000;

3
dbms/src/Common/Exception.cpp
View File

@ -23,6 +23,7 @@ namespace ErrorCodes
extern const int UNKNOWN_EXCEPTION; extern const int UNKNOWN_EXCEPTION;
extern const int CANNOT_TRUNCATE_FILE; extern const int CANNOT_TRUNCATE_FILE;
extern const int NOT_IMPLEMENTED; extern const int NOT_IMPLEMENTED;
extern const int LOGICAL_ERROR;
} }
@ -33,6 +34,8 @@ Exception::Exception()
Exception::Exception(const std::string & msg, int code) Exception::Exception(const std::string & msg, int code)
: Poco::Exception(msg, code) : Poco::Exception(msg, code)
{ {
// In debug builds, treat LOGICAL_ERROR as an assertion failure.
assert(code != ErrorCodes::LOGICAL_ERROR);
} }
Exception::Exception(CreateFromPocoTag, const Poco::Exception & exc) Exception::Exception(CreateFromPocoTag, const Poco::Exception & exc)

79
dbms/src/Common/FileChecker.cpp
View File

@ -1,5 +1,4 @@
#include <common/JSON.h> #include <common/JSON.h>
#include <Poco/Path.h>
#include <IO/WriteBufferFromFile.h> #include <IO/WriteBufferFromFile.h>
#include <IO/ReadBufferFromFile.h> #include <IO/ReadBufferFromFile.h>
#include <IO/WriteBufferFromString.h> #include <IO/WriteBufferFromString.h>
@ -13,28 +12,26 @@
namespace DB namespace DB
{ {
FileChecker::FileChecker(DiskPtr disk_, const String & file_info_path_) : disk(std::move(disk_))
FileChecker::FileChecker(const std::string & file_info_path_)
{ {
setPath(file_info_path_); setPath(file_info_path_);
} }
void FileChecker::setPath(const std::string & file_info_path_) void FileChecker::setPath(const String & file_info_path_)
{ {
files_info_path = file_info_path_; files_info_path = file_info_path_;
Poco::Path path(files_info_path); tmp_files_info_path = parentPath(files_info_path) + "tmp_" + fileName(files_info_path);
tmp_files_info_path = path.parent().toString() + "tmp_" + path.getFileName();
} }
void FileChecker::update(const Poco::File & file) void FileChecker::update(const String & file_path)
{ {
initialize(); initialize();
updateImpl(file); updateImpl(file_path);
save(); save();
} }
void FileChecker::update(const Files::const_iterator & begin, const Files::const_iterator & end) void FileChecker::update(const Strings::const_iterator & begin, const Strings::const_iterator & end)
{ {
initialize(); initialize();
for (auto it = begin; it != end; ++it) for (auto it = begin; it != end; ++it)
@ -44,9 +41,8 @@ void FileChecker::update(const Files::const_iterator & begin, const Files::const
CheckResults FileChecker::check() const CheckResults FileChecker::check() const
{ {
/** Read the files again every time you call `check` - so as not to violate the constancy. // Read the files again every time you call `check` - so as not to violate the constancy.
* `check` method is rarely called. // `check` method is rarely called.
*/
CheckResults results; CheckResults results;
Map local_map; Map local_map;
@ -57,22 +53,21 @@ CheckResults FileChecker::check() const
for (const auto & name_size : local_map) for (const auto & name_size : local_map)
{ {
Poco::Path path = Poco::Path(files_info_path).parent().toString() + "/" + name_size.first; const String & name = name_size.first;
Poco::File file(path); String path = parentPath(files_info_path) + name;
if (!file.exists()) if (!disk->exists(path))
{ {
results.emplace_back(path.getFileName(), false, "File " + file.path() + " doesn't exist"); results.emplace_back(name, false, "File " + path + " doesn't exist");
break; break;
} }
auto real_size = disk->getFileSize(path);
size_t real_size = file.getSize();
if (real_size != name_size.second) if (real_size != name_size.second)
{ {
results.emplace_back(path.getFileName(), false, "Size of " + file.path() + " is wrong. Size is " + toString(real_size) + " but should be " + toString(name_size.second)); results.emplace_back(name, false, "Size of " + path + " is wrong. Size is " + toString(real_size) + " but should be " + toString(name_size.second));
break; break;
} }
results.emplace_back(path.getFileName(), true, ""); results.emplace_back(name, true, "");
} }
return results; return results;
@ -87,64 +82,54 @@ void FileChecker::initialize()
initialized = true; initialized = true;
} }
void FileChecker::updateImpl(const Poco::File & file) void FileChecker::updateImpl(const String & file_path)
{ {
map[Poco::Path(file.path()).getFileName()] = file.getSize(); map[fileName(file_path)] = disk->getFileSize(file_path);
} }
void FileChecker::save() const void FileChecker::save() const
{ {
{ {
WriteBufferFromFile out(tmp_files_info_path); std::unique_ptr<WriteBuffer> out = disk->writeFile(tmp_files_info_path);
/// So complex JSON structure - for compatibility with the old format. /// So complex JSON structure - for compatibility with the old format.
writeCString("{\"yandex\":{", out); writeCString("{\"yandex\":{", *out);
auto settings = FormatSettings(); auto settings = FormatSettings();
for (auto it = map.begin(); it != map.end(); ++it) for (auto it = map.begin(); it != map.end(); ++it)
{ {
if (it != map.begin()) if (it != map.begin())
writeString(",", out); writeString(",", *out);
/// `escapeForFileName` is not really needed. But it is left for compatibility with the old code. /// `escapeForFileName` is not really needed. But it is left for compatibility with the old code.
writeJSONString(escapeForFileName(it->first), out, settings); writeJSONString(escapeForFileName(it->first), *out, settings);
writeString(":{\"size\":\"", out); writeString(":{\"size\":\"", *out);
writeIntText(it->second, out); writeIntText(it->second, *out);
writeString("\"}", out); writeString("\"}", *out);
} }
writeCString("}}", out); writeCString("}}", *out);
out.next(); out->next();
} }
Poco::File current_file(files_info_path); disk->replaceFile(tmp_files_info_path, files_info_path);
if (current_file.exists())
{
std::string old_file_name = files_info_path + ".old";
current_file.renameTo(old_file_name);
Poco::File(tmp_files_info_path).renameTo(files_info_path);
Poco::File(old_file_name).remove();
}
else
Poco::File(tmp_files_info_path).renameTo(files_info_path);
} }
void FileChecker::load(Map & local_map, const std::string & path) void FileChecker::load(Map & local_map, const String & path) const
{ {
local_map.clear(); local_map.clear();
if (!Poco::File(path).exists()) if (!disk->exists(path))
return; return;
ReadBufferFromFile in(path); std::unique_ptr<ReadBuffer> in = disk->readFile(path);
WriteBufferFromOwnString out; WriteBufferFromOwnString out;
/// The JSON library does not support whitespace. We delete them. Inefficient. /// The JSON library does not support whitespace. We delete them. Inefficient.
while (!in.eof()) while (!in->eof())
{ {
char c; char c;
readChar(c, in); readChar(c, *in);
if (!isspace(c)) if (!isspace(c))
writeChar(c, out); writeChar(c, out);
} }

33
dbms/src/Common/FileChecker.h
View File

@ -1,9 +1,8 @@
#pragma once #pragma once
#include <string>
#include <common/logger_useful.h> #include <common/logger_useful.h>
#include <Poco/File.h>
#include <Storages/CheckResults.h> #include <Storages/CheckResults.h>
#include <Disks/IDisk.h>
namespace DB namespace DB
@ -12,29 +11,27 @@ namespace DB
/// stores the sizes of all columns, and can check whether the columns are corrupted /// stores the sizes of all columns, and can check whether the columns are corrupted
class FileChecker class FileChecker
{ {
private:
/// File name -> size.
using Map = std::map<std::string, size_t>;
public: public:
using Files = std::vector<Poco::File>; FileChecker(DiskPtr disk_, const String & file_info_path_);
void setPath(const String & file_info_path_);
FileChecker(const std::string & file_info_path_); void update(const String & file_path);
void setPath(const std::string & file_info_path_); void update(const Strings::const_iterator & begin, const Strings::const_iterator & end);
void update(const Poco::File & file);
void update(const Files::const_iterator & begin, const Files::const_iterator & end);
/// Check the files whose parameters are specified in sizes.json /// Check the files whose parameters are specified in sizes.json
CheckResults check() const; CheckResults check() const;
private: private:
void initialize(); /// File name -> size.
void updateImpl(const Poco::File & file); using Map = std::map<String, UInt64>;
void save() const;
static void load(Map & local_map, const std::string & path);
std::string files_info_path; void initialize();
std::string tmp_files_info_path; void updateImpl(const String & file_path);
void save() const;
void load(Map & local_map, const String & path) const;
DiskPtr disk;
String files_info_path;
String tmp_files_info_path;
/// The data from the file is read lazily. /// The data from the file is read lazily.
Map map; Map map;

25
dbms/src/Common/PODArray.h
View File

@ -150,7 +150,7 @@ protected:
bool isAllocatedFromStack() const bool isAllocatedFromStack() const
{ {
constexpr size_t stack_threshold = TAllocator::getStackThreshold(); static constexpr size_t stack_threshold = TAllocator::getStackThreshold();
return (stack_threshold > 0) && (allocated_bytes() <= stack_threshold); return (stack_threshold > 0) && (allocated_bytes() <= stack_threshold);
} }
@ -453,7 +453,8 @@ public:
this->c_end += bytes_to_copy; this->c_end += bytes_to_copy;
} }
void swap(PODArray & rhs) template <typename... TAllocatorParams>
void swap(PODArray & rhs, TAllocatorParams &&... allocator_params)
{ {
#ifndef NDEBUG #ifndef NDEBUG
this->unprotect(); this->unprotect();
@ -463,7 +464,7 @@ public:
/// Swap two PODArray objects, arr1 and arr2, that satisfy the following conditions: /// Swap two PODArray objects, arr1 and arr2, that satisfy the following conditions:
/// - The elements of arr1 are stored on stack. /// - The elements of arr1 are stored on stack.
/// - The elements of arr2 are stored on heap. /// - The elements of arr2 are stored on heap.
auto swap_stack_heap = [this](PODArray & arr1, PODArray & arr2) auto swap_stack_heap = [&](PODArray & arr1, PODArray & arr2)
{ {
size_t stack_size = arr1.size(); size_t stack_size = arr1.size();
size_t stack_allocated = arr1.allocated_bytes(); size_t stack_allocated = arr1.allocated_bytes();
@ -480,18 +481,18 @@ public:
arr1.c_end = arr1.c_start + this->byte_size(heap_size); arr1.c_end = arr1.c_start + this->byte_size(heap_size);
/// Allocate stack space for arr2. /// Allocate stack space for arr2.
arr2.alloc(stack_allocated); arr2.alloc(stack_allocated, std::forward<TAllocatorParams>(allocator_params)...);
/// Copy the stack content. /// Copy the stack content.
memcpy(arr2.c_start, stack_c_start, this->byte_size(stack_size)); memcpy(arr2.c_start, stack_c_start, this->byte_size(stack_size));
arr2.c_end = arr2.c_start + this->byte_size(stack_size); arr2.c_end = arr2.c_start + this->byte_size(stack_size);
}; };
auto do_move = [this](PODArray & src, PODArray & dest) auto do_move = [&](PODArray & src, PODArray & dest)
{ {
if (src.isAllocatedFromStack()) if (src.isAllocatedFromStack())
{ {
dest.dealloc(); dest.dealloc();
dest.alloc(src.allocated_bytes()); dest.alloc(src.allocated_bytes(), std::forward<TAllocatorParams>(allocator_params)...);
memcpy(dest.c_start, src.c_start, this->byte_size(src.size())); memcpy(dest.c_start, src.c_start, this->byte_size(src.size()));
dest.c_end = dest.c_start + (src.c_end - src.c_start); dest.c_end = dest.c_start + (src.c_end - src.c_start);
@ -569,24 +570,26 @@ public:
} }
} }
void assign(size_t n, const T & x) template <typename... TAllocatorParams>
void assign(size_t n, const T & x, TAllocatorParams &&... allocator_params)
{ {
this->resize(n); this->resize(n, std::forward<TAllocatorParams>(allocator_params)...);
std::fill(begin(), end(), x); std::fill(begin(), end(), x);
} }
template <typename It1, typename It2> template <typename It1, typename It2, typename... TAllocatorParams>
void assign(It1 from_begin, It2 from_end) void assign(It1 from_begin, It2 from_end, TAllocatorParams &&... allocator_params)
{ {
size_t required_capacity = from_end - from_begin; size_t required_capacity = from_end - from_begin;
if (required_capacity > this->capacity()) if (required_capacity > this->capacity())
this->reserve(roundUpToPowerOfTwoOrZero(required_capacity)); this->reserve(roundUpToPowerOfTwoOrZero(required_capacity), std::forward<TAllocatorParams>(allocator_params)...);
size_t bytes_to_copy = this->byte_size(required_capacity); size_t bytes_to_copy = this->byte_size(required_capacity);
memcpy(this->c_start, reinterpret_cast<const void *>(&*from_begin), bytes_to_copy); memcpy(this->c_start, reinterpret_cast<const void *>(&*from_begin), bytes_to_copy);
this->c_end = this->c_start + bytes_to_copy; this->c_end = this->c_start + bytes_to_copy;
} }
// ISO C++ has strict ambiguity rules, thus we cannot apply TAllocatorParams here.
void assign(const PODArray & from) void assign(const PODArray & from)
{ {
assign(from.begin(), from.end()); assign(from.begin(), from.end());

4
dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp
View File

@ -85,7 +85,7 @@ TEST(zkutil, multi_async)
ops.clear(); ops.clear();
auto res = fut.get(); auto res = fut.get();
ASSERT_TRUE(res.error == Coordination::ZOK); ASSERT_EQ(res.error, Coordination::ZOK);
ASSERT_EQ(res.responses.size(), 2); ASSERT_EQ(res.responses.size(), 2);
} }
@ -121,7 +121,7 @@ TEST(zkutil, multi_async)
ops.clear(); ops.clear();
auto res = fut.get(); auto res = fut.get();
ASSERT_TRUE(res.error == Coordination::ZNODEEXISTS); ASSERT_EQ(res.error, Coordination::ZNODEEXISTS);
ASSERT_EQ(res.responses.size(), 2); ASSERT_EQ(res.responses.size(), 2);
} }
} }

6
dbms/src/Compression/CompressionCodecT64.cpp
View File

@ -516,13 +516,13 @@ UInt32 CompressionCodecT64::doCompressData(const char * src, UInt32 src_size, ch
break; break;
} }
throw Exception("Connot compress with T64", ErrorCodes::CANNOT_COMPRESS); throw Exception("Cannot compress with T64", ErrorCodes::CANNOT_COMPRESS);
} }
void CompressionCodecT64::doDecompressData(const char * src, UInt32 src_size, char * dst, UInt32 uncompressed_size) const void CompressionCodecT64::doDecompressData(const char * src, UInt32 src_size, char * dst, UInt32 uncompressed_size) const
{ {
if (!src_size) if (!src_size)
throw Exception("Connot decompress with T64", ErrorCodes::CANNOT_DECOMPRESS); throw Exception("Cannot decompress with T64", ErrorCodes::CANNOT_DECOMPRESS);
UInt8 cookie = unalignedLoad<UInt8>(src); UInt8 cookie = unalignedLoad<UInt8>(src);
src += 1; src += 1;
@ -553,7 +553,7 @@ void CompressionCodecT64::doDecompressData(const char * src, UInt32 src_size, ch
break; break;
} }
throw Exception("Connot decompress with T64", ErrorCodes::CANNOT_DECOMPRESS); throw Exception("Cannot decompress with T64", ErrorCodes::CANNOT_DECOMPRESS);
} }
void CompressionCodecT64::useInfoAboutType(DataTypePtr data_type) void CompressionCodecT64::useInfoAboutType(DataTypePtr data_type)

2
dbms/src/Core/ExternalTable.cpp
View File

@ -160,7 +160,7 @@ void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header,
/// Create table /// Create table
NamesAndTypesList columns = sample_block.getNamesAndTypesList(); NamesAndTypesList columns = sample_block.getNamesAndTypesList();
StoragePtr storage = StorageMemory::create("_external", data.second, ColumnsDescription{columns}, ConstraintsDescription{}); StoragePtr storage = StorageMemory::create(StorageID("_external", data.second), ColumnsDescription{columns}, ConstraintsDescription{});
storage->startup(); storage->startup();
context.addExternalTable(data.second, storage); context.addExternalTable(data.second, storage);
BlockOutputStreamPtr output = storage->write(ASTPtr(), context); BlockOutputStreamPtr output = storage->write(ASTPtr(), context);

2
dbms/src/Core/ExternalTable.h
View File

@ -99,7 +99,7 @@ class ExternalTablesHandler : public Poco::Net::PartHandler, BaseExternalTable
public: public:
ExternalTablesHandler(Context & context_, const Poco::Net::NameValueCollection & params_) : context(context_), params(params_) {} ExternalTablesHandler(Context & context_, const Poco::Net::NameValueCollection & params_) : context(context_), params(params_) {}
void handlePart(const Poco::Net::MessageHeader & header, std::istream & stream); void handlePart(const Poco::Net::MessageHeader & header, std::istream & stream) override;
private: private:
Context & context; Context & context;

6
dbms/src/Core/MySQLProtocol.h
View File

@ -1030,6 +1030,7 @@ public:
LOG_TRACE(log, "Authentication method match."); LOG_TRACE(log, "Authentication method match.");
} }
bool sent_public_key = false;
if (auth_response == "\1") if (auth_response == "\1")
{ {
LOG_TRACE(log, "Client requests public key."); LOG_TRACE(log, "Client requests public key.");
@ -1050,6 +1051,7 @@ public:
AuthMoreData data(pem); AuthMoreData data(pem);
packet_sender->sendPacket(data, true); packet_sender->sendPacket(data, true);
sent_public_key = true;
AuthSwitchResponse response; AuthSwitchResponse response;
packet_sender->receivePacket(response); packet_sender->receivePacket(response);
@ -1069,13 +1071,15 @@ public:
*/ */
if (!is_secure_connection && !auth_response->empty() && auth_response != String("\0", 1)) if (!is_secure_connection && !auth_response->empty() && auth_response != String("\0", 1))
{ {
LOG_TRACE(log, "Received nonempty password"); LOG_TRACE(log, "Received nonempty password.");
auto ciphertext = reinterpret_cast<unsigned char *>(auth_response->data()); auto ciphertext = reinterpret_cast<unsigned char *>(auth_response->data());
unsigned char plaintext[RSA_size(&private_key)]; unsigned char plaintext[RSA_size(&private_key)];
int plaintext_size = RSA_private_decrypt(auth_response->size(), ciphertext, plaintext, &private_key, RSA_PKCS1_OAEP_PADDING); int plaintext_size = RSA_private_decrypt(auth_response->size(), ciphertext, plaintext, &private_key, RSA_PKCS1_OAEP_PADDING);
if (plaintext_size == -1) if (plaintext_size == -1)
{ {
if (!sent_public_key)
LOG_WARNING(log, "Client could have encrypted password with different public key since it didn't request it from server.");
throw Exception("Failed to decrypt auth data. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR); throw Exception("Failed to decrypt auth data. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);
} }

23
dbms/src/Core/Settings.h
View File

@ -65,14 +65,14 @@ struct Settings : public SettingsCollection<Settings>
M(SettingSeconds, send_timeout, DBMS_DEFAULT_SEND_TIMEOUT_SEC, "", 0) \ M(SettingSeconds, send_timeout, DBMS_DEFAULT_SEND_TIMEOUT_SEC, "", 0) \
M(SettingSeconds, tcp_keep_alive_timeout, 0, "The time in seconds the connection needs to remain idle before TCP starts sending keepalive probes", 0) \ M(SettingSeconds, tcp_keep_alive_timeout, 0, "The time in seconds the connection needs to remain idle before TCP starts sending keepalive probes", 0) \
M(SettingMilliseconds, queue_max_wait_ms, 0, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.", 0) \ M(SettingMilliseconds, queue_max_wait_ms, 0, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.", 0) \
M(SettingMilliseconds, connection_pool_max_wait_ms, 0, "The wait time when connection pool is full.", 0) \ M(SettingMilliseconds, connection_pool_max_wait_ms, 0, "The wait time when the connection pool is full.", 0) \
M(SettingMilliseconds, replace_running_query_max_wait_ms, 5000, "The wait time for running query with the same query_id to finish when setting 'replace_running_query' is active.", 0) \ M(SettingMilliseconds, replace_running_query_max_wait_ms, 5000, "The wait time for running query with the same query_id to finish when setting 'replace_running_query' is active.", 0) \
M(SettingMilliseconds, kafka_max_wait_ms, 5000, "The wait time for reading from Kafka before retry.", 0) \ M(SettingMilliseconds, kafka_max_wait_ms, 5000, "The wait time for reading from Kafka before retry.", 0) \
M(SettingUInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.", 0) \ M(SettingUInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.", 0) \
M(SettingUInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.", 0) \ M(SettingUInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.", 0) \
M(SettingUInt64, distributed_connections_pool_size, DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE, "Maximum number of connections with one remote server in the pool.", 0) \ M(SettingUInt64, distributed_connections_pool_size, DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE, "Maximum number of connections with one remote server in the pool.", 0) \
M(SettingUInt64, connections_with_failover_max_tries, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, "The maximum number of attempts to connect to replicas.", 0) \ M(SettingUInt64, connections_with_failover_max_tries, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, "The maximum number of attempts to connect to replicas.", 0) \
M(SettingUInt64, s3_min_upload_part_size, 512*1024*1024, "The mininum size of part to upload during multipart upload to S3.", 0) \ M(SettingUInt64, s3_min_upload_part_size, 512*1024*1024, "The minimum size of part to upload during multipart upload to S3.", 0) \
M(SettingBool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.", IMPORTANT) \ M(SettingBool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.", IMPORTANT) \
M(SettingBool, use_uncompressed_cache, true, "Whether to use the cache of uncompressed blocks.", 0) \ M(SettingBool, use_uncompressed_cache, true, "Whether to use the cache of uncompressed blocks.", 0) \
M(SettingBool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \ M(SettingBool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \
@ -184,8 +184,8 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, input_format_tsv_empty_as_default, false, "Treat empty fields in TSV input as default values.", 0) \ M(SettingBool, input_format_tsv_empty_as_default, false, "Treat empty fields in TSV input as default values.", 0) \
M(SettingBool, input_format_null_as_default, false, "For text input formats initialize null fields with default values if data type of this field is not nullable", 0) \ M(SettingBool, input_format_null_as_default, false, "For text input formats initialize null fields with default values if data type of this field is not nullable", 0) \
\ \
M(SettingBool, input_format_values_interpret_expressions, true, "For Values format: if field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \ M(SettingBool, input_format_values_interpret_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \
M(SettingBool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \ M(SettingBool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \
M(SettingBool, input_format_values_accurate_types_of_literals, true, "For Values format: when parsing and interpreting expressions using template, check actual type of literal to avoid possible overflow and precision issues.", 0) \ M(SettingBool, input_format_values_accurate_types_of_literals, true, "For Values format: when parsing and interpreting expressions using template, check actual type of literal to avoid possible overflow and precision issues.", 0) \
\ \
M(SettingBool, output_format_json_quote_64bit_integers, true, "Controls quoting of 64-bit integers in JSON output format.", 0) \ M(SettingBool, output_format_json_quote_64bit_integers, true, "Controls quoting of 64-bit integers in JSON output format.", 0) \
@ -213,7 +213,7 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, join_use_nulls, 0, "Use NULLs for non-joined rows of outer JOINs for types that can be inside Nullable. If false, use default value of corresponding columns data type.", IMPORTANT) \ M(SettingBool, join_use_nulls, 0, "Use NULLs for non-joined rows of outer JOINs for types that can be inside Nullable. If false, use default value of corresponding columns data type.", IMPORTANT) \
\ \
M(SettingJoinStrictness, join_default_strictness, JoinStrictness::ALL, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \ M(SettingJoinStrictness, join_default_strictness, JoinStrictness::ALL, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \
M(SettingBool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys maping to be consistent with LEFT one.", IMPORTANT) \ M(SettingBool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys mapping to be consistent with LEFT one.", IMPORTANT) \
\ \
M(SettingUInt64, preferred_block_size_bytes, 1000000, "", 0) \ M(SettingUInt64, preferred_block_size_bytes, 1000000, "", 0) \
\ \
@ -250,8 +250,8 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, empty_result_for_aggregation_by_empty_set, false, "Return empty result when aggregating without keys on empty set.", 0) \ M(SettingBool, empty_result_for_aggregation_by_empty_set, false, "Return empty result when aggregating without keys on empty set.", 0) \
M(SettingBool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \ M(SettingBool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \
M(SettingUInt64, odbc_max_field_size, 1024, "Max size of filed can be read from ODBC dictionary. Long strings are truncated.", 0) \ M(SettingUInt64, odbc_max_field_size, 1024, "Max size of filed can be read from ODBC dictionary. Long strings are truncated.", 0) \
M(SettingUInt64, query_profiler_real_time_period_ns, 1000000000, "Highly experimental. Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(SettingUInt64, query_profiler_real_time_period_ns, 1000000000, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
M(SettingUInt64, query_profiler_cpu_time_period_ns, 1000000000, "Highly experimental. Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(SettingUInt64, query_profiler_cpu_time_period_ns, 1000000000, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
\ \
\ \
/** Limits during query execution are part of the settings. \ /** Limits during query execution are part of the settings. \
@ -311,9 +311,9 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, join_any_take_last_row, false, "When disabled (default) ANY JOIN will take the first found row for a key. When enabled, it will take the last row seen if there are multiple rows for the same key.", IMPORTANT) \ M(SettingBool, join_any_take_last_row, false, "When disabled (default) ANY JOIN will take the first found row for a key. When enabled, it will take the last row seen if there are multiple rows for the same key.", IMPORTANT) \
M(SettingBool, partial_merge_join, false, "Use partial merge join instead of hash join for LEFT and INNER JOINs.", 0) \ M(SettingBool, partial_merge_join, false, "Use partial merge join instead of hash join for LEFT and INNER JOINs.", 0) \
M(SettingBool, partial_merge_join_optimizations, false, "Enable optimizations in partial merge join", 0) \ M(SettingBool, partial_merge_join_optimizations, false, "Enable optimizations in partial merge join", 0) \
M(SettingUInt64, default_max_bytes_in_join, 100000000, "Maximum size of right-side table if limit's required but max_bytes_in_join is not set.", 0) \ M(SettingUInt64, default_max_bytes_in_join, 100000000, "Maximum size of right-side table if limit is required but max_bytes_in_join is not set.", 0) \
M(SettingUInt64, partial_merge_join_rows_in_right_blocks, 10000, "Split right-hand joining data in blocks of specified size. It's a portion of data indexed by min-max values and possibly unloaded on disk.", 0) \ M(SettingUInt64, partial_merge_join_rows_in_right_blocks, 10000, "Split right-hand joining data in blocks of specified size. It's a portion of data indexed by min-max values and possibly unloaded on disk.", 0) \
M(SettingUInt64, partial_merge_join_rows_in_left_blocks, 10000, "Group left-hand joining data in bigger blocks. Setting it to a bigger value increase JOIN performance and memory usage.", 0) \ M(SettingUInt64, partial_merge_join_rows_in_left_blocks, 10000, "Group left-hand joining data in bigger blocks. Setting it to a bigger value increases JOIN performance and memory usage.", 0) \
\ \
M(SettingUInt64, max_rows_to_transfer, 0, "Maximum size (in rows) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ M(SettingUInt64, max_rows_to_transfer, 0, "Maximum size (in rows) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \
M(SettingUInt64, max_bytes_to_transfer, 0, "Maximum size (in uncompressed bytes) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ M(SettingUInt64, max_bytes_to_transfer, 0, "Maximum size (in uncompressed bytes) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \
@ -361,7 +361,7 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \ M(SettingBool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \
M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql' and 'odbc' table functions.", 0) \ M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql' and 'odbc' table functions.", 0) \
\ \
M(SettingBool, experimental_use_processors, false, "Use processors pipeline.", 0) \ M(SettingBool, experimental_use_processors, true, "Use processors pipeline.", 0) \
\ \
M(SettingBool, allow_hyperscan, true, "Allow functions that use Hyperscan library. Disable to avoid potentially long compilation times and excessive resource usage.", 0) \ M(SettingBool, allow_hyperscan, true, "Allow functions that use Hyperscan library. Disable to avoid potentially long compilation times and excessive resource usage.", 0) \
M(SettingBool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.", 0) \ M(SettingBool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.", 0) \
@ -372,7 +372,7 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, allow_drop_detached, false, "Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries", 0) \ M(SettingBool, allow_drop_detached, false, "Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries", 0) \
\ \
M(SettingSeconds, distributed_replica_error_half_life, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_DECREASE_ERROR_PERIOD, "Time period reduces replica error counter by 2 times.", 0) \ M(SettingSeconds, distributed_replica_error_half_life, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_DECREASE_ERROR_PERIOD, "Time period reduces replica error counter by 2 times.", 0) \
M(SettingUInt64, distributed_replica_error_cap, DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT, "Max number of errors per replica, prevents piling up increadible amount of errors if replica was offline for some time and allows it to be reconsidered in a shorter amount of time.", 0) \ M(SettingUInt64, distributed_replica_error_cap, DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT, "Max number of errors per replica, prevents piling up an incredible amount of errors if replica was offline for some time and allows it to be reconsidered in a shorter amount of time.", 0) \
\ \
M(SettingBool, allow_experimental_live_view, false, "Enable LIVE VIEW. Not mature enough.", 0) \ M(SettingBool, allow_experimental_live_view, false, "Enable LIVE VIEW. Not mature enough.", 0) \
M(SettingSeconds, live_view_heartbeat_interval, DEFAULT_LIVE_VIEW_HEARTBEAT_INTERVAL_SEC, "The heartbeat interval in seconds to indicate live query is alive.", 0) \ M(SettingSeconds, live_view_heartbeat_interval, DEFAULT_LIVE_VIEW_HEARTBEAT_INTERVAL_SEC, "The heartbeat interval in seconds to indicate live query is alive.", 0) \
@ -395,6 +395,7 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, allow_experimental_data_skipping_indices, true, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \ M(SettingBool, allow_experimental_data_skipping_indices, true, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \
M(SettingBool, merge_tree_uniform_read_distribution, true, "Obsolete setting, does nothing. Will be removed after 2020-05-20", 0) \ M(SettingBool, merge_tree_uniform_read_distribution, true, "Obsolete setting, does nothing. Will be removed after 2020-05-20", 0) \
M(SettingUInt64, mark_cache_min_lifetime, 0, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \ M(SettingUInt64, mark_cache_min_lifetime, 0, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \
M(SettingUInt64, max_parser_depth, 1000, "Maximum parser depth.", 0) \
DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS)

3
dbms/src/Core/iostream_debug_helpers.cpp
View File

@ -47,7 +47,8 @@ std::ostream & operator<<(std::ostream & stream, const IDataType & what)
std::ostream & operator<<(std::ostream & stream, const IStorage & what) std::ostream & operator<<(std::ostream & stream, const IStorage & what)
{ {
stream << "IStorage(name = " << what.getName() << ", tableName = " << what.getTableName() << ") {" auto table_id = what.getStorageID();
stream << "IStorage(name = " << what.getName() << ", tableName = " << table_id.table_name << ") {"
<< what.getColumns().getAllPhysical().toString() << "}"; << what.getColumns().getAllPhysical().toString() << "}";
return stream; return stream;
} }

3
dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp
View File

@ -62,6 +62,9 @@ Block AddingDefaultsBlockInputStream::readImpl()
if (evaluate_block.has(column.first)) if (evaluate_block.has(column.first))
evaluate_block.erase(column.first); evaluate_block.erase(column.first);
if (!evaluate_block.columns())
evaluate_block.insert({ColumnConst::create(ColumnUInt8::create(1, 0), res.rows()), std::make_shared<DataTypeUInt8>(), "_dummy"});
evaluateMissingDefaults(evaluate_block, header.getNamesAndTypesList(), column_defaults, context, false); evaluateMissingDefaults(evaluate_block, header.getNamesAndTypesList(), column_defaults, context, false);
std::unordered_map<size_t, MutableColumnPtr> mixed_columns; std::unordered_map<size_t, MutableColumnPtr> mixed_columns;

20
dbms/src/DataStreams/AggregatingSortedBlockInputStream.cpp
View File

@ -60,8 +60,6 @@ AggregatingSortedBlockInputStream::AggregatingSortedBlockInputStream(
const BlockInputStreams & inputs_, const SortDescription & description_, size_t max_block_size_) const BlockInputStreams & inputs_, const SortDescription & description_, size_t max_block_size_)
: MergingSortedBlockInputStream(inputs_, description_, max_block_size_) : MergingSortedBlockInputStream(inputs_, description_, max_block_size_)
{ {
ColumnNumbers positions;
/// Fill in the column numbers that need to be aggregated. /// Fill in the column numbers that need to be aggregated.
for (size_t i = 0; i < num_columns; ++i) for (size_t i = 0; i < num_columns; ++i)
{ {
@ -96,7 +94,7 @@ AggregatingSortedBlockInputStream::AggregatingSortedBlockInputStream(
columns_to_simple_aggregate.emplace_back(std::move(desc)); columns_to_simple_aggregate.emplace_back(std::move(desc));
if (recursiveRemoveLowCardinality(column.type).get() != column.type.get()) if (recursiveRemoveLowCardinality(column.type).get() != column.type.get())
positions.emplace_back(i); converted_lc_columns.emplace_back(i);
} }
else else
{ {
@ -105,10 +103,12 @@ AggregatingSortedBlockInputStream::AggregatingSortedBlockInputStream(
} }
} }
if (!positions.empty()) result_header = header;
if (!converted_lc_columns.empty())
{ {
for (auto & input : children) for (auto & input : children)
input = std::make_shared<RemovingLowCardinalityBlockInputStream>(input, positions); input = std::make_shared<RemovingLowCardinalityBlockInputStream>(input, converted_lc_columns);
header = children.at(0)->getHeader(); header = children.at(0)->getHeader();
} }
@ -134,7 +134,15 @@ Block AggregatingSortedBlockInputStream::readImpl()
columns_to_aggregate[i] = typeid_cast<ColumnAggregateFunction *>(merged_columns[column_numbers_to_aggregate[i]].get()); columns_to_aggregate[i] = typeid_cast<ColumnAggregateFunction *>(merged_columns[column_numbers_to_aggregate[i]].get());
merge(merged_columns, queue_without_collation); merge(merged_columns, queue_without_collation);
return header.cloneWithColumns(std::move(merged_columns));
for (auto & pos : converted_lc_columns)
{
auto & from_type = header.getByPosition(pos).type;
auto & to_type = result_header.getByPosition(pos).type;
merged_columns[pos] = (*recursiveTypeConversion(std::move(merged_columns[pos]), from_type, to_type)).mutate();
}
return result_header.cloneWithColumns(std::move(merged_columns));
} }

5
dbms/src/DataStreams/AggregatingSortedBlockInputStream.h
View File

@ -31,6 +31,8 @@ public:
bool isSortedOutput() const override { return true; } bool isSortedOutput() const override { return true; }
Block getHeader() const override { return result_header; }
protected: protected:
/// Can return 1 more records than max_block_size. /// Can return 1 more records than max_block_size.
Block readImpl() override; Block readImpl() override;
@ -52,6 +54,9 @@ private:
SharedBlockRowRef current_key; /// The current primary key. SharedBlockRowRef current_key; /// The current primary key.
SharedBlockRowRef next_key; /// The primary key of the next row. SharedBlockRowRef next_key; /// The primary key of the next row.
Block result_header;
ColumnNumbers converted_lc_columns;
/** We support two different cursors - with Collation and without. /** We support two different cursors - with Collation and without.
* Templates are used instead of polymorphic SortCursor and calls to virtual functions. * Templates are used instead of polymorphic SortCursor and calls to virtual functions.
*/ */

2
dbms/src/DataStreams/BlockIO.h
View File

@ -66,6 +66,8 @@ struct BlockIO
finish_callback = rhs.finish_callback; finish_callback = rhs.finish_callback;
exception_callback = rhs.exception_callback; exception_callback = rhs.exception_callback;
null_format = rhs.null_format;
return *this; return *this;
} }
}; };

23
dbms/src/DataStreams/ConvertingBlockInputStream.cpp
View File

@ -5,7 +5,6 @@
#include <Common/quoteString.h> #include <Common/quoteString.h>
#include <Parsers/IAST.h> #include <Parsers/IAST.h>
namespace DB namespace DB
{ {
@ -65,8 +64,18 @@ ConvertingBlockInputStream::ConvertingBlockInputStream(
throw Exception("Cannot find column " + backQuote(res_elem.name) + " in source stream", throw Exception("Cannot find column " + backQuote(res_elem.name) + " in source stream",
ErrorCodes::THERE_IS_NO_COLUMN); ErrorCodes::THERE_IS_NO_COLUMN);
break; break;
case MatchColumnsMode::NameOrDefault:
if (input_header.has(res_elem.name))
conversion[result_col_num] = input_header.getPositionByName(res_elem.name);
else
conversion[result_col_num] = USE_DEFAULT;
break;
} }
if (conversion[result_col_num] == USE_DEFAULT)
continue;
const auto & src_elem = input_header.getByPosition(conversion[result_col_num]); const auto & src_elem = input_header.getByPosition(conversion[result_col_num]);
/// Check constants. /// Check constants.
@ -100,9 +109,18 @@ Block ConvertingBlockInputStream::readImpl()
Block res = header.cloneEmpty(); Block res = header.cloneEmpty();
for (size_t res_pos = 0, size = conversion.size(); res_pos < size; ++res_pos) for (size_t res_pos = 0, size = conversion.size(); res_pos < size; ++res_pos)
{ {
const auto & src_elem = src.getByPosition(conversion[res_pos]);
auto & res_elem = res.getByPosition(res_pos); auto & res_elem = res.getByPosition(res_pos);
if (conversion[res_pos] == USE_DEFAULT)
{
// Create a column with default values
auto column_with_defaults = res_elem.type->createColumn()->cloneResized(src.rows());
res_elem.column = std::move(column_with_defaults);
continue;
}
const auto & src_elem = src.getByPosition(conversion[res_pos]);
ColumnPtr converted = castColumnWithDiagnostic(src_elem, res_elem, context); ColumnPtr converted = castColumnWithDiagnostic(src_elem, res_elem, context);
if (isColumnConst(*src_elem.column) && !isColumnConst(*res_elem.column)) if (isColumnConst(*src_elem.column) && !isColumnConst(*res_elem.column))
@ -114,3 +132,4 @@ Block ConvertingBlockInputStream::readImpl()
} }
} }

5
dbms/src/DataStreams/ConvertingBlockInputStream.h
View File

@ -28,7 +28,9 @@ public:
/// Require same number of columns in source and result. Match columns by corresponding positions, regardless to names. /// Require same number of columns in source and result. Match columns by corresponding positions, regardless to names.
Position, Position,
/// Find columns in source by their names. Allow excessive columns in source. /// Find columns in source by their names. Allow excessive columns in source.
Name Name,
/// Find columns in source by their names if present else use the default. Allow excessive columns in source.
NameOrDefault
}; };
ConvertingBlockInputStream( ConvertingBlockInputStream(
@ -48,6 +50,7 @@ private:
/// How to construct result block. Position in source block, where to get each column. /// How to construct result block. Position in source block, where to get each column.
using Conversion = std::vector<size_t>; using Conversion = std::vector<size_t>;
const size_t USE_DEFAULT = static_cast<size_t>(-1);
Conversion conversion; Conversion conversion;
}; };

81
dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
View File

@ -18,7 +18,7 @@ namespace DB
{ {
PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
const String & database, const String & table, const StoragePtr & storage_, const StoragePtr & storage_,
const Context & context_, const ASTPtr & query_ptr_, bool no_destination) const Context & context_, const ASTPtr & query_ptr_, bool no_destination)
: storage(storage_), context(context_), query_ptr(query_ptr_) : storage(storage_), context(context_), query_ptr(query_ptr_)
{ {
@ -32,47 +32,44 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
/// Moreover, deduplication for AggregatingMergeTree children could produce false positives due to low size of inserting blocks /// Moreover, deduplication for AggregatingMergeTree children could produce false positives due to low size of inserting blocks
bool disable_deduplication_for_children = !no_destination && storage->supportsDeduplication(); bool disable_deduplication_for_children = !no_destination && storage->supportsDeduplication();
if (!table.empty()) auto table_id = storage->getStorageID();
Dependencies dependencies = context.getDependencies(table_id);
/// We need special context for materialized views insertions
if (!dependencies.empty())
{ {
Dependencies dependencies = context.getDependencies(database, table); views_context = std::make_unique<Context>(context);
// Do not deduplicate insertions into MV if the main insertion is Ok
if (disable_deduplication_for_children)
views_context->getSettingsRef().insert_deduplicate = false;
}
/// We need special context for materialized views insertions for (const auto & database_table : dependencies)
if (!dependencies.empty()) {
auto dependent_table = context.getTable(database_table);
ASTPtr query;
BlockOutputStreamPtr out;
if (auto * materialized_view = dynamic_cast<const StorageMaterializedView *>(dependent_table.get()))
{ {
views_context = std::make_unique<Context>(context); StoragePtr inner_table = materialized_view->getTargetTable();
// Do not deduplicate insertions into MV if the main insertion is Ok auto inner_table_id = inner_table->getStorageID();
if (disable_deduplication_for_children) query = materialized_view->getInnerQuery();
views_context->getSettingsRef().insert_deduplicate = false; std::unique_ptr<ASTInsertQuery> insert = std::make_unique<ASTInsertQuery>();
insert->database = inner_table_id.database_name;
insert->table = inner_table_id.table_name;
ASTPtr insert_query_ptr(insert.release());
InterpreterInsertQuery interpreter(insert_query_ptr, *views_context);
BlockIO io = interpreter.execute();
out = io.out;
} }
else if (dynamic_cast<const StorageLiveView *>(dependent_table.get()))
out = std::make_shared<PushingToViewsBlockOutputStream>(dependent_table, *views_context, ASTPtr(), true);
else
out = std::make_shared<PushingToViewsBlockOutputStream>(dependent_table, *views_context, ASTPtr());
for (const auto & database_table : dependencies) views.emplace_back(ViewInfo{std::move(query), database_table, std::move(out)});
{
auto dependent_table = context.getTable(database_table.first, database_table.second);
ASTPtr query;
BlockOutputStreamPtr out;
if (auto * materialized_view = dynamic_cast<const StorageMaterializedView *>(dependent_table.get()))
{
StoragePtr inner_table = materialized_view->getTargetTable();
query = materialized_view->getInnerQuery();
std::unique_ptr<ASTInsertQuery> insert = std::make_unique<ASTInsertQuery>();
insert->database = inner_table->getDatabaseName();
insert->table = inner_table->getTableName();
ASTPtr insert_query_ptr(insert.release());
InterpreterInsertQuery interpreter(insert_query_ptr, *views_context);
BlockIO io = interpreter.execute();
out = io.out;
}
else if (dynamic_cast<const StorageLiveView *>(dependent_table.get()))
out = std::make_shared<PushingToViewsBlockOutputStream>(
database_table.first, database_table.second, dependent_table, *views_context, ASTPtr(), true);
else
out = std::make_shared<PushingToViewsBlockOutputStream>(
database_table.first, database_table.second, dependent_table, *views_context, ASTPtr());
views.emplace_back(ViewInfo{std::move(query), database_table.first, database_table.second, std::move(out)});
}
} }
/* Do not push to destination table if the flag is set */ /* Do not push to destination table if the flag is set */
@ -161,7 +158,7 @@ void PushingToViewsBlockOutputStream::writePrefix()
} }
catch (Exception & ex) catch (Exception & ex)
{ {
ex.addMessage("while write prefix to view " + view.database + "." + view.table); ex.addMessage("while write prefix to view " + view.table_id.getNameForLogs());
throw; throw;
} }
} }
@ -180,7 +177,7 @@ void PushingToViewsBlockOutputStream::writeSuffix()
} }
catch (Exception & ex) catch (Exception & ex)
{ {
ex.addMessage("while write prefix to view " + view.database + "." + view.table); ex.addMessage("while write prefix to view " + view.table_id.getNameForLogs());
throw; throw;
} }
} }
@ -223,7 +220,7 @@ void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_n
/// InterpreterSelectQuery will do processing of alias columns. /// InterpreterSelectQuery will do processing of alias columns.
Context local_context = *views_context; Context local_context = *views_context;
local_context.addViewSource( local_context.addViewSource(
StorageValues::create(storage->getDatabaseName(), storage->getTableName(), storage->getColumns(), StorageValues::create(storage->getStorageID(), storage->getColumns(),
block)); block));
select.emplace(view.query, local_context, SelectQueryOptions()); select.emplace(view.query, local_context, SelectQueryOptions());
in = std::make_shared<MaterializingBlockInputStream>(select->execute().in); in = std::make_shared<MaterializingBlockInputStream>(select->execute().in);
@ -233,7 +230,7 @@ void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_n
/// and two-level aggregation is triggered). /// and two-level aggregation is triggered).
in = std::make_shared<SquashingBlockInputStream>( in = std::make_shared<SquashingBlockInputStream>(
in, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes); in, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes);
in = std::make_shared<ConvertingBlockInputStream>(context, in, view.out->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Name); in = std::make_shared<ConvertingBlockInputStream>(context, in, view.out->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::NameOrDefault);
} }
else else
in = std::make_shared<OneBlockInputStream>(block); in = std::make_shared<OneBlockInputStream>(block);
@ -250,7 +247,7 @@ void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_n
} }
catch (Exception & ex) catch (Exception & ex)
{ {
ex.addMessage("while pushing to view " + backQuoteIfNeed(view.database) + "." + backQuoteIfNeed(view.table)); ex.addMessage("while pushing to view " + view.table_id.getNameForLogs());
throw; throw;
} }
} }

6
dbms/src/DataStreams/PushingToViewsBlockOutputStream.h
View File

@ -17,8 +17,7 @@ class ReplicatedMergeTreeBlockOutputStream;
class PushingToViewsBlockOutputStream : public IBlockOutputStream class PushingToViewsBlockOutputStream : public IBlockOutputStream
{ {
public: public:
PushingToViewsBlockOutputStream( PushingToViewsBlockOutputStream(const StoragePtr & storage_,
const String & database, const String & table, const StoragePtr & storage_,
const Context & context_, const ASTPtr & query_ptr_, bool no_destination = false); const Context & context_, const ASTPtr & query_ptr_, bool no_destination = false);
Block getHeader() const override; Block getHeader() const override;
@ -39,8 +38,7 @@ private:
struct ViewInfo struct ViewInfo
{ {
ASTPtr query; ASTPtr query;
String database; StorageID table_id;
String table;
BlockOutputStreamPtr out; BlockOutputStreamPtr out;
}; };

2
dbms/src/DataTypes/DataTypeFactory.cpp
View File

@ -35,7 +35,7 @@ DataTypePtr DataTypeFactory::get(const ASTPtr & ast) const
if (const auto * func = ast->as<ASTFunction>()) if (const auto * func = ast->as<ASTFunction>())
{ {
if (func->parameters) if (func->parameters)
throw Exception("Data type cannot have multiple parenthesed parameters.", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE); throw Exception("Data type cannot have multiple parenthesized parameters.", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE);
return get(func->name, func->arguments); return get(func->name, func->arguments);
} }

4
dbms/src/DataTypes/DataTypeSet.h
View File

@ -1,6 +1,7 @@
#pragma once #pragma once
#include <DataTypes/IDataTypeDummy.h> #include <DataTypes/IDataTypeDummy.h>
#include <Columns/ColumnSet.h>
namespace DB namespace DB
@ -18,6 +19,9 @@ public:
bool equals(const IDataType & rhs) const override { return typeid(rhs) == typeid(*this); } bool equals(const IDataType & rhs) const override { return typeid(rhs) == typeid(*this); }
bool isParametric() const override { return true; } bool isParametric() const override { return true; }
// Used for expressions analysis.
MutableColumnPtr createColumn() const override { return ColumnSet::create(0, nullptr); }
// Used only for debugging, making it DUMPABLE // Used only for debugging, making it DUMPABLE
Field getDefault() const override { return Tuple(); } Field getDefault() const override { return Tuple(); }
}; };

4
dbms/src/DataTypes/tests/gtest_data_type_get_common_type.cpp
View File

@ -14,7 +14,7 @@ static bool operator==(const IDataType & left, const IDataType & right)
return left.equals(right); return left.equals(right);
} }
static std::ostream & operator<<(std::ostream & ostr, const IDataType & dt) std::ostream & operator<<(std::ostream & ostr, const IDataType & dt)
{ {
return ostr << dt.getName(); return ostr << dt.getName();
} }
@ -60,7 +60,7 @@ std::ostream & operator<<(std::ostream & ostr, const TypesTestCase & test_case)
class TypeTest : public ::testing::TestWithParam<TypesTestCase> class TypeTest : public ::testing::TestWithParam<TypesTestCase>
{ {
public: public:
void SetUp() void SetUp() override
{ {
const auto & p = GetParam(); const auto & p = GetParam();
from_types = typesFromString(p.from_types); from_types = typesFromString(p.from_types);

12
dbms/src/Databases/DatabaseDictionary.cpp
View File

@ -52,7 +52,7 @@ Tables DatabaseDictionary::listTables(const Context & context, const FilterByNam
auto dict_name = dict_ptr->getName(); auto dict_name = dict_ptr->getName();
const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
tables[dict_name] = StorageDictionary::create(getDatabaseName(), dict_name, ColumnsDescription{columns}, context, true, dict_name); tables[dict_name] = StorageDictionary::create(StorageID(getDatabaseName(), dict_name), ColumnsDescription{columns}, context, true, dict_name);
} }
} }
return tables; return tables;
@ -74,7 +74,7 @@ StoragePtr DatabaseDictionary::tryGetTable(
{ {
const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
return StorageDictionary::create(getDatabaseName(), table_name, ColumnsDescription{columns}, context, true, table_name); return StorageDictionary::create(StorageID(getDatabaseName(), table_name), ColumnsDescription{columns}, context, true, table_name);
} }
return {}; return {};
@ -109,11 +109,12 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context,
buffer << ") Engine = Dictionary(" << backQuoteIfNeed(table_name) << ")"; buffer << ") Engine = Dictionary(" << backQuoteIfNeed(table_name) << ")";
} }
auto settings = context.getSettingsRef();
ParserCreateQuery parser; ParserCreateQuery parser;
const char * pos = query.data(); const char * pos = query.data();
std::string error_message; std::string error_message;
auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message,
/* hilite = */ false, "", /* allow_multi_statements = */ false, 0); /* hilite = */ false, "", /* allow_multi_statements = */ false, 0, settings.max_parser_depth);
if (!ast && throw_on_error) if (!ast && throw_on_error)
throw Exception(error_message, ErrorCodes::SYNTAX_ERROR); throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
@ -121,15 +122,16 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context,
return ast; return ast;
} }
ASTPtr DatabaseDictionary::getCreateDatabaseQuery() const ASTPtr DatabaseDictionary::getCreateDatabaseQuery(const Context & context) const
{ {
String query; String query;
{ {
WriteBufferFromString buffer(query); WriteBufferFromString buffer(query);
buffer << "CREATE DATABASE " << backQuoteIfNeed(database_name) << " ENGINE = Dictionary"; buffer << "CREATE DATABASE " << backQuoteIfNeed(database_name) << " ENGINE = Dictionary";
} }
auto settings = context.getSettingsRef();
ParserCreateQuery parser; ParserCreateQuery parser;
return parseQuery(parser, query.data(), query.data() + query.size(), "", 0); return parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth);
} }
void DatabaseDictionary::shutdown() void DatabaseDictionary::shutdown()

2
dbms/src/Databases/DatabaseDictionary.h
View File

@ -41,7 +41,7 @@ public:
bool empty(const Context & context) const override; bool empty(const Context & context) const override;
ASTPtr getCreateDatabaseQuery() const override; ASTPtr getCreateDatabaseQuery(const Context & context) const override;
void shutdown() override; void shutdown() override;

4
dbms/src/Databases/DatabaseLazy.cpp
View File

@ -122,7 +122,7 @@ StoragePtr DatabaseLazy::tryGetTable(
std::lock_guard lock(mutex); std::lock_guard lock(mutex);
auto it = tables_cache.find(table_name); auto it = tables_cache.find(table_name);
if (it == tables_cache.end()) if (it == tables_cache.end())
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); return {};
if (it->second.table) if (it->second.table)
{ {
@ -230,7 +230,7 @@ StoragePtr DatabaseLazy::loadTable(const Context & context, const String & table
StoragePtr table; StoragePtr table;
Context context_copy(context); /// some tables can change context, but not LogTables Context context_copy(context); /// some tables can change context, but not LogTables
auto ast = parseQueryFromMetadata(table_metadata_path, /*throw_on_error*/ true, /*remove_empty*/false); auto ast = parseQueryFromMetadata(context, table_metadata_path, /*throw_on_error*/ true, /*remove_empty*/false);
if (ast) if (ast)
{ {
auto & ast_create = ast->as<const ASTCreateQuery &>(); auto & ast_create = ast->as<const ASTCreateQuery &>();

3
dbms/src/Databases/DatabaseMemory.cpp
View File

@ -9,6 +9,7 @@ namespace DB
DatabaseMemory::DatabaseMemory(const String & name_) DatabaseMemory::DatabaseMemory(const String & name_)
: DatabaseWithOwnTablesBase(name_, "DatabaseMemory(" + name_ + ")") : DatabaseWithOwnTablesBase(name_, "DatabaseMemory(" + name_ + ")")
, data_path("data/" + escapeForFileName(database_name) + "/")
{} {}
void DatabaseMemory::createTable( void DatabaseMemory::createTable(
@ -27,7 +28,7 @@ void DatabaseMemory::removeTable(
detachTable(table_name); detachTable(table_name);
} }
ASTPtr DatabaseMemory::getCreateDatabaseQuery() const ASTPtr DatabaseMemory::getCreateDatabaseQuery(const Context & /*context*/) const
{ {
auto create_query = std::make_shared<ASTCreateQuery>(); auto create_query = std::make_shared<ASTCreateQuery>();
create_query->database = database_name; create_query->database = database_name;

14
dbms/src/Databases/DatabaseMemory.h
View File

@ -1,6 +1,8 @@
#pragma once #pragma once
#include <Databases/DatabasesCommon.h> #include <Databases/DatabasesCommon.h>
#include <Common/escapeForFileName.h>
#include <Parsers/ASTCreateQuery.h>
namespace Poco { class Logger; } namespace Poco { class Logger; }
@ -31,7 +33,17 @@ public:
const Context & context, const Context & context,
const String & table_name) override; const String & table_name) override;
ASTPtr getCreateDatabaseQuery() const override; ASTPtr getCreateDatabaseQuery(const Context & /*context*/) const override;
/// DatabaseMemory allows to create tables, which store data on disk.
/// It's needed to create such tables in default database of clickhouse-local.
/// TODO May be it's better to use DiskMemory for such tables.
/// To save data on disk it's possible to explicitly CREATE DATABASE db ENGINE=Ordinary in clickhouse-local.
String getTableDataPath(const String & table_name) const override { return data_path + escapeForFileName(table_name) + "/"; }
String getTableDataPath(const ASTCreateQuery & query) const override { return getTableDataPath(query.table); }
private:
String data_path;
}; };
} }

12
dbms/src/Databases/DatabaseMySQL.cpp
View File

@ -132,9 +132,9 @@ static ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr
{ {
/// init create query. /// init create query.
auto table_id = storage->getStorageID();
create_table_query->table = storage->getTableName(); create_table_query->table = table_id.table_name;
create_table_query->database = storage->getDatabaseName(); create_table_query->database = table_id.database_name;
for (const auto & column_type_and_name : storage->getColumns().getOrdinary()) for (const auto & column_type_and_name : storage->getColumns().getOrdinary())
{ {
@ -144,7 +144,7 @@ static ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr
columns_expression_list->children.emplace_back(column_declaration); columns_expression_list->children.emplace_back(column_declaration);
} }
auto mysql_table_name = std::make_shared<ASTLiteral>(storage->getTableName()); auto mysql_table_name = std::make_shared<ASTLiteral>(table_id.table_name);
auto storage_engine_arguments = table_storage_define->as<ASTStorage>()->engine->arguments; auto storage_engine_arguments = table_storage_define->as<ASTStorage>()->engine->arguments;
storage_engine_arguments->children.insert(storage_engine_arguments->children.begin() + 2, mysql_table_name); storage_engine_arguments->children.insert(storage_engine_arguments->children.begin() + 2, mysql_table_name);
} }
@ -181,7 +181,7 @@ time_t DatabaseMySQL::getObjectMetadataModificationTime(const String & table_nam
return time_t(local_tables_cache[table_name].first); return time_t(local_tables_cache[table_name].first);
} }
ASTPtr DatabaseMySQL::getCreateDatabaseQuery() const ASTPtr DatabaseMySQL::getCreateDatabaseQuery(const Context & /*context*/) const
{ {
const auto & create_query = std::make_shared<ASTCreateQuery>(); const auto & create_query = std::make_shared<ASTCreateQuery>();
create_query->database = database_name; create_query->database = database_name;
@ -239,7 +239,7 @@ void DatabaseMySQL::fetchLatestTablesStructureIntoCache(const std::map<String, U
} }
local_tables_cache[table_name] = std::make_pair(table_modification_time, StorageMySQL::create( local_tables_cache[table_name] = std::make_pair(table_modification_time, StorageMySQL::create(
database_name, table_name, std::move(mysql_pool), database_name_in_mysql, table_name, StorageID(database_name, table_name), std::move(mysql_pool), database_name_in_mysql, table_name,
false, "", ColumnsDescription{columns_name_and_type}, ConstraintsDescription{}, global_context)); false, "", ColumnsDescription{columns_name_and_type}, ConstraintsDescription{}, global_context));
} }
} }

2
dbms/src/Databases/DatabaseMySQL.h
View File

@ -32,7 +32,7 @@ public:
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override; DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
ASTPtr getCreateDatabaseQuery() const override; ASTPtr getCreateDatabaseQuery(const Context & /*context*/) const override;
bool isTableExist(const Context & context, const String & name) const override; bool isTableExist(const Context & context, const String & name) const override;

29
dbms/src/Databases/DatabaseOnDisk.cpp
View File

@ -68,9 +68,12 @@ std::pair<String, StoragePtr> createTableFromAST(
ast_create_query.table, ast_create_query.table,
StorageFactory::instance().get( StorageFactory::instance().get(
ast_create_query, ast_create_query,
table_data_path_relative, ast_create_query.table, database_name, context, context.getGlobalContext(), table_data_path_relative,
columns, constraints, context,
true, has_force_restore_data_flag) context.getGlobalContext(),
columns,
constraints,
has_force_restore_data_flag)
}; };
} }
@ -211,7 +214,7 @@ void DatabaseOnDisk::renameTable(
if (!table) if (!table)
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
ASTPtr ast = parseQueryFromMetadata(getObjectMetadataPath(table_name)); ASTPtr ast = parseQueryFromMetadata(context, getObjectMetadataPath(table_name));
if (!ast) if (!ast)
throw Exception("There is no metadata file for table " + backQuote(table_name) + ".", ErrorCodes::FILE_DOESNT_EXIST); throw Exception("There is no metadata file for table " + backQuote(table_name) + ".", ErrorCodes::FILE_DOESNT_EXIST);
auto & create = ast->as<ASTCreateQuery &>(); auto & create = ast->as<ASTCreateQuery &>();
@ -244,7 +247,7 @@ ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(const Context & context, const St
ASTPtr ast; ASTPtr ast;
auto table_metadata_path = getObjectMetadataPath(table_name); auto table_metadata_path = getObjectMetadataPath(table_name);
ast = getCreateQueryFromMetadata(table_metadata_path, throw_on_error); ast = getCreateQueryFromMetadata(context, table_metadata_path, throw_on_error);
if (!ast && throw_on_error) if (!ast && throw_on_error)
{ {
/// Handle system.* tables for which there are no table.sql files. /// Handle system.* tables for which there are no table.sql files.
@ -260,20 +263,21 @@ ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(const Context & context, const St
return ast; return ast;
} }
ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const ASTPtr DatabaseOnDisk::getCreateDatabaseQuery(const Context & context) const
{ {
ASTPtr ast; ASTPtr ast;
auto settings = context.getSettingsRef();
auto metadata_dir_path = getMetadataPath(); auto metadata_dir_path = getMetadataPath();
auto database_metadata_path = metadata_dir_path.substr(0, metadata_dir_path.size() - 1) + ".sql"; auto database_metadata_path = metadata_dir_path.substr(0, metadata_dir_path.size() - 1) + ".sql";
ast = getCreateQueryFromMetadata(database_metadata_path, true); ast = getCreateQueryFromMetadata(context, database_metadata_path, true);
if (!ast) if (!ast)
{ {
/// Handle databases (such as default) for which there are no database.sql files. /// Handle databases (such as default) for which there are no database.sql files.
/// If database.sql doesn't exist, then engine is Ordinary /// If database.sql doesn't exist, then engine is Ordinary
String query = "CREATE DATABASE " + backQuoteIfNeed(getDatabaseName()) + " ENGINE = Ordinary"; String query = "CREATE DATABASE " + backQuoteIfNeed(getDatabaseName()) + " ENGINE = Ordinary";
ParserCreateQuery parser; ParserCreateQuery parser;
ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0); ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth);
} }
return ast; return ast;
@ -353,7 +357,7 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati
} }
} }
ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const String & metadata_file_path, bool throw_on_error /*= true*/, bool remove_empty /*= false*/) const ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const Context & context, const String & metadata_file_path, bool throw_on_error /*= true*/, bool remove_empty /*= false*/) const
{ {
String query; String query;
@ -380,11 +384,12 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const String & metadata_file_path,
return nullptr; return nullptr;
} }
auto settings = context.getSettingsRef();
ParserCreateQuery parser; ParserCreateQuery parser;
const char * pos = query.data(); const char * pos = query.data();
std::string error_message; std::string error_message;
auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false, auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false,
"in file " + getMetadataPath(), /* allow_multi_statements = */ false, 0); "in file " + getMetadataPath(), /* allow_multi_statements = */ false, 0, settings.max_parser_depth);
if (!ast && throw_on_error) if (!ast && throw_on_error)
throw Exception(error_message, ErrorCodes::SYNTAX_ERROR); throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
@ -394,9 +399,9 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const String & metadata_file_path,
return ast; return ast;
} }
ASTPtr DatabaseOnDisk::getCreateQueryFromMetadata(const String & database_metadata_path, bool throw_on_error) const ASTPtr DatabaseOnDisk::getCreateQueryFromMetadata(const Context & context, const String & database_metadata_path, bool throw_on_error) const
{ {
ASTPtr ast = parseQueryFromMetadata(database_metadata_path, throw_on_error); ASTPtr ast = parseQueryFromMetadata(context, database_metadata_path, throw_on_error);
if (ast) if (ast)
{ {

6
dbms/src/Databases/DatabaseOnDisk.h
View File

@ -52,7 +52,7 @@ public:
const String & to_table_name, const String & to_table_name,
TableStructureWriteLockHolder & lock) override; TableStructureWriteLockHolder & lock) override;
ASTPtr getCreateDatabaseQuery() const override; ASTPtr getCreateDatabaseQuery(const Context & context) const override;
void drop(const Context & context) override; void drop(const Context & context) override;
@ -74,8 +74,8 @@ protected:
const String & table_name, const String & table_name,
bool throw_on_error) const override; bool throw_on_error) const override;
ASTPtr parseQueryFromMetadata(const String & metadata_file_path, bool throw_on_error = true, bool remove_empty = false) const; ASTPtr parseQueryFromMetadata(const Context & context, const String & metadata_file_path, bool throw_on_error = true, bool remove_empty = false) const;
ASTPtr getCreateQueryFromMetadata(const String & metadata_path, bool throw_on_error) const; ASTPtr getCreateQueryFromMetadata(const Context & context, const String & metadata_path, bool throw_on_error) const;
const String metadata_path; const String metadata_path;

4
dbms/src/Databases/DatabaseOrdinary.cpp
View File

@ -122,12 +122,12 @@ void DatabaseOrdinary::loadStoredObjects(
FileNames file_names; FileNames file_names;
size_t total_dictionaries = 0; size_t total_dictionaries = 0;
iterateMetadataFiles(context, [&file_names, &total_dictionaries, this](const String & file_name) iterateMetadataFiles(context, [&context, &file_names, &total_dictionaries, this](const String & file_name)
{ {
String full_path = getMetadataPath() + file_name; String full_path = getMetadataPath() + file_name;
try try
{ {
auto ast = parseQueryFromMetadata(full_path, /*throw_on_error*/ true, /*remove_empty*/false); auto ast = parseQueryFromMetadata(context, full_path, /*throw_on_error*/ true, /*remove_empty*/false);
if (ast) if (ast)
{ {
auto * create_query = ast->as<ASTCreateQuery>(); auto * create_query = ast->as<ASTCreateQuery>();

4
dbms/src/Databases/DatabaseWithDictionaries.cpp
View File

@ -222,7 +222,7 @@ StoragePtr DatabaseWithDictionaries::getDictionaryStorage(const Context & contex
{ {
const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
return StorageDictionary::create(database_name, table_name, ColumnsDescription{columns}, context, true, dict_name); return StorageDictionary::create(StorageID(database_name, table_name), ColumnsDescription{columns}, context, true, dict_name);
} }
return nullptr; return nullptr;
} }
@ -235,7 +235,7 @@ ASTPtr DatabaseWithDictionaries::getCreateDictionaryQueryImpl(
ASTPtr ast; ASTPtr ast;
auto dictionary_metadata_path = getObjectMetadataPath(dictionary_name); auto dictionary_metadata_path = getObjectMetadataPath(dictionary_name);
ast = getCreateQueryFromMetadata(dictionary_metadata_path, throw_on_error); ast = getCreateQueryFromMetadata(context, dictionary_metadata_path, throw_on_error);
if (!ast && throw_on_error) if (!ast && throw_on_error)
{ {
/// Handle system.* tables for which there are no table.sql files. /// Handle system.* tables for which there are no table.sql files.

2
dbms/src/Databases/DatabaseWithDictionaries.h
View File

@ -32,7 +32,7 @@ public:
protected: protected:
DatabaseWithDictionaries(const String & name, const String & metadata_path_, const String & logger) DatabaseWithDictionaries(const String & name, const String & metadata_path_, const String & logger)
: DatabaseOnDisk(name, metadata_path_, logger) {} : DatabaseOnDisk(name, metadata_path_, logger) {}
void attachToExternalDictionariesLoader(Context & context); void attachToExternalDictionariesLoader(Context & context);
void detachFromExternalDictionariesLoader(); void detachFromExternalDictionariesLoader();

10
dbms/src/Databases/IDatabase.h
View File

@ -56,13 +56,13 @@ public:
DatabaseTablesSnapshotIterator(Tables && tables_) : tables(tables_), it(tables.begin()) {} DatabaseTablesSnapshotIterator(Tables && tables_) : tables(tables_), it(tables.begin()) {}
void next() { ++it; } void next() override { ++it; }
bool isValid() const { return it != tables.end(); } bool isValid() const override { return it != tables.end(); }
const String & name() const { return it->first; } const String & name() const override { return it->first; }
const StoragePtr & table() const { return it->second; } const StoragePtr & table() const override { return it->second; }
}; };
/// Copies list of dictionaries and iterates through such snapshot. /// Copies list of dictionaries and iterates through such snapshot.
@ -262,7 +262,7 @@ public:
} }
/// Get the CREATE DATABASE query for current database. /// Get the CREATE DATABASE query for current database.
virtual ASTPtr getCreateDatabaseQuery() const = 0; virtual ASTPtr getCreateDatabaseQuery(const Context & /*context*/) const = 0;
/// Get name of database. /// Get name of database.
String getDatabaseName() const { return database_name; } String getDatabaseName() const { return database_name; }

2
dbms/src/Dictionaries/DictionaryBlockInputStream.h
View File

@ -43,7 +43,7 @@ public:
using GetColumnsFunction = std::function<ColumnsWithTypeAndName(const Columns &, const std::vector<DictionaryAttribute> & attributes)>; using GetColumnsFunction = std::function<ColumnsWithTypeAndName(const Columns &, const std::vector<DictionaryAttribute> & attributes)>;
// Used to separate key columns format for storage and view. // Used to separate key columns format for storage and view.
// Calls get_key_columns_function to get key column for dictionary get fuction call // Calls get_key_columns_function to get key column for dictionary get function call
// and get_view_columns_function to get key representation. // and get_view_columns_function to get key representation.
// Now used in trie dictionary, where columns are stored as ip and mask, and are showed as string // Now used in trie dictionary, where columns are stored as ip and mask, and are showed as string
DictionaryBlockInputStream( DictionaryBlockInputStream(

1
dbms/src/Dictionaries/FileDictionarySource.cpp
View File

@ -3,6 +3,7 @@
#include <IO/ReadBufferFromFile.h> #include <IO/ReadBufferFromFile.h>
#include <Interpreters/Context.h> #include <Interpreters/Context.h>
#include <Poco/File.h> #include <Poco/File.h>
#include <Common/StringUtils/StringUtils.h>
#include "DictionarySourceFactory.h" #include "DictionarySourceFactory.h"
#include "DictionaryStructure.h" #include "DictionaryStructure.h"
#include "registerDictionaries.h" #include "registerDictionaries.h"

3
dbms/src/Disks/CMakeLists.txt
View File

@ -0,0 +1,3 @@
if (ENABLE_TESTS)
add_subdirectory (tests)
endif ()

Some files were not shown because too many files have changed in this diff Show More