thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-28 10:31:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into master

Browse Source
This commit is contained in:
alexey-milovidov 2018-11-23 23:41:38 +03:00 committed by GitHub
commit 507990d5f2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
346 changed files with 6322 additions and 2696 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.clang-format
View File

@ -1,6 +1,6 @@
BasedOnStyle: WebKit
Language: Cpp
AlignAfterOpenBracket: false
AlignAfterOpenBracket: AlwaysBreak
BreakBeforeBraces: Custom
BraceWrapping:
AfterClass: true
@ -25,7 +25,7 @@ Standard: Cpp11
PointerAlignment: Middle
MaxEmptyLinesToKeep: 2
KeepEmptyLinesAtTheStartOfBlocks: false
AllowShortFunctionsOnASingleLine: Empty
AllowShortFunctionsOnASingleLine: InlineOnly
AlwaysBreakTemplateDeclarations: true
IndentCaseLabels: true
SpaceAfterTemplateKeyword: true

1
.gitignore vendored
View File

@ -11,6 +11,7 @@
/build
/build_*
/build-*
/docs/build
/docs/edit
/docs/tools/venv/

3
.gitmodules vendored
View File

@ -52,3 +52,6 @@
[submodule "contrib/boost"]
path = contrib/boost
url = https://github.com/ClickHouse-Extras/boost-extra.git
[submodule "contrib/base64"]
path = contrib/base64
url = https://github.com/aklomp/base64.git

0
.travis.yml → .travis.yml.bak
View File

13
CHANGELOG.md
View File

@ -1,3 +1,16 @@
## ClickHouse release 18.14.15, 2018-11-21
### Bug fixes:
* The size of memory chunk was overestimated while deserializing the column of type `Array(String)` that leads to "Memory limit exceeded" errors. The issue appeared in version 18.12.13. [#3589](https://github.com/yandex/ClickHouse/issues/3589)
## ClickHouse release 18.14.14, 2018-11-20
### Bug fixes:
* Fixed `ON CLUSTER` queries when cluster configured as secure (flag `<secure>`). [#3599](https://github.com/yandex/ClickHouse/pull/3599)
### Build changes:
* Fixed problems (llvm-7 from system, macos) [#3582](https://github.com/yandex/ClickHouse/pull/3582)
## ClickHouse release 18.14.11, 2018-10-29
### Bug fixes:

13
CHANGELOG_RU.md
View File

@ -1,3 +1,16 @@
## ClickHouse release 18.14.15, 2018-11-21
### Исправления ошибок:
* При чтении столбцов типа `Array(String)`, размер требуемого куска памяти оценивался слишком большим, что приводило к исключению "Memory limit exceeded" при выполнении запроса. Ошибка появилась в версии 18.12.13. [#3589](https://github.com/yandex/ClickHouse/issues/3589)
## ClickHouse release 18.14.14, 2018-11-20
### Исправления ошибок:
* Исправлена работа запросов `ON CLUSTER` в случае, когда в конфигурации кластера включено шифрование (флаг `<secure>`). [#3599](https://github.com/yandex/ClickHouse/pull/3599)
### Улучшения процесса сборки ClickHouse:
* Испрпавлены проблемы сборки (llvm-7 из системы, macos) [#3582](https://github.com/yandex/ClickHouse/pull/3582)
## ClickHouse release 18.14.13, 2018-11-08
### Исправления ошибок:

1
CMakeLists.txt
View File

@ -262,6 +262,7 @@ include (cmake/find_llvm.cmake)
include (cmake/find_cpuid.cmake)
include (cmake/find_hdfs3.cmake)
include (cmake/find_consistent-hashing.cmake)
include (cmake/find_base64.cmake)
if (ENABLE_TESTS)
include (cmake/find_gtest.cmake)
endif ()

3
README.md
View File

@ -2,11 +2,10 @@
ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time.
🎤🥂 **ClickHouse Meetup in [Amsterdam on November 15](https://events.yandex.com/events/meetings/15-11-2018/)** 🍰🔥🐻
## Useful Links
* [Official website](https://clickhouse.yandex/) has quick high-level overview of ClickHouse on main page.
* [Tutorial](https://clickhouse.yandex/tutorial.html) shows how to set up and query small ClickHouse cluster.
* [Documentation](https://clickhouse.yandex/docs/en/) provides more in-depth information.
* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
* [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any.

12
cmake/find_base64.cmake Normal file
View File

@ -0,0 +1,12 @@
option (ENABLE_BASE64 "Enable base64" ON)
if (ENABLE_BASE64)
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64")
message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
else()
set (BASE64_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64/include)
set (BASE64_LIBRARY base64)
set (USE_BASE64 1)
endif()
endif ()

4
cmake/find_odbc.cmake
View File

@ -82,6 +82,10 @@ if (ENABLE_ODBC)
ODBC_INCLUDE_DIRECTORIES
ODBC_LIBRARIES)
if (USE_STATIC_LIBRARIES)
list(APPEND ODBC_LIBRARIES ${LTDL_LIBRARY})
endif ()
mark_as_advanced(ODBC_FOUND ODBC_LIBRARIES ODBC_INCLUDE_DIRECTORIES)
endif ()
endif ()

4
cmake/find_poco.cmake
View File

@ -93,8 +93,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
endif ()
if (OPENSSL_FOUND AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL))
set (Poco_NetSSL_LIBRARY PocoNetSSL)
set (Poco_Crypto_LIBRARY PocoCrypto)
set (Poco_NetSSL_LIBRARY PocoNetSSL ${OPENSSL_LIBRARIES})
set (Poco_Crypto_LIBRARY PocoCrypto ${OPENSSL_LIBRARIES})
endif ()
if (USE_STATIC_LIBRARIES AND USE_INTERNAL_ZLIB_LIBRARY)

1
cmake/lib_name.cmake
View File

@ -1,5 +1,4 @@
set(DIVIDE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libdivide)
set(CITYHASH_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/cityhash102/include)
set(COMMON_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/libs/libcommon/include ${ClickHouse_BINARY_DIR}/libs/libcommon/include)
set(DBMS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/dbms/src ${ClickHouse_BINARY_DIR}/dbms/src)
set(DOUBLE_CONVERSION_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/double-conversion)

3
cmake/print_include_directories.cmake
View File

@ -10,6 +10,9 @@ list(APPEND dirs ${dirs1})
get_property (dirs1 TARGET common PROPERTY INCLUDE_DIRECTORIES)
list(APPEND dirs ${dirs1})
get_property (dirs1 TARGET cityhash PROPERTY INCLUDE_DIRECTORIES)
list(APPEND dirs ${dirs1})
if (USE_INTERNAL_BOOST_LIBRARY)
get_property (dirs1 TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY} PROPERTY INCLUDE_DIRECTORIES)
list(APPEND dirs ${dirs1})

34
cmake/test_cpu.cmake
View File

@ -45,6 +45,38 @@ if (HAVE_SSE42)
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
endif ()
set (TEST_FLAG "-mssse3")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <tmmintrin.h>
int main() {
__m64 a = _mm_abs_pi8(__m64());
(void)a;
return 0;
}
" HAVE_SSSE3)
set (TEST_FLAG "-mavx")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <immintrin.h>
int main() {
auto a = _mm256_insert_epi8(__m256i(), 0, 0);
(void)a;
return 0;
}
" HAVE_AVX)
set (TEST_FLAG "-mavx2")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <immintrin.h>
int main() {
auto a = _mm256_add_epi16(__m256i(), __m256i());
(void)a;
return 0;
}
" HAVE_AVX2)
# gcc -dM -E -mpopcnt - < /dev/null | sort > gcc-dump-popcnt
#define __POPCNT__ 1
@ -65,5 +97,3 @@ if (HAVE_POPCNT AND NOT ARCH_AARCH64)
endif ()
cmake_pop_check_state ()
# TODO: add here sse3 test if you want use it

4
contrib/CMakeLists.txt vendored
View File

@ -200,3 +200,7 @@ if (USE_INTERNAL_HDFS3_LIBRARY)
endif ()
add_subdirectory(libhdfs3-cmake)
endif ()
if (USE_BASE64)
add_subdirectory (base64-cmake)
endif()

1
contrib/base64 vendored Submodule

@ -0,0 +1 @@
Subproject commit a27c565d1b6c676beaf297fe503c4518185666f7

1
contrib/base64-cmake/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
config.h

52
contrib/base64-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,52 @@
SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64)
set(base64_compile_instructions "")
LIST(LENGTH base64_compile_instructions 0)
macro(cast_to_bool var instruction)
if (HAVE_${var})
set(base64_${var} 1)
set(base64_${var}_opt ${instruction})
else()
set(base64_${var} 0)
endif()
endmacro()
cast_to_bool(SSSE3 "-mssse3")
cast_to_bool(SSE41 "-msse4.1")
cast_to_bool(SSE42 "-msse4.2")
cast_to_bool(AVX "-mavx")
cast_to_bool(AVX2 "-mavx2")
# write config.h file, to include it in application
file(READ config-header.tpl header)
file(WRITE config.h ${header})
file(APPEND config.h "#define HAVE_SSSE3 ${base64_SSSE3}\n")
file(APPEND config.h "#define HAVE_SSE41 ${base64_SSE41}\n")
file(APPEND config.h "#define HAVE_SSE42 ${base64_SSE42}\n")
file(APPEND config.h "#define HAVE_AVX ${base64_AVX}\n")
file(APPEND config.h "#define HAVE_AVX2 ${base64_AVX2}\n")
set(HAVE_FAST_UNALIGNED_ACCESS 0)
if (${base64_SSSE3} OR ${base64_SSE41} OR ${base64_SSE42} OR ${base64_AVX} OR ${base64_AVX2})
set(HAVE_FAST_UNALIGNED_ACCESS 1)
endif ()
file(APPEND config.h "#define HAVE_FAST_UNALIGNED_ACCESS " ${HAVE_FAST_UNALIGNED_ACCESS} "\n")
add_library(base64 ${LINK_MODE}
${LIBRARY_DIR}/lib/lib.c
${LIBRARY_DIR}/lib/codec_choose.c
${LIBRARY_DIR}/lib/arch/avx/codec.c
${LIBRARY_DIR}/lib/arch/avx2/codec.c
${LIBRARY_DIR}/lib/arch/generic/codec.c
${LIBRARY_DIR}/lib/arch/neon32/codec.c
${LIBRARY_DIR}/lib/arch/neon64/codec.c
${LIBRARY_DIR}/lib/arch/sse41/codec.c
${LIBRARY_DIR}/lib/arch/sse42/codec.c
${LIBRARY_DIR}/lib/arch/ssse3/codec.c
${LIBRARY_DIR}/lib/codecs.h
config.h)
target_compile_options(base64 PRIVATE ${base64_SSSE3_opt} ${base64_SSE41_opt} ${base64_SSE42_opt} ${base64_AVX_opt} ${base64_AVX2_opt})
target_include_directories(base64 PRIVATE ${LIBRARY_DIR}/include .)

2
contrib/base64-cmake/config-header.tpl Normal file
View File

@ -0,0 +1,2 @@
#define HAVE_NEON32 0
#define HAVE_NEON64 0

1
contrib/cityhash102/CMakeLists.txt
View File

@ -1,6 +1,5 @@
add_library(cityhash
src/city.cc
include/citycrc.h
include/city.h
src/config.h)

2
contrib/poco vendored

@ -1 +1 @@
Subproject commit 566162b324e0b73eb360a1cd38077c9287cc1106
Subproject commit 20c1d877773b6a672f1bbfe3290dfea42a117ed5

2
contrib/ssl vendored

@ -1 +1 @@
Subproject commit 919f6f1331d500bfdd26f8bbbf88e92c0119879b
Subproject commit dbbbcdbbd17785566f8f9c107b714f9e213d7293

51
dbms/CMakeLists.txt
View File

@ -119,7 +119,7 @@ endif ()
if (USE_EMBEDDED_COMPILER)
llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
target_link_libraries (dbms ${REQUIRED_LLVM_LIBRARIES})
target_link_libraries (dbms PRIVATE ${REQUIRED_LLVM_LIBRARIES})
target_include_directories (dbms SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
endif ()
@ -150,34 +150,49 @@ if (NOT ARCH_ARM AND CPUID_LIBRARY)
endif()
target_link_libraries (clickhouse_common_io
PUBLIC
common
PRIVATE
string_utils
widechar_width
${LINK_LIBRARIES_ONLY_ON_X86_64}
${LZ4_LIBRARY}
${ZSTD_LIBRARY}
${DOUBLE_CONVERSION_LIBRARIES}
pocoext
PUBLIC
${Poco_Net_LIBRARY}
${Poco_Util_LIBRARY}
${Poco_Foundation_LIBRARY}
${RE2_LIBRARY}
${RE2_ST_LIBRARY}
${CITYHASH_LIBRARIES}
PRIVATE
${ZLIB_LIBRARIES}
${EXECINFO_LIBRARY}
${ELF_LIBRARY}
PUBLIC
${Boost_SYSTEM_LIBRARY}
PRIVATE
apple_rt
${CMAKE_DL_LIBS}
${HDFS3_LIBRARY}
)
target_link_libraries (dbms
PRIVATE
clickhouse_parsers
clickhouse_common_config
PUBLIC
clickhouse_common_io
pocoext
PUBLIC
${MYSQLXX_LIBRARY}
${RE2_LIBRARY}
${RE2_ST_LIBRARY}
PRIVATE
${BTRIE_LIBRARIES}
${Boost_PROGRAM_OPTIONS_LIBRARY}
PUBLIC
${Boost_SYSTEM_LIBRARY}
)
if (NOT USE_INTERNAL_RE2_LIBRARY)
@ -194,8 +209,8 @@ if (Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
endif()
if (USE_POCO_SQLODBC)
target_link_libraries (clickhouse_common_io ${Poco_SQL_LIBRARY})
target_link_libraries (dbms ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
target_link_libraries (clickhouse_common_io PRIVATE ${Poco_SQL_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
if (NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIR})
target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR} PUBLIC ${Poco_SQL_INCLUDE_DIR})
@ -209,48 +224,44 @@ if (Poco_Data_FOUND)
endif()
if (USE_POCO_DATAODBC)
target_link_libraries (clickhouse_common_io ${Poco_Data_LIBRARY})
target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
target_link_libraries (clickhouse_common_io PRIVATE ${Poco_Data_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_DataODBC_LIBRARY})
if (NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
endif()
endif()
if (USE_POCO_MONGODB)
target_link_libraries (dbms ${Poco_MongoDB_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_MongoDB_LIBRARY})
endif()
if (USE_POCO_NETSSL)
target_link_libraries (clickhouse_common_io ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
target_link_libraries (dbms ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
target_link_libraries (clickhouse_common_io PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
endif()
target_link_libraries (dbms ${Poco_Foundation_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_Foundation_LIBRARY})
if (USE_ICU)
target_link_libraries (dbms ${ICU_LIBS})
target_link_libraries (dbms PRIVATE ${ICU_LIBS})
target_include_directories (dbms SYSTEM PRIVATE ${ICU_INCLUDE_DIR})
endif ()
if (USE_CAPNP)
target_link_libraries (dbms ${CAPNP_LIBRARY})
target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARY})
if (NOT USE_INTERNAL_CAPNP_LIBRARY)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR})
endif ()
endif ()
if (USE_RDKAFKA)
target_link_libraries (dbms ${RDKAFKA_LIBRARY})
target_link_libraries (dbms PRIVATE ${RDKAFKA_LIBRARY})
if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
endif ()
endif ()
target_link_libraries(dbms ${OPENSSL_CRYPTO_LIBRARY})
target_link_libraries (dbms
Threads::Threads
)
target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY} Threads::Threads)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
@ -286,6 +297,6 @@ if (ENABLE_TESTS)
# attach all dbms gtest sources
grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms dbms_gtest_sources)
add_executable(unit_tests_dbms ${dbms_gtest_sources})
target_link_libraries(unit_tests_dbms gtest_main dbms)
target_link_libraries(unit_tests_dbms PRIVATE gtest_main dbms clickhouse_common_zookeeper)
add_check(unit_tests_dbms)
endif ()

2
dbms/cmake/version.cmake
View File

@ -22,3 +22,5 @@ endif ()
set (VERSION_NAME "${PROJECT_NAME}" CACHE STRING "")
set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}" CACHE STRING "")
set (VERSION_SO "${VERSION_STRING}" CACHE STRING "")
math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000")

26
dbms/programs/CMakeLists.txt
View File

@ -48,45 +48,45 @@ else ()
link_directories (${LLVM_LIBRARY_DIRS})
endif ()
add_executable (clickhouse main.cpp)
target_link_libraries (clickhouse clickhouse_common_io)
target_link_libraries (clickhouse PRIVATE clickhouse_common_io)
target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
if (ENABLE_CLICKHOUSE_SERVER)
target_link_libraries (clickhouse clickhouse-server-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-server-lib)
endif ()
if (ENABLE_CLICKHOUSE_CLIENT)
target_link_libraries (clickhouse clickhouse-client-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-client-lib)
endif ()
if (ENABLE_CLICKHOUSE_LOCAL)
target_link_libraries (clickhouse clickhouse-local-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-local-lib)
endif ()
if (ENABLE_CLICKHOUSE_BENCHMARK)
target_link_libraries (clickhouse clickhouse-benchmark-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-benchmark-lib)
endif ()
if (ENABLE_CLICKHOUSE_PERFORMANCE)
target_link_libraries (clickhouse clickhouse-performance-test-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-performance-test-lib)
endif ()
if (ENABLE_CLICKHOUSE_COPIER)
target_link_libraries (clickhouse clickhouse-copier-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-copier-lib)
endif ()
if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
target_link_libraries (clickhouse clickhouse-extract-from-config-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-extract-from-config-lib)
endif ()
if (ENABLE_CLICKHOUSE_COMPRESSOR)
target_link_libraries (clickhouse clickhouse-compressor-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-compressor-lib)
endif ()
if (ENABLE_CLICKHOUSE_FORMAT)
target_link_libraries (clickhouse clickhouse-format-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-format-lib)
endif ()
if (ENABLE_CLICKHOUSE_OBFUSCATOR)
target_link_libraries (clickhouse clickhouse-obfuscator-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-obfuscator-lib)
endif ()
if (USE_EMBEDDED_COMPILER)
target_link_libraries (clickhouse clickhouse-compiler-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-compiler-lib)
endif ()
if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
target_link_libraries (clickhouse clickhouse-odbc-bridge-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-odbc-bridge-lib)
endif()
set (CLICKHOUSE_BUNDLE)

6
dbms/programs/benchmark/Benchmark.cpp
View File

@ -42,10 +42,8 @@ namespace DB
namespace ErrorCodes
{
extern const int POCO_EXCEPTION;
extern const int STD_EXCEPTION;
extern const int UNKNOWN_EXCEPTION;
extern const int BAD_ARGUMENTS;
extern const int EMPTY_DATA_PASSED;
}
class Benchmark
@ -170,7 +168,7 @@ private:
}
if (queries.empty())
throw Exception("Empty list of queries.");
throw Exception("Empty list of queries.", ErrorCodes::EMPTY_DATA_PASSED);
std::cerr << "Loaded " << queries.size() << " queries.\n";
}

4
dbms/programs/benchmark/CMakeLists.txt
View File

@ -1,8 +1,8 @@
add_library (clickhouse-benchmark-lib ${LINK_MODE} Benchmark.cpp)
target_link_libraries (clickhouse-benchmark-lib clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-benchmark-lib PRIVATE clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-benchmark clickhouse-benchmark.cpp)
target_link_libraries (clickhouse-benchmark clickhouse-benchmark-lib clickhouse_aggregate_functions)
target_link_libraries (clickhouse-benchmark PRIVATE clickhouse-benchmark-lib clickhouse_aggregate_functions)
endif ()

4
dbms/programs/clang/CMakeLists.txt
View File

@ -6,9 +6,9 @@ if (CLICKHOUSE_SPLIT_BINARY)
if (USE_EMBEDDED_COMPILER)
link_directories (${LLVM_LIBRARY_DIRS})
add_executable (clickhouse-clang clickhouse-clang.cpp)
target_link_libraries (clickhouse-clang clickhouse-compiler-lib)
target_link_libraries (clickhouse-clang PRIVATE clickhouse-compiler-lib)
add_executable (clickhouse-lld clickhouse-lld.cpp)
target_link_libraries (clickhouse-lld clickhouse-compiler-lib)
target_link_libraries (clickhouse-lld PRIVATE clickhouse-compiler-lib)
install (TARGETS clickhouse-clang clickhouse-lld RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
endif ()
endif ()

5
dbms/programs/client/CMakeLists.txt
View File

@ -1,13 +1,12 @@
add_library (clickhouse-client-lib ${LINK_MODE} Client.cpp)
target_link_libraries (clickhouse-client-lib clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-client-lib PRIVATE clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (READLINE_INCLUDE_DIR)
target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
endif ()
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-client clickhouse-client.cpp)
target_link_libraries (clickhouse-client clickhouse-client-lib)
target_link_libraries (clickhouse-client PRIVATE clickhouse-client-lib)
endif ()
install (FILES clickhouse-client.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-client COMPONENT clickhouse-client RENAME config.xml)

3
dbms/programs/client/Client.cpp
View File

@ -87,9 +87,6 @@ namespace DB
namespace ErrorCodes
{
extern const int POCO_EXCEPTION;
extern const int STD_EXCEPTION;
extern const int UNKNOWN_EXCEPTION;
extern const int NETWORK_ERROR;
extern const int NO_DATA_TO_INSERT;
extern const int BAD_ARGUMENTS;

2
dbms/programs/client/Suggest.h
View File

@ -184,7 +184,7 @@ public:
}
catch (...)
{
std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false) << "\n";
std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n";
}
/// Note that keyword suggestions are available even if we cannot load data from server.

4
dbms/programs/compressor/CMakeLists.txt
View File

@ -1,8 +1,8 @@
add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp)
target_link_libraries (clickhouse-compressor-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
# Also in utils
add_executable (clickhouse-compressor clickhouse-compressor.cpp)
target_link_libraries (clickhouse-compressor clickhouse-compressor-lib)
target_link_libraries (clickhouse-compressor PRIVATE clickhouse-compressor-lib)
endif ()

2
dbms/programs/copier/CMakeLists.txt
View File

@ -1,5 +1,5 @@
add_library (clickhouse-copier-lib ${LINK_MODE} ClusterCopier.cpp)
target_link_libraries (clickhouse-copier-lib clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions)
target_link_libraries (clickhouse-copier-lib PRIVATE clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions daemon)
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-copier clickhouse-copier.cpp)

4
dbms/programs/extract-from-config/CMakeLists.txt
View File

@ -1,7 +1,7 @@
add_library (clickhouse-extract-from-config-lib ${LINK_MODE} ExtractFromConfig.cpp)
target_link_libraries (clickhouse-extract-from-config-lib clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-extract-from-config-lib PRIVATE clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-extract-from-config clickhouse-extract-from-config.cpp)
target_link_libraries (clickhouse-extract-from-config clickhouse-extract-from-config-lib)
target_link_libraries (clickhouse-extract-from-config PRIVATE clickhouse-extract-from-config-lib)
endif ()

4
dbms/programs/format/CMakeLists.txt
View File

@ -1,6 +1,6 @@
add_library (clickhouse-format-lib ${LINK_MODE} Format.cpp)
target_link_libraries (clickhouse-format-lib dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-format-lib PRIVATE dbms clickhouse_common_io clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-format clickhouse-format.cpp)
target_link_libraries (clickhouse-format clickhouse-format-lib)
target_link_libraries (clickhouse-format PRIVATE clickhouse-format-lib)
endif ()

4
dbms/programs/local/CMakeLists.txt
View File

@ -1,7 +1,7 @@
add_library (clickhouse-local-lib ${LINK_MODE} LocalServer.cpp)
target_link_libraries (clickhouse-local-lib clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-local-lib PRIVATE clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-local clickhouse-local.cpp)
target_link_libraries (clickhouse-local clickhouse-local-lib)
target_link_libraries (clickhouse-local PRIVATE clickhouse-local-lib)
endif ()

4
dbms/programs/obfuscator/CMakeLists.txt
View File

@ -1,8 +1,8 @@
add_library (clickhouse-obfuscator-lib ${LINK_MODE} Obfuscator.cpp)
target_link_libraries (clickhouse-obfuscator-lib dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-obfuscator-lib PRIVATE dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-obfuscator clickhouse-obfuscator.cpp)
set_target_properties(clickhouse-obfuscator PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
target_link_libraries (clickhouse-obfuscator clickhouse-obfuscator-lib)
target_link_libraries (clickhouse-obfuscator PRIVATE clickhouse-obfuscator-lib)
endif ()

12
dbms/programs/odbc-bridge/CMakeLists.txt
View File

@ -9,23 +9,23 @@ add_library (clickhouse-odbc-bridge-lib ${LINK_MODE}
validateODBCConnectionString.cpp
)
target_link_libraries (clickhouse-odbc-bridge-lib clickhouse_common_io daemon dbms)
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE clickhouse_common_io daemon dbms)
target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
if (USE_POCO_SQLODBC)
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQLODBC_LIBRARY})
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQLODBC_LIBRARY})
target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR})
endif ()
if (Poco_SQL_FOUND)
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQL_LIBRARY})
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQL_LIBRARY})
endif ()
if (USE_POCO_DATAODBC)
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_DataODBC_LIBRARY})
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_DataODBC_LIBRARY})
target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
endif()
if (Poco_Data_FOUND)
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_Data_LIBRARY})
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_Data_LIBRARY})
endif ()
@ -35,5 +35,5 @@ endif ()
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-odbc-bridge odbc-bridge.cpp)
target_link_libraries (clickhouse-odbc-bridge clickhouse-odbc-bridge-lib)
target_link_libraries (clickhouse-odbc-bridge PRIVATE clickhouse-odbc-bridge-lib)
endif ()

2
dbms/programs/odbc-bridge/tests/CMakeLists.txt
View File

@ -1,2 +1,2 @@
add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp)
target_link_libraries (validate-odbc-connection-string clickhouse-odbc-bridge-lib)
target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse-odbc-bridge-lib clickhouse_common_io)

4
dbms/programs/performance-test/CMakeLists.txt
View File

@ -1,8 +1,8 @@
add_library (clickhouse-performance-test-lib ${LINK_MODE} PerformanceTest.cpp)
target_link_libraries (clickhouse-performance-test-lib clickhouse_common_io dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-performance-test-lib PRIVATE dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-performance-test clickhouse-performance-test.cpp)
target_link_libraries (clickhouse-performance-test clickhouse-performance-test-lib dbms)
target_link_libraries (clickhouse-performance-test PRIVATE clickhouse-performance-test-lib)
endif ()

46
dbms/programs/performance-test/PerformanceTest.cpp
View File

@ -49,10 +49,10 @@ namespace DB
{
namespace ErrorCodes
{
extern const int POCO_EXCEPTION;
extern const int STD_EXCEPTION;
extern const int UNKNOWN_EXCEPTION;
extern const int NOT_IMPLEMENTED;
extern const int LOGICAL_ERROR;
extern const int BAD_ARGUMENTS;
extern const int FILE_DOESNT_EXIST;
}
static String pad(size_t padding)
@ -156,7 +156,7 @@ struct StopConditionsSet
else if (key == "average_speed_not_changing_for_ms")
average_speed_not_changing_for_ms.value = stop_conditions_view->getUInt64(key);
else
throw DB::Exception("Met unkown stop condition: " + key);
throw DB::Exception("Met unkown stop condition: " + key, DB::ErrorCodes::LOGICAL_ERROR);
++initialized_count;
}
@ -521,7 +521,7 @@ public:
{
if (input_files.size() < 1)
{
throw DB::Exception("No tests were specified", 0);
throw DB::Exception("No tests were specified", DB::ErrorCodes::BAD_ARGUMENTS);
}
std::string name;
@ -694,7 +694,7 @@ private:
size_t ram_size_needed = config->getUInt64("preconditions.ram_size");
size_t actual_ram = getMemoryAmount();
if (!actual_ram)
throw DB::Exception("ram_size precondition not available on this platform", ErrorCodes::NOT_IMPLEMENTED);
throw DB::Exception("ram_size precondition not available on this platform", DB::ErrorCodes::NOT_IMPLEMENTED);
if (ram_size_needed > actual_ram)
{
@ -868,12 +868,12 @@ private:
if (!test_config->has("query") && !test_config->has("query_file"))
{
throw DB::Exception("Missing query fields in test's config: " + test_name);
throw DB::Exception("Missing query fields in test's config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
}
if (test_config->has("query") && test_config->has("query_file"))
{
throw DB::Exception("Found both query and query_file fields. Choose only one");
throw DB::Exception("Found both query and query_file fields. Choose only one", DB::ErrorCodes::BAD_ARGUMENTS);
}
if (test_config->has("query"))
@ -885,7 +885,7 @@ private:
{
const String filename = test_config->getString("query_file");
if (filename.empty())
throw DB::Exception("Empty file name");
throw DB::Exception("Empty file name", DB::ErrorCodes::BAD_ARGUMENTS);
bool tsv = fs::path(filename).extension().string() == ".tsv";
@ -909,7 +909,7 @@ private:
if (queries.empty())
{
throw DB::Exception("Did not find any query to execute: " + test_name);
throw DB::Exception("Did not find any query to execute: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
}
if (test_config->has("substitutions"))
@ -929,7 +929,7 @@ private:
if (!test_config->has("type"))
{
throw DB::Exception("Missing type property in config: " + test_name);
throw DB::Exception("Missing type property in config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
}
String config_exec_type = test_config->getString("type");
@ -938,7 +938,7 @@ private:
else if (config_exec_type == "once")
exec_type = ExecutionType::Once;
else
throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name);
throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
times_to_run = test_config->getUInt("times_to_run", 1);
@ -951,7 +951,7 @@ private:
}
if (stop_conditions_template.empty())
throw DB::Exception("No termination conditions were found in config");
throw DB::Exception("No termination conditions were found in config", DB::ErrorCodes::BAD_ARGUMENTS);
for (size_t i = 0; i < times_to_run * queries.size(); ++i)
stop_conditions_by_run.push_back(stop_conditions_template);
@ -978,7 +978,7 @@ private:
else
{
if (lite_output)
throw DB::Exception("Specify main_metric for lite output");
throw DB::Exception("Specify main_metric for lite output", DB::ErrorCodes::BAD_ARGUMENTS);
}
if (metrics.size() > 0)
@ -1023,22 +1023,14 @@ private:
if (exec_type == ExecutionType::Loop)
{
for (const String & metric : metrics)
{
if (std::find(non_loop_metrics.begin(), non_loop_metrics.end(), metric) != non_loop_metrics.end())
{
throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")");
}
}
throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS);
}
else
{
for (const String & metric : metrics)
{
if (std::find(loop_metrics.begin(), loop_metrics.end(), metric) != loop_metrics.end())
{
throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")");
}
}
throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS);
}
}
@ -1439,7 +1431,7 @@ try
if (input_files.empty())
{
std::cerr << std::endl;
throw DB::Exception("Did not find any xml files");
throw DB::Exception("Did not find any xml files", DB::ErrorCodes::BAD_ARGUMENTS);
}
else
std::cerr << " found " << input_files.size() << " files." << std::endl;
@ -1454,7 +1446,7 @@ try
fs::path file(filename);
if (!fs::exists(file))
throw DB::Exception("File '" + filename + "' does not exist");
throw DB::Exception("File '" + filename + "' does not exist", DB::ErrorCodes::FILE_DOESNT_EXIST);
if (fs::is_directory(file))
{
@ -1463,7 +1455,7 @@ try
else
{
if (file.extension().string() != ".xml")
throw DB::Exception("File '" + filename + "' does not have .xml extension");
throw DB::Exception("File '" + filename + "' does not have .xml extension", DB::ErrorCodes::BAD_ARGUMENTS);
collected_files.push_back(filename);
}
}

8
dbms/programs/server/CMakeLists.txt
View File

@ -10,12 +10,16 @@ add_library (clickhouse-server-lib ${LINK_MODE}
TCPHandler.cpp
)
target_link_libraries (clickhouse-server-lib clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions)
target_link_libraries (clickhouse-server-lib PRIVATE clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY})
if (USE_POCO_NETSSL)
target_link_libraries (clickhouse-server-lib PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
endif ()
target_include_directories (clickhouse-server-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-server clickhouse-server.cpp)
target_link_libraries (clickhouse-server clickhouse-server-lib)
target_link_libraries (clickhouse-server PRIVATE clickhouse-server-lib)
install (TARGETS clickhouse-server ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
endif ()

1
dbms/programs/server/HTTPHandler.cpp
View File

@ -270,7 +270,6 @@ void HTTPHandler::processQuery(
std::string query_id = params.get("query_id", "");
context.setUser(user, password, request.clientAddress(), quota_key);
context.setCurrentQueryId(query_id);
CurrentThread::attachQueryContext(context);
/// The user could specify session identifier and session timeout.
/// It allows to modify settings, create temporary tables and reuse them in subsequent requests.

3
dbms/programs/server/InterserverIOHTTPHandler.cpp
View File

@ -19,9 +19,6 @@ namespace DB
namespace ErrorCodes
{
extern const int ABORTED;
extern const int POCO_EXCEPTION;
extern const int STD_EXCEPTION;
extern const int UNKNOWN_EXCEPTION;
extern const int TOO_MANY_SIMULTANEOUS_QUERIES;
}

8
dbms/programs/server/Server.cpp
View File

@ -55,6 +55,7 @@
namespace CurrentMetrics
{
extern const Metric Revision;
extern const Metric VersionInteger;
}
namespace DB
@ -66,6 +67,8 @@ namespace ErrorCodes
extern const int SUPPORT_IS_DISABLED;
extern const int ARGUMENT_OUT_OF_BOUND;
extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
extern const int INVALID_CONFIG_PARAMETER;
extern const int SYSTEM_ERROR;
}
@ -73,7 +76,7 @@ static std::string getCanonicalPath(std::string && path)
{
Poco::trimInPlace(path);
if (path.empty())
throw Exception("path configuration parameter is empty");
throw Exception("path configuration parameter is empty", ErrorCodes::INVALID_CONFIG_PARAMETER);
if (path.back() != '/')
path += '/';
return std::move(path);
@ -108,6 +111,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
registerStorages();
CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get());
CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
/** Context contains all that query execution is dependent:
* settings, available functions, data types, aggregate functions, databases...
@ -141,7 +145,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
{
LOG_TRACE(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds.");
if (0 != mlockall(MCL_CURRENT))
LOG_WARNING(log, "Failed mlockall: " + errnoToString());
LOG_WARNING(log, "Failed mlockall: " + errnoToString(ErrorCodes::SYSTEM_ERROR));
else
LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed");
}

2
dbms/programs/server/TCPHandler.cpp
View File

@ -718,7 +718,7 @@ bool TCPHandler::receiveData()
{
NamesAndTypesList columns = block.getNamesAndTypesList();
storage = StorageMemory::create(external_table_name,
ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, ColumnComments{}});
storage->startup();
query_context.addExternalTable(external_table_name, storage);
}

14
dbms/programs/server/config.xml
View File

@ -164,6 +164,20 @@
</replica>
</shard>
</test_shard_localhost>
<test_cluster_two_shards_localhost>
<shard>
<replica>
<host>localhost</host>
<port>9000</port>
</replica>
</shard>
<shard>
<replica>
<host>localhost</host>
<port>9000</port>
</replica>
</shard>
</test_cluster_two_shards_localhost>
<test_shard_localhost_secure>
<shard>
<replica>

2
dbms/scripts/gen-bias-data.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/python3.4
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import sys

2
dbms/src/AggregateFunctions/AggregateFunctionAvg.cpp
View File

@ -12,7 +12,7 @@ namespace
template <typename T>
struct Avg
{
using FieldType = std::conditional_t<IsDecimalNumber<T>, Decimal128, typename NearestFieldType<T>::Type>;
using FieldType = std::conditional_t<IsDecimalNumber<T>, Decimal128, NearestFieldType<T>>;
using Function = AggregateFunctionAvg<T, AggregateFunctionAvgData<FieldType>>;
};

2
dbms/src/AggregateFunctions/AggregateFunctionSum.cpp
View File

@ -14,7 +14,7 @@ template <typename T>
struct SumSimple
{
/// @note It uses slow Decimal128 (cause we need such a variant). sumWithOverflow is faster for Decimal32/64
using ResultType = std::conditional_t<IsDecimalNumber<T>, Decimal128, typename NearestFieldType<T>::Type>;
using ResultType = std::conditional_t<IsDecimalNumber<T>, Decimal128, NearestFieldType<T>>;
using AggregateDataType = AggregateFunctionSumData<ResultType>;
using Function = AggregateFunctionSum<T, ResultType, AggregateDataType>;
};

2
dbms/src/AggregateFunctions/AggregateFunctionSumMap.h
View File

@ -52,7 +52,7 @@ struct AggregateFunctionSumMapData
template <typename T>
class AggregateFunctionSumMap final : public IAggregateFunctionDataHelper<
AggregateFunctionSumMapData<typename NearestFieldType<T>::Type>, AggregateFunctionSumMap<T>>
AggregateFunctionSumMapData<NearestFieldType<T>>, AggregateFunctionSumMap<T>>
{
private:
using ColVecType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;

3
dbms/src/AggregateFunctions/AggregateFunctionUniq.cpp
View File

@ -130,9 +130,6 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory & factory)
factory.registerFunction("uniqExact",
createAggregateFunctionUniq<true, AggregateFunctionUniqExactData, AggregateFunctionUniqExactData<String>>);
factory.registerFunction("uniqCombined",
createAggregateFunctionUniq<false, AggregateFunctionUniqCombinedData, AggregateFunctionUniqCombinedData<UInt64>>);
}
}

88
dbms/src/AggregateFunctions/AggregateFunctionUniq.h
View File

@ -22,7 +22,6 @@
#include <Common/typeid_cast.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <AggregateFunctions/UniqCombinedBiasData.h>
#include <AggregateFunctions/UniqVariadicHash.h>
@ -124,46 +123,6 @@ struct AggregateFunctionUniqExactData<String>
static String getName() { return "uniqExact"; }
};
template <typename T>
struct AggregateFunctionUniqCombinedData
{
using Key = UInt32;
using Set = CombinedCardinalityEstimator<
Key,
HashSet<Key, TrivialHash, HashTableGrower<>>,
16,
14,
17,
TrivialHash,
UInt32,
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
HyperLogLogMode::FullFeatured>;
Set set;
static String getName() { return "uniqCombined"; }
};
template <>
struct AggregateFunctionUniqCombinedData<String>
{
using Key = UInt64;
using Set = CombinedCardinalityEstimator<
Key,
HashSet<Key, TrivialHash, HashTableGrower<>>,
16,
14,
17,
TrivialHash,
UInt64,
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
HyperLogLogMode::FullFeatured>;
Set set;
static String getName() { return "uniqCombined"; }
};
namespace detail
{
@ -199,39 +158,6 @@ template <> struct AggregateFunctionUniqTraits<Float64>
}
};
/** Hash function for uniqCombined.
*/
template <typename T> struct AggregateFunctionUniqCombinedTraits
{
static UInt32 hash(T x) { return static_cast<UInt32>(intHash64(x)); }
};
template <> struct AggregateFunctionUniqCombinedTraits<UInt128>
{
static UInt32 hash(UInt128 x)
{
return sipHash64(x);
}
};
template <> struct AggregateFunctionUniqCombinedTraits<Float32>
{
static UInt32 hash(Float32 x)
{
UInt64 res = ext::bit_cast<UInt64>(x);
return static_cast<UInt32>(intHash64(res));
}
};
template <> struct AggregateFunctionUniqCombinedTraits<Float64>
{
static UInt32 hash(Float64 x)
{
UInt64 res = ext::bit_cast<UInt64>(x);
return static_cast<UInt32>(intHash64(res));
}
};
/** The structure for the delegation work to add one element to the `uniq` aggregate functions.
* Used for partial specialization to add strings.
@ -255,19 +181,6 @@ struct OneAdder
data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
}
}
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqCombinedData<T>>)
{
if constexpr (!std::is_same_v<T, String>)
{
const auto & value = static_cast<const ColumnVector<T> &>(column).getData()[row_num];
data.set.insert(AggregateFunctionUniqCombinedTraits<T>::hash(value));
}
else
{
StringRef value = column.getDataAt(row_num);
data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
}
}
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqExactData<T>>)
{
if constexpr (!std::is_same_v<T, String>)
@ -387,5 +300,4 @@ public:
const char * getHeaderFilePath() const override { return __FILE__; }
};
}

127
dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp Normal file
View File

@ -0,0 +1,127 @@
#include <AggregateFunctions/AggregateFunctionUniqCombined.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/Helpers.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int ARGUMENT_OUT_OF_BOUND;
}
namespace
{
template <UInt8 K>
struct WithK
{
template <typename T>
using AggregateFunction = AggregateFunctionUniqCombined<T, K>;
template <bool is_exact, bool argument_is_tuple>
using AggregateFunctionVariadic = AggregateFunctionUniqCombinedVariadic<is_exact, argument_is_tuple, K>;
};
template <UInt8 K>
AggregateFunctionPtr createAggregateFunctionWithK(const DataTypes & argument_types)
{
/// We use exact hash function if the arguments are not contiguous in memory, because only exact hash function has support for this case.
bool use_exact_hash_function = !isAllArgumentsContiguousInMemory(argument_types);
if (argument_types.size() == 1)
{
const IDataType & argument_type = *argument_types[0];
AggregateFunctionPtr res(createWithNumericType<WithK<K>::template AggregateFunction>(*argument_types[0]));
WhichDataType which(argument_type);
if (res)
return res;
else if (which.isDate())
return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeDate::FieldType>>();
else if (which.isDateTime())
return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeDateTime::FieldType>>();
else if (which.isStringOrFixedString())
return std::make_shared<typename WithK<K>::template AggregateFunction<String>>();
else if (which.isUUID())
return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeUUID::FieldType>>();
else if (which.isTuple())
{
if (use_exact_hash_function)
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<true, true>>(argument_types);
else
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<false, true>>(argument_types);
}
}
/// "Variadic" method also works as a fallback generic case for a single argument.
if (use_exact_hash_function)
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<true, false>>(argument_types);
else
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<false, false>>(argument_types);
}
AggregateFunctionPtr createAggregateFunctionUniqCombined(
const std::string & name, const DataTypes & argument_types, const Array & params)
{
/// log2 of the number of cells in HyperLogLog.
/// Reasonable default value, selected to be comparable in quality with "uniq" aggregate function.
UInt8 precision = 17;
if (!params.empty())
{
if (params.size() != 1)
throw Exception(
"Aggregate function " + name + " requires one parameter or less.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
UInt64 precision_param = applyVisitor(FieldVisitorConvertToNumber<UInt64>(), params[0]);
// This range is hardcoded below
if (precision_param > 20 || precision_param < 12)
throw Exception(
"Parameter for aggregate function " + name + "is out or range: [12, 20].", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
precision = precision_param;
}
if (argument_types.empty())
throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
switch (precision)
{
case 12:
return createAggregateFunctionWithK<12>(argument_types);
case 13:
return createAggregateFunctionWithK<13>(argument_types);
case 14:
return createAggregateFunctionWithK<14>(argument_types);
case 15:
return createAggregateFunctionWithK<15>(argument_types);
case 16:
return createAggregateFunctionWithK<16>(argument_types);
case 17:
return createAggregateFunctionWithK<17>(argument_types);
case 18:
return createAggregateFunctionWithK<18>(argument_types);
case 19:
return createAggregateFunctionWithK<19>(argument_types);
case 20:
return createAggregateFunctionWithK<20>(argument_types);
}
__builtin_unreachable();
}
} // namespace
void registerAggregateFunctionUniqCombined(AggregateFunctionFactory & factory)
{
factory.registerFunction("uniqCombined", createAggregateFunctionUniqCombined);
}
} // namespace DB

229
dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.h Normal file
View File

@ -0,0 +1,229 @@
#pragma once
#include <Common/CombinedCardinalityEstimator.h>
#include <DataTypes/DataTypeTuple.h>
#include <DataTypes/DataTypeUUID.h>
#include <DataTypes/DataTypesNumber.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <AggregateFunctions/UniqCombinedBiasData.h>
#include <AggregateFunctions/UniqVariadicHash.h>
#include <ext/bit_cast.h>
#include <Columns/ColumnVector.h>
#include <Columns/ColumnsNumber.h>
#include <Common/FieldVisitors.h>
#include <Common/SipHash.h>
#include <Common/typeid_cast.h>
namespace DB
{
namespace detail
{
/** Hash function for uniqCombined.
*/
template <typename T>
struct AggregateFunctionUniqCombinedTraits
{
static UInt32 hash(T x)
{
return static_cast<UInt32>(intHash64(x));
}
};
template <>
struct AggregateFunctionUniqCombinedTraits<UInt128>
{
static UInt32 hash(UInt128 x)
{
return sipHash64(x);
}
};
template <>
struct AggregateFunctionUniqCombinedTraits<Float32>
{
static UInt32 hash(Float32 x)
{
UInt64 res = ext::bit_cast<UInt64>(x);
return static_cast<UInt32>(intHash64(res));
}
};
template <>
struct AggregateFunctionUniqCombinedTraits<Float64>
{
static UInt32 hash(Float64 x)
{
UInt64 res = ext::bit_cast<UInt64>(x);
return static_cast<UInt32>(intHash64(res));
}
};
} // namespace detail
template <typename Key, UInt8 K>
struct AggregateFunctionUniqCombinedDataWithKey
{
// TODO(ilezhankin): pre-generate values for |UniqCombinedBiasData|,
// at the moment gen-bias-data.py script doesn't work.
// We want to migrate from |HashSet| to |HyperLogLogCounter| when the sizes in memory become almost equal.
// The size per element in |HashSet| is sizeof(Key)*2 bytes, and the overall size of |HyperLogLogCounter| is 2^K * 6 bits.
// For Key=UInt32 we can calculate: 2^X * 4 * 2 ≤ 2^(K-3) * 6 ⇒ X ≤ K-4.
using Set = CombinedCardinalityEstimator<Key, HashSet<Key, TrivialHash, HashTableGrower<>>, 16, K - 4, K, TrivialHash, Key>;
Set set;
};
template <typename Key>
struct AggregateFunctionUniqCombinedDataWithKey<Key, 17>
{
using Set = CombinedCardinalityEstimator<Key,
HashSet<Key, TrivialHash, HashTableGrower<>>,
16,
13,
17,
TrivialHash,
Key,
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
HyperLogLogMode::FullFeatured>;
Set set;
};
template <typename T, UInt8 K>
struct AggregateFunctionUniqCombinedData : public AggregateFunctionUniqCombinedDataWithKey<UInt32, K>
{
};
template <UInt8 K>
struct AggregateFunctionUniqCombinedData<String, K> : public AggregateFunctionUniqCombinedDataWithKey<UInt64, K>
{
};
template <typename T, UInt8 K>
class AggregateFunctionUniqCombined final
: public IAggregateFunctionDataHelper<AggregateFunctionUniqCombinedData<T, K>, AggregateFunctionUniqCombined<T, K>>
{
public:
String getName() const override
{
return "uniqCombined";
}
DataTypePtr getReturnType() const override
{
return std::make_shared<DataTypeUInt64>();
}
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
{
if constexpr (!std::is_same_v<T, String>)
{
const auto & value = static_cast<const ColumnVector<T> &>(*columns[0]).getData()[row_num];
this->data(place).set.insert(detail::AggregateFunctionUniqCombinedTraits<T>::hash(value));
}
else
{
StringRef value = columns[0]->getDataAt(row_num);
this->data(place).set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
}
}
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
{
this->data(place).set.merge(this->data(rhs).set);
}
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
{
this->data(place).set.write(buf);
}
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
{
this->data(place).set.read(buf);
}
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
{
static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
}
const char * getHeaderFilePath() const override
{
return __FILE__;
}
};
/** For multiple arguments. To compute, hashes them.
* You can pass multiple arguments as is; You can also pass one argument - a tuple.
* But (for the possibility of efficient implementation), you can not pass several arguments, among which there are tuples.
*/
template <bool is_exact, bool argument_is_tuple, UInt8 K>
class AggregateFunctionUniqCombinedVariadic final : public IAggregateFunctionDataHelper<AggregateFunctionUniqCombinedData<UInt64, K>,
AggregateFunctionUniqCombinedVariadic<is_exact, argument_is_tuple, K>>
{
private:
size_t num_args = 0;
public:
explicit AggregateFunctionUniqCombinedVariadic(const DataTypes & arguments)
{
if (argument_is_tuple)
num_args = typeid_cast<const DataTypeTuple &>(*arguments[0]).getElements().size();
else
num_args = arguments.size();
}
String getName() const override
{
return "uniqCombined";
}
DataTypePtr getReturnType() const override
{
return std::make_shared<DataTypeUInt64>();
}
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
{
this->data(place).set.insert(typename AggregateFunctionUniqCombinedData<UInt64, K>::Set::value_type(
UniqVariadicHash<is_exact, argument_is_tuple>::apply(num_args, columns, row_num)));
}
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
{
this->data(place).set.merge(this->data(rhs).set);
}
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
{
this->data(place).set.write(buf);
}
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
{
this->data(place).set.read(buf);
}
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
{
static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
}
const char * getHeaderFilePath() const override
{
return __FILE__;
}
};
} // namespace DB

4
dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h
View File

@ -16,10 +16,12 @@
namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
struct ComparePairFirst final
@ -191,7 +193,7 @@ public:
const auto time_arg = arguments.front().get();
if (!WhichDataType(time_arg).isDateTime() && !WhichDataType(time_arg).isUInt32())
throw Exception{"Illegal type " + time_arg->getName() + " of first argument of aggregate function " + getName()
+ ", must be DateTime or UInt32"};
+ ", must be DateTime or UInt32", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
for (const auto i : ext::range(1, arguments.size()))
{

2
dbms/src/AggregateFunctions/CMakeLists.txt
View File

@ -20,5 +20,5 @@ list(REMOVE_ITEM clickhouse_aggregate_functions_headers
)
add_library(clickhouse_aggregate_functions ${LINK_MODE} ${clickhouse_aggregate_functions_sources})
target_link_libraries(clickhouse_aggregate_functions dbms)
target_link_libraries(clickhouse_aggregate_functions PRIVATE dbms)
target_include_directories (clickhouse_aggregate_functions BEFORE PRIVATE ${COMMON_INCLUDE_DIR})

2
dbms/src/AggregateFunctions/registerAggregateFunctions.cpp
View File

@ -21,6 +21,7 @@ void registerAggregateFunctionsStatisticsSimple(AggregateFunctionFactory &);
void registerAggregateFunctionSum(AggregateFunctionFactory &);
void registerAggregateFunctionSumMap(AggregateFunctionFactory &);
void registerAggregateFunctionsUniq(AggregateFunctionFactory &);
void registerAggregateFunctionUniqCombined(AggregateFunctionFactory &);
void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &);
void registerAggregateFunctionTopK(AggregateFunctionFactory &);
void registerAggregateFunctionsBitwise(AggregateFunctionFactory &);
@ -55,6 +56,7 @@ void registerAggregateFunctions()
registerAggregateFunctionSum(factory);
registerAggregateFunctionSumMap(factory);
registerAggregateFunctionsUniq(factory);
registerAggregateFunctionUniqCombined(factory);
registerAggregateFunctionUniqUpTo(factory);
registerAggregateFunctionTopK(factory);
registerAggregateFunctionsBitwise(factory);

4
dbms/src/Client/Connection.cpp
View File

@ -108,14 +108,14 @@ void Connection::connect()
disconnect();
/// Add server address to exception. Also Exception will remember stack trace. It's a pity that more precise exception type is lost.
throw NetException(e.displayText(), "(" + getDescription() + ")", ErrorCodes::NETWORK_ERROR);
throw NetException(e.displayText() + " (" + getDescription() + ")", ErrorCodes::NETWORK_ERROR);
}
catch (Poco::TimeoutException & e)
{
disconnect();
/// Add server address to exception. Also Exception will remember stack trace. It's a pity that more precise exception type is lost.
throw NetException(e.displayText(), "(" + getDescription() + ")", ErrorCodes::SOCKET_TIMEOUT);
throw NetException(e.displayText() + " (" + getDescription() + ")", ErrorCodes::SOCKET_TIMEOUT);
}
}

2
dbms/src/Columns/ColumnConst.h
View File

@ -209,7 +209,7 @@ public:
Field getField() const { return getDataColumn()[0]; }
template <typename T>
T getValue() const { return getField().safeGet<typename NearestFieldType<T>::Type>(); }
T getValue() const { return getField().safeGet<NearestFieldType<T>>(); }
};
}

8
dbms/src/Columns/ColumnDecimal.cpp
View File

@ -213,8 +213,8 @@ void ColumnDecimal<T>::getExtremes(Field & min, Field & max) const
{
if (data.size() == 0)
{
min = typename NearestFieldType<T>::Type(0, scale);
max = typename NearestFieldType<T>::Type(0, scale);
min = NearestFieldType<T>(0, scale);
max = NearestFieldType<T>(0, scale);
return;
}
@ -229,8 +229,8 @@ void ColumnDecimal<T>::getExtremes(Field & min, Field & max) const
cur_max = x;
}
min = typename NearestFieldType<T>::Type(cur_min, scale);
max = typename NearestFieldType<T>::Type(cur_max, scale);
min = NearestFieldType<T>(cur_min, scale);
max = NearestFieldType<T>(cur_max, scale);
}
template class ColumnDecimal<Decimal32>;

2
dbms/src/Columns/ColumnDecimal.h
View File

@ -91,7 +91,7 @@ public:
void insertFrom(const IColumn & src, size_t n) override { data.push_back(static_cast<const Self &>(src).getData()[n]); }
void insertData(const char * pos, size_t /*length*/) override;
void insertDefault() override { data.push_back(T()); }
void insert(const Field & x) override { data.push_back(DB::get<typename NearestFieldType<T>::Type>(x)); }
void insert(const Field & x) override { data.push_back(DB::get<NearestFieldType<T>>(x)); }
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
void popBack(size_t n) override { data.resize_assume_reserved(data.size() - n); }

84
dbms/src/Columns/ColumnUnique.h
View File

@ -81,12 +81,14 @@ public:
{
return column_holder->allocatedBytes()
+ index.allocatedBytes()
+ (cached_null_mask ? cached_null_mask->allocatedBytes() : 0);
+ (nested_null_mask ? nested_null_mask->allocatedBytes() : 0);
}
void forEachSubcolumn(IColumn::ColumnCallback callback) override
{
callback(column_holder);
index.setColumn(getRawColumnPtr());
if (is_nullable)
nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask);
}
const UInt64 * tryGetSavedHash() const override { return index.tryGetSavedHash(); }
@ -100,8 +102,8 @@ private:
ReverseIndex<UInt64, ColumnType> index;
/// For DataTypeNullable, stores null map.
mutable ColumnPtr cached_null_mask;
mutable ColumnPtr cached_column_nullable;
ColumnPtr nested_null_mask;
ColumnPtr nested_column_nullable;
class IncrementalHash
{
@ -118,6 +120,9 @@ private:
mutable IncrementalHash hash;
void createNullMask();
void updateNullMask();
static size_t numSpecialValues(bool is_nullable) { return is_nullable ? 2 : 1; }
size_t numSpecialValues() const { return numSpecialValues(is_nullable); }
@ -148,6 +153,7 @@ ColumnUnique<ColumnType>::ColumnUnique(const ColumnUnique & other)
, index(numSpecialValues(is_nullable), 0)
{
index.setColumn(getRawColumnPtr());
createNullMask();
}
template <typename ColumnType>
@ -158,6 +164,7 @@ ColumnUnique<ColumnType>::ColumnUnique(const IDataType & type)
const auto & holder_type = is_nullable ? *static_cast<const DataTypeNullable &>(type).getNestedType() : type;
column_holder = holder_type.createColumn()->cloneResized(numSpecialValues());
index.setColumn(getRawColumnPtr());
createNullMask();
}
template <typename ColumnType>
@ -172,32 +179,51 @@ ColumnUnique<ColumnType>::ColumnUnique(MutableColumnPtr && holder, bool is_nulla
throw Exception("Holder column for ColumnUnique can't be nullable.", ErrorCodes::ILLEGAL_COLUMN);
index.setColumn(getRawColumnPtr());
createNullMask();
}
template <typename ColumnType>
void ColumnUnique<ColumnType>::createNullMask()
{
if (is_nullable)
{
size_t size = getRawColumnPtr()->size();
if (!nested_null_mask)
{
ColumnUInt8::MutablePtr null_mask = ColumnUInt8::create(size, UInt8(0));
null_mask->getData()[getNullValueIndex()] = 1;
nested_null_mask = std::move(null_mask);
nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask);
}
else
throw Exception("Null mask for ColumnUnique is already created.", ErrorCodes::LOGICAL_ERROR);
}
}
template <typename ColumnType>
void ColumnUnique<ColumnType>::updateNullMask()
{
if (is_nullable)
{
if (!nested_null_mask)
throw Exception("Null mask for ColumnUnique is was not created.", ErrorCodes::LOGICAL_ERROR);
size_t size = getRawColumnPtr()->size();
if (nested_null_mask->size() != size)
{
IColumn & null_mask = nested_null_mask->assumeMutableRef();
static_cast<ColumnUInt8 &>(null_mask).getData().resize_fill(size);
}
}
}
template <typename ColumnType>
const ColumnPtr & ColumnUnique<ColumnType>::getNestedColumn() const
{
if (is_nullable)
{
size_t size = getRawColumnPtr()->size();
if (!cached_null_mask)
{
ColumnUInt8::MutablePtr null_mask = ColumnUInt8::create(size, UInt8(0));
null_mask->getData()[getNullValueIndex()] = 1;
cached_null_mask = std::move(null_mask);
cached_column_nullable = ColumnNullable::create(column_holder, cached_null_mask);
}
return nested_column_nullable;
if (cached_null_mask->size() != size)
{
MutableColumnPtr null_mask = (*std::move(cached_null_mask)).mutate();
static_cast<ColumnUInt8 &>(*null_mask).getData().resize_fill(size);
cached_null_mask = std::move(null_mask);
cached_column_nullable = ColumnNullable::create(column_holder, cached_null_mask);
}
return cached_column_nullable;
}
return column_holder;
}
@ -227,6 +253,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsert(const Field & x)
if (pos != prev_size)
column->popBack(1);
updateNullMask();
return pos;
}
@ -260,6 +288,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsertData(const char * pos, size_t lengt
index.insertFromLastRow();
}
updateNullMask();
return insertion_point;
}
@ -288,6 +318,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsertDataWithTerminatingZero(const char
if (position != prev_size)
column->popBack(1);
updateNullMask();
return static_cast<size_t>(position);
}
@ -305,7 +337,7 @@ StringRef ColumnUnique<ColumnType>::serializeValueIntoArena(size_t n, Arena & ar
size_t nested_size = 0;
if (n == getNullValueIndex())
if (n != getNullValueIndex())
nested_size = column_holder->serializeValueIntoArena(n, arena, begin).size;
return StringRef(pos, sizeof(null_flag) + nested_size);
@ -343,6 +375,8 @@ size_t ColumnUnique<ColumnType>::uniqueDeserializeAndInsertFromArena(const char
if (index_pos != prev_size)
column->popBack(1);
updateNullMask();
return static_cast<size_t>(index_pos);
}
@ -533,6 +567,8 @@ MutableColumnPtr ColumnUnique<ColumnType>::uniqueInsertRangeFrom(const IColumn &
if (!positions_column)
throw Exception("Can't find index type for ColumnUnique", ErrorCodes::LOGICAL_ERROR);
updateNullMask();
return positions_column;
}
@ -577,6 +613,8 @@ IColumnUnique::IndexesWithOverflow ColumnUnique<ColumnType>::uniqueInsertRangeWi
if (!positions_column)
throw Exception("Can't find index type for ColumnUnique", ErrorCodes::LOGICAL_ERROR);
updateNullMask();
IColumnUnique::IndexesWithOverflow indexes_with_overflow;
indexes_with_overflow.indexes = std::move(positions_column);
indexes_with_overflow.overflowed_keys = std::move(overflowed_keys);

4
dbms/src/Columns/ColumnVector.cpp
View File

@ -311,8 +311,8 @@ void ColumnVector<T>::getExtremes(Field & min, Field & max) const
cur_max = x;
}
min = typename NearestFieldType<T>::Type(cur_min);
max = typename NearestFieldType<T>::Type(cur_max);
min = NearestFieldType<T>(cur_min);
max = NearestFieldType<T>(cur_max);
}
/// Explicit template instantiations - to avoid code bloat in headers.

2
dbms/src/Columns/ColumnVector.h
View File

@ -244,7 +244,7 @@ public:
void insert(const Field & x) override
{
data.push_back(DB::get<typename NearestFieldType<T>::Type>(x));
data.push_back(DB::get<NearestFieldType<T>>(x));
}
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;

1
dbms/src/Common/ClickHouseRevision.cpp
View File

@ -4,4 +4,5 @@
namespace ClickHouseRevision
{
unsigned get() { return VERSION_REVISION; }
unsigned getVersionInteger() { return VERSION_INTEGER; }
}

1
dbms/src/Common/ClickHouseRevision.h
View File

@ -3,4 +3,5 @@
namespace ClickHouseRevision
{
unsigned get();
unsigned getVersionInteger();
}

5
dbms/src/Common/Config/CMakeLists.txt
View File

@ -1,9 +1,8 @@
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
add_headers_and_sources(clickhouse_common_config .)
add_library(clickhouse_common_config ${LINK_MODE} ${clickhouse_common_config_headers} ${clickhouse_common_config_sources})
target_link_libraries (clickhouse_common_config clickhouse_common_zookeeper string_utils ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY})
target_include_directories (clickhouse_common_config PRIVATE ${DBMS_INCLUDE_DIR})
target_link_libraries(clickhouse_common_config PUBLIC common PRIVATE clickhouse_common_zookeeper string_utils PUBLIC ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY})
target_include_directories(clickhouse_common_config PUBLIC ${DBMS_INCLUDE_DIR})

22
dbms/src/Common/CounterInFile.h
View File

@ -18,7 +18,15 @@
#include <common/Types.h>
#define SMALL_READ_WRITE_BUFFER_SIZE 16
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_OPEN_FILE;
extern const int CANNOT_READ_ALL_DATA;
extern const int ATTEMPT_TO_READ_AFTER_EOF;
}
}
/** Stores a number in the file.
@ -26,6 +34,9 @@
*/
class CounterInFile
{
private:
static inline constexpr size_t SMALL_READ_WRITE_BUFFER_SIZE = 16;
public:
/// path - the name of the file, including the path
CounterInFile(const std::string & path_) : path(path_) {}
@ -56,13 +67,13 @@ public:
int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0666);
if (-1 == fd)
DB::throwFromErrno("Cannot open file " + path);
DB::throwFromErrno("Cannot open file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
try
{
int flock_ret = flock(fd, LOCK_EX);
if (-1 == flock_ret)
DB::throwFromErrno("Cannot lock file " + path);
DB::throwFromErrno("Cannot lock file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
if (!file_doesnt_exists)
{
@ -130,7 +141,7 @@ public:
int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0666);
if (-1 == fd)
DB::throwFromErrno("Cannot open file " + path);
DB::throwFromErrno("Cannot open file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
try
{
@ -178,6 +189,3 @@ private:
std::string path;
std::mutex mutex;
};
#undef SMALL_READ_WRITE_BUFFER_SIZE

3
dbms/src/Common/CurrentMetrics.cpp
View File

@ -39,7 +39,8 @@
M(StorageBufferRows, "Number of rows in buffers of Buffer tables") \
M(StorageBufferBytes, "Number of bytes in buffers of Buffer tables") \
M(DictCacheRequests, "Number of requests in fly to data sources of dictionaries of cache type.") \
M(Revision, "Revision of the server. It is a number incremented for every release or release candidate.") \
M(Revision, "Revision of the server. It is a number incremented for every release or release candidate except patch releases.") \
M(VersionInteger, "Version of the server in a single integer number in base-1000. For example, version 11.22.33 is translated to 11022033.") \
M(RWLockWaitingReaders, "Number of threads waiting for read on a table RWLock.") \
M(RWLockWaitingWriters, "Number of threads waiting for write on a table RWLock.") \
M(RWLockActiveReaders, "Number of threads holding read lock in a table RWLock.") \

5
dbms/src/Common/ErrorCodes.cpp
View File

@ -396,6 +396,11 @@ namespace ErrorCodes
extern const int MULTIPLE_ASSIGNMENTS_TO_COLUMN = 419;
extern const int CANNOT_UPDATE_COLUMN = 420;
extern const int CANNOT_ADD_DIFFERENT_AGGREGATE_STATES = 421;
extern const int UNSUPPORTED_URI_SCHEME = 422;
extern const int CANNOT_GETTIMEOFDAY = 423;
extern const int CANNOT_LINK = 424;
extern const int SYSTEM_ERROR = 425;
extern const int NULL_POINTER_DEREFERENCE = 426;
extern const int KEEPER_EXCEPTION = 999;
extern const int POCO_EXCEPTION = 1000;

2
dbms/src/Common/Exception.cpp
View File

@ -52,7 +52,7 @@ std::string errnoToString(int code, int e)
void throwFromErrno(const std::string & s, int code, int e)
{
throw ErrnoException(s + ", " + errnoToString(code, e));
throw ErrnoException(s + ", " + errnoToString(code, e), code, e);
}
void tryLogCurrentException(const char * log_name, const std::string & start_of_message)

38
dbms/src/Common/Exception.h
View File

@ -14,19 +14,25 @@ namespace Poco { class Logger; }
namespace DB
{
namespace ErrorCodes
{
extern const int POCO_EXCEPTION;
}
class Exception : public Poco::Exception
{
public:
Exception() {} /// For deferred initialization.
Exception(const std::string & msg, int code = 0) : Poco::Exception(msg, code) {}
Exception(const std::string & msg, const std::string & arg, int code = 0) : Poco::Exception(msg, arg, code) {}
Exception(const std::string & msg, const Exception & exc, int code = 0) : Poco::Exception(msg, exc, code), trace(exc.trace) {}
explicit Exception(const Poco::Exception & exc) : Poco::Exception(exc.displayText()) {}
Exception(const std::string & msg, int code) : Poco::Exception(msg, code) {}
Exception(const std::string & msg, const Exception & nested_exception, int code)
: Poco::Exception(msg, nested_exception, code), trace(nested_exception.trace) {}
const char * name() const throw() override { return "DB::Exception"; }
const char * className() const throw() override { return "DB::Exception"; }
DB::Exception * clone() const override { return new DB::Exception(*this); }
enum CreateFromPocoTag { CreateFromPoco };
Exception(CreateFromPocoTag, const Poco::Exception & exc) : Poco::Exception(exc.displayText(), ErrorCodes::POCO_EXCEPTION) {}
Exception * clone() const override { return new Exception(*this); }
void rethrow() const override { throw *this; }
const char * name() const throw() override { return "DB::Exception"; }
/// Add something to the existing message.
void addMessage(const std::string & arg) { extendedMessage(arg); }
@ -35,6 +41,8 @@ public:
private:
StackTrace trace;
const char * className() const throw() override { return "DB::Exception"; }
};
@ -42,25 +50,27 @@ private:
class ErrnoException : public Exception
{
public:
ErrnoException(const std::string & msg, int code = 0, int saved_errno_ = 0)
ErrnoException(const std::string & msg, int code, int saved_errno_)
: Exception(msg, code), saved_errno(saved_errno_) {}
ErrnoException(const std::string & msg, const std::string & arg, int code = 0, int saved_errno_ = 0)
: Exception(msg, arg, code), saved_errno(saved_errno_) {}
ErrnoException(const std::string & msg, const Exception & exc, int code = 0, int saved_errno_ = 0)
: Exception(msg, exc, code), saved_errno(saved_errno_) {}
ErrnoException * clone() const override { return new ErrnoException(*this); }
void rethrow() const override { throw *this; }
int getErrno() const { return saved_errno; }
private:
int saved_errno;
const char * name() const throw() override { return "DB::ErrnoException"; }
const char * className() const throw() override { return "DB::ErrnoException"; }
};
using Exceptions = std::vector<std::exception_ptr>;
std::string errnoToString(int code = 0, int the_errno = errno);
[[noreturn]] void throwFromErrno(const std::string & s, int code = 0, int the_errno = errno);
std::string errnoToString(int code, int the_errno = errno);
[[noreturn]] void throwFromErrno(const std::string & s, int code, int the_errno = errno);
/** Try to write an exception to the log (and forget about it).

16
dbms/src/Common/HashTable/SmallTable.h
View File

@ -3,6 +3,15 @@
#include <Common/HashTable/HashMap.h>
namespace DB
{
namespace ErrorCodes
{
extern const int INCORRECT_DATA;
}
}
/** Replacement of the hash table for a small number (<10) of keys.
* Implemented as an array with linear search.
* The array is located inside the object.
@ -13,7 +22,6 @@
* you should check if the table is not full,
* and do a `fallback` in this case (for example, use a real hash table).
*/
template
<
typename Key,
@ -86,7 +94,7 @@ public:
DB::readVarUInt(size, in);
if (size > capacity)
throw DB::Exception("Illegal size");
throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
is_initialized = true;
}
@ -306,7 +314,7 @@ public:
DB::readVarUInt(new_size, rb);
if (new_size > capacity)
throw DB::Exception("Illegal size");
throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
for (size_t i = 0; i < new_size; ++i)
buf[i].read(rb);
@ -324,7 +332,7 @@ public:
DB::readText(new_size, rb);
if (new_size > capacity)
throw DB::Exception("Illegal size");
throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
for (size_t i = 0; i < new_size; ++i)
{

13
dbms/src/Common/NetException.h
View File

@ -6,20 +6,17 @@
namespace DB
{
class NetException : public DB::Exception
class NetException : public Exception
{
public:
NetException(const std::string & msg, int code = 0) : DB::Exception(msg, code) {}
NetException(const std::string & msg, const std::string & arg, int code = 0) : DB::Exception(msg, arg, code) {}
NetException(const std::string & msg, const DB::Exception & exc, int code = 0) : DB::Exception(msg, exc, code) {}
NetException(const std::string & msg, int code) : Exception(msg, code) {}
explicit NetException(const DB::Exception & exc) : DB::Exception(exc) {}
explicit NetException(const Poco::Exception & exc) : DB::Exception(exc.displayText()) {}
NetException * clone() const override { return new NetException(*this); }
void rethrow() const override { throw *this; }
private:
const char * name() const throw() override { return "DB::NetException"; }
const char * className() const throw() override { return "DB::NetException"; }
DB::NetException * clone() const override { return new DB::NetException(*this); }
void rethrow() const override { throw *this; }
};
}

2
dbms/src/Common/PoolWithFailoverBase.h
View File

@ -305,7 +305,7 @@ void PoolWithFailoverBase<TNestedPool>::reportError(const Entry & entry)
return;
}
}
throw DB::Exception("Can't find pool to report error.");
throw DB::Exception("Can't find pool to report error", DB::ErrorCodes::LOGICAL_ERROR);
}
template <typename TNestedPool>

1
dbms/src/Common/ProfileEvents.cpp
View File

@ -170,6 +170,7 @@
M(OSWriteBytes, "Number of bytes written to disks or block devices. Doesn't include bytes that are in page cache dirty pages. May not include data that was written by OS asynchronously.") \
M(OSReadChars, "Number of bytes read from filesystem, including page cache.") \
M(OSWriteChars, "Number of bytes written to filesystem, including page cache.") \
M(CreatedHTTPConnections, "Total amount of created HTTP connections (closed or opened).") \
namespace ProfileEvents
{

30
dbms/src/Common/ShellCommand.cpp
View File

@ -4,9 +4,11 @@
#include <dlfcn.h>
#include <Common/Exception.h>
#include <Common/ShellCommand.h>
#include <common/logger_useful.h>
#include <IO/WriteBufferFromVector.h>
#include <IO/WriteHelpers.h>
#include <port/unistd.h>
#include <csignal>
namespace DB
@ -75,13 +77,27 @@ namespace
namespace DB
{
ShellCommand::ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd, bool terminate_in_destructor_)
: pid(pid)
, terminate_in_destructor(terminate_in_destructor_)
, log(&Poco::Logger::get("ShellCommand"))
, in(in_fd)
, out(out_fd)
, err(err_fd) {}
ShellCommand::~ShellCommand()
{
if (!wait_called)
if (terminate_in_destructor)
{
int retcode = kill(pid, SIGTERM);
if (retcode != 0)
LOG_WARNING(log, "Cannot kill pid " << pid << " errno '" << errnoToString(retcode) << "'");
}
else if (!wait_called)
tryWait();
}
std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only)
std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor)
{
/** Here it is written that with a normal call `vfork`, there is a chance of deadlock in multithreaded programs,
* because of the resolving of characters in the shared library
@ -128,7 +144,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, c
_exit(int(ReturnCodes::CANNOT_EXEC));
}
std::unique_ptr<ShellCommand> res(new ShellCommand(pid, pipe_stdin.write_fd, pipe_stdout.read_fd, pipe_stderr.read_fd));
std::unique_ptr<ShellCommand> res(new ShellCommand(pid, pipe_stdin.write_fd, pipe_stdout.read_fd, pipe_stderr.read_fd, terminate_in_destructor));
/// Now the ownership of the file descriptors is passed to the result.
pipe_stdin.write_fd = -1;
@ -139,7 +155,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, c
}
std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command, bool pipe_stdin_only)
std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command, bool pipe_stdin_only, bool terminate_in_destructor)
{
/// Arguments in non-constant chunks of memory (as required for `execv`).
/// Moreover, their copying must be done before calling `vfork`, so after `vfork` do a minimum of things.
@ -149,11 +165,11 @@ std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command,
char * const argv[] = { argv0.data(), argv1.data(), argv2.data(), nullptr };
return executeImpl("/bin/sh", argv, pipe_stdin_only);
return executeImpl("/bin/sh", argv, pipe_stdin_only, terminate_in_destructor);
}
std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & path, const std::vector<std::string> & arguments)
std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & path, const std::vector<std::string> & arguments, bool terminate_in_destructor)
{
size_t argv_sum_size = path.size() + 1;
for (const auto & arg : arguments)
@ -174,7 +190,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & pa
argv[arguments.size() + 1] = nullptr;
return executeImpl(path.data(), argv.data(), false);
return executeImpl(path.data(), argv.data(), false, terminate_in_destructor);
}

16
dbms/src/Common/ShellCommand.h
View File

@ -28,11 +28,13 @@ class ShellCommand
private:
pid_t pid;
bool wait_called = false;
bool terminate_in_destructor;
ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd)
: pid(pid), in(in_fd), out(out_fd), err(err_fd) {}
Poco::Logger * log;
static std::unique_ptr<ShellCommand> executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only);
ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd, bool terminate_in_destructor_);
static std::unique_ptr<ShellCommand> executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor);
public:
WriteBufferFromFile in; /// If the command reads from stdin, do not forget to call in.close() after writing all the data there.
@ -41,11 +43,13 @@ public:
~ShellCommand();
/// Run the command using /bin/sh -c
static std::unique_ptr<ShellCommand> execute(const std::string & command, bool pipe_stdin_only = false);
/// Run the command using /bin/sh -c.
/// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
static std::unique_ptr<ShellCommand> execute(const std::string & command, bool pipe_stdin_only = false, bool terminate_in_destructor = false);
/// Run the executable with the specified arguments. `arguments` - without argv[0].
static std::unique_ptr<ShellCommand> executeDirect(const std::string & path, const std::vector<std::string> & arguments);
/// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
static std::unique_ptr<ShellCommand> executeDirect(const std::string & path, const std::vector<std::string> & arguments, bool terminate_in_destructor = false);
/// Wait for the process to end, throw an exception if the code is not 0 or if the process was not completed by itself.
void wait();

22
dbms/src/Common/StatusFile.cpp
View File

@ -20,6 +20,14 @@
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_OPEN_FILE;
extern const int CANNOT_CLOSE_FILE;
extern const int CANNOT_TRUNCATE_FILE;
extern const int CANNOT_SEEK_THROUGH_FILE;
}
StatusFile::StatusFile(const std::string & path_)
: path(path_)
@ -43,7 +51,7 @@ StatusFile::StatusFile(const std::string & path_)
fd = ::open(path.c_str(), O_WRONLY | O_CREAT, 0666);
if (-1 == fd)
throwFromErrno("Cannot open file " + path);
throwFromErrno("Cannot open file " + path, ErrorCodes::CANNOT_OPEN_FILE);
try
{
@ -51,16 +59,16 @@ StatusFile::StatusFile(const std::string & path_)
if (-1 == flock_ret)
{
if (errno == EWOULDBLOCK)
throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running.");
throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running.", ErrorCodes::CANNOT_OPEN_FILE);
else
throwFromErrno("Cannot lock file " + path);
throwFromErrno("Cannot lock file " + path, ErrorCodes::CANNOT_OPEN_FILE);
}
if (0 != ftruncate(fd, 0))
throwFromErrno("Cannot ftruncate " + path);
throwFromErrno("Cannot ftruncate " + path, ErrorCodes::CANNOT_TRUNCATE_FILE);
if (0 != lseek(fd, 0, SEEK_SET))
throwFromErrno("Cannot lseek " + path);
throwFromErrno("Cannot lseek " + path, ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
/// Write information about current server instance to the file.
{
@ -82,10 +90,10 @@ StatusFile::StatusFile(const std::string & path_)
StatusFile::~StatusFile()
{
if (0 != close(fd))
LOG_ERROR(&Logger::get("StatusFile"), "Cannot close file " << path << ", " << errnoToString());
LOG_ERROR(&Logger::get("StatusFile"), "Cannot close file " << path << ", " << errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
if (0 != unlink(path.c_str()))
LOG_ERROR(&Logger::get("StatusFile"), "Cannot unlink file " << path << ", " << errnoToString());
LOG_ERROR(&Logger::get("StatusFile"), "Cannot unlink file " << path << ", " << errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
}
}

57
dbms/src/Common/XDBCBridgeHelper.h
View File

@ -68,6 +68,7 @@ protected:
public:
using Configuration = Poco::Util::AbstractConfiguration;
Context & context;
const Configuration & config;
static constexpr inline auto DEFAULT_HOST = "localhost";
@ -78,8 +79,8 @@ public:
static constexpr inline auto IDENTIFIER_QUOTE_HANDLER = "/identifier_quote";
static constexpr inline auto PING_OK_ANSWER = "Ok.";
XDBCBridgeHelper(const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_)
: http_timeout(http_timeout_), connection_string(connection_string_), config(config_)
XDBCBridgeHelper(Context & global_context_, const Poco::Timespan & http_timeout_, const std::string & connection_string_)
: http_timeout(http_timeout_), connection_string(connection_string_), context(global_context_), config(context.getConfigRef())
{
size_t bridge_port = config.getUInt(BridgeHelperMixin::configPrefix() + ".port", DEFAULT_PORT);
std::string bridge_host = config.getString(BridgeHelperMixin::configPrefix() + ".host", DEFAULT_HOST);
@ -210,7 +211,8 @@ private:
/* Contains logic for instantiation of the bridge instance */
void startBridge() const
{
BridgeHelperMixin::startBridge(config, log, http_timeout);
auto cmd = BridgeHelperMixin::startBridge(config, log, http_timeout);
context.addXDBCBridgeCommand(std::move(cmd));
}
};
@ -230,7 +232,7 @@ struct JDBCBridgeMixin
return "JDBC";
}
static void startBridge(const Poco::Util::AbstractConfiguration &, const Poco::Logger *, const Poco::Timespan &)
static std::unique_ptr<ShellCommand> startBridge(const Poco::Util::AbstractConfiguration &, const Poco::Logger *, const Poco::Timespan &)
{
throw Exception("jdbc-bridge is not running. Please, start it manually", ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING);
}
@ -253,11 +255,13 @@ struct ODBCBridgeMixin
return "ODBC";
}
static void startBridge(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout)
static std::unique_ptr<ShellCommand> startBridge(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout)
{
/// Path to executable folder
Poco::Path path{config.getString("application.dir", "/usr/bin")};
std::vector<std::string> cmd_args;
path.setFileName(
#if CLICKHOUSE_SPLIT_BINARY
"clickhouse-odbc-bridge"
@ -268,34 +272,35 @@ struct ODBCBridgeMixin
std::stringstream command;
command << path.toString() <<
#if CLICKHOUSE_SPLIT_BINARY
" "
#else
" odbc-bridge "
#if !CLICKHOUSE_SPLIT_BINARY
cmd_args.push_back("odbc-bridge");
#endif
;
command << "--http-port " << config.getUInt(configPrefix() + ".port", DEFAULT_PORT) << ' ';
command << "--listen-host " << config.getString(configPrefix() + ".listen_host", XDBCBridgeHelper<ODBCBridgeMixin>::DEFAULT_HOST)
<< ' ';
command << "--http-timeout " << http_timeout.totalMicroseconds() << ' ';
cmd_args.push_back("--http-port");
cmd_args.push_back(std::to_string(config.getUInt(configPrefix() + ".port", DEFAULT_PORT)));
cmd_args.push_back("--listen-host");
cmd_args.push_back(config.getString(configPrefix() + ".listen_host", XDBCBridgeHelper<ODBCBridgeMixin>::DEFAULT_HOST));
cmd_args.push_back("--http-timeout");
cmd_args.push_back(std::to_string(http_timeout.totalMicroseconds()));
if (config.has("logger." + configPrefix() + "_log"))
command << "--log-path " << config.getString("logger." + configPrefix() + "_log") << ' ';
{
cmd_args.push_back("--log-path");
cmd_args.push_back(config.getString("logger." + configPrefix() + "_log"));
}
if (config.has("logger." + configPrefix() + "_errlog"))
command << "--err-log-path " << config.getString("logger." + configPrefix() + "_errlog") << ' ';
{
cmd_args.push_back("--err-log-path");
cmd_args.push_back(config.getString("logger." + configPrefix() + "_errlog"));
}
if (config.has("logger." + configPrefix() + "_level"))
command << "--log-level " << config.getString("logger." + configPrefix() + "_level") << ' ';
command << "&"; /// we don't want to wait this process
{
cmd_args.push_back("--log-level");
cmd_args.push_back(config.getString("logger." + configPrefix() + "_level"));
}
auto command_str = command.str();
LOG_TRACE(log, "Starting " + serviceAlias());
std::cerr << command_str << std::endl;
LOG_TRACE(log, "Starting " + serviceAlias() + " with command: " << command_str);
auto cmd = ShellCommand::execute(command_str);
cmd->wait();
return ShellCommand::executeDirect(path.toString(), cmd_args, true);
}
};
}

3
dbms/src/Common/ZooKeeper/CMakeLists.txt
View File

@ -4,7 +4,8 @@ add_headers_and_sources(clickhouse_common_zookeeper .)
add_library(clickhouse_common_zookeeper ${LINK_MODE} ${clickhouse_common_zookeeper_headers} ${clickhouse_common_zookeeper_sources})
target_link_libraries (clickhouse_common_zookeeper clickhouse_common_io)
target_link_libraries (clickhouse_common_zookeeper PUBLIC clickhouse_common_io common PRIVATE string_utils PUBLIC ${Poco_Util_LIBRARY})
target_include_directories(clickhouse_common_zookeeper PUBLIC ${DBMS_INCLUDE_DIR})
if (ENABLE_TESTS)
add_subdirectory (tests)

18
dbms/src/Common/ZooKeeper/ZooKeeperHolder.cpp
View File

@ -1,5 +1,15 @@
#include "ZooKeeperHolder.h"
namespace DB
{
namespace ErrorCodes
{
extern const int NULL_POINTER_DEREFERENCE;
}
}
using namespace zkutil;
ZooKeeperHolder::UnstorableZookeeperHandler ZooKeeperHolder::getZooKeeper()
@ -47,7 +57,7 @@ ZooKeeperHolder::UnstorableZookeeperHandler::UnstorableZookeeperHandler(ZooKeepe
ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->()
{
if (zk_ptr == nullptr)
throw DB::Exception(nullptr_exception_message);
throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
return zk_ptr.get();
}
@ -55,20 +65,20 @@ ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->()
const ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->() const
{
if (zk_ptr == nullptr)
throw DB::Exception(nullptr_exception_message);
throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
return zk_ptr.get();
}
ZooKeeper & ZooKeeperHolder::UnstorableZookeeperHandler::operator*()
{
if (zk_ptr == nullptr)
throw DB::Exception(nullptr_exception_message);
throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
return *zk_ptr;
}
const ZooKeeper & ZooKeeperHolder::UnstorableZookeeperHandler::operator*() const
{
if (zk_ptr == nullptr)
throw DB::Exception(nullptr_exception_message);
throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
return *zk_ptr;
}

16
dbms/src/Common/ZooKeeper/tests/CMakeLists.txt
View File

@ -1,23 +1,23 @@
add_executable(zkutil_test_commands zkutil_test_commands.cpp)
target_link_libraries(zkutil_test_commands clickhouse_common_zookeeper)
target_link_libraries(zkutil_test_commands PRIVATE clickhouse_common_zookeeper)
add_executable(zkutil_test_commands_new_lib zkutil_test_commands_new_lib.cpp)
target_link_libraries(zkutil_test_commands_new_lib clickhouse_common_zookeeper)
target_link_libraries(zkutil_test_commands_new_lib PRIVATE clickhouse_common_zookeeper)
add_executable(zkutil_test_lock zkutil_test_lock.cpp)
target_link_libraries(zkutil_test_lock clickhouse_common_zookeeper)
target_link_libraries(zkutil_test_lock PRIVATE clickhouse_common_zookeeper)
add_executable(zkutil_expiration_test zkutil_expiration_test.cpp)
target_link_libraries(zkutil_expiration_test clickhouse_common_zookeeper)
target_link_libraries(zkutil_expiration_test PRIVATE clickhouse_common_zookeeper)
add_executable(zkutil_test_async zkutil_test_async.cpp)
target_link_libraries(zkutil_test_async clickhouse_common_zookeeper)
target_link_libraries(zkutil_test_async PRIVATE clickhouse_common_zookeeper)
add_executable(zkutil_zookeeper_holder zkutil_zookeeper_holder.cpp)
target_link_libraries(zkutil_zookeeper_holder clickhouse_common_zookeeper)
target_link_libraries(zkutil_zookeeper_holder PRIVATE clickhouse_common_zookeeper)
add_executable (zk_many_watches_reconnect zk_many_watches_reconnect.cpp)
target_link_libraries (zk_many_watches_reconnect clickhouse_common_zookeeper clickhouse_common_config)
target_link_libraries (zk_many_watches_reconnect PRIVATE clickhouse_common_zookeeper clickhouse_common_config)
add_executable (zookeeper_impl zookeeper_impl.cpp)
target_link_libraries (zookeeper_impl clickhouse_common_zookeeper)
target_link_libraries (zookeeper_impl PRIVATE clickhouse_common_zookeeper)

1
dbms/src/Common/config.h.in
View File

@ -16,3 +16,4 @@
#cmakedefine01 USE_POCO_NETSSL
#cmakedefine01 CLICKHOUSE_SPLIT_BINARY
#cmakedefine01 ENABLE_INSERT_INFILE
#cmakedefine01 USE_BASE64

1
dbms/src/Common/config_version.h.in
View File

@ -23,6 +23,7 @@
#cmakedefine VERSION_FULL "@VERSION_FULL@"
#cmakedefine VERSION_DESCRIBE "@VERSION_DESCRIBE@"
#cmakedefine VERSION_GITHASH "@VERSION_GITHASH@"
#cmakedefine VERSION_INTEGER @VERSION_INTEGER@
#if defined(VERSION_MAJOR)
#define DBMS_VERSION_MAJOR VERSION_MAJOR

14
dbms/src/Common/createHardLink.cpp
View File

@ -8,6 +8,12 @@
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_STAT;
extern const int CANNOT_LINK;
}
void createHardLink(const String & source_path, const String & destination_path)
{
if (0 != link(source_path.c_str(), destination_path.c_str()))
@ -20,16 +26,16 @@ void createHardLink(const String & source_path, const String & destination_path)
struct stat destination_descr;
if (0 != lstat(source_path.c_str(), &source_descr))
throwFromErrno("Cannot stat " + source_path);
throwFromErrno("Cannot stat " + source_path, ErrorCodes::CANNOT_STAT);
if (0 != lstat(destination_path.c_str(), &destination_descr))
throwFromErrno("Cannot stat " + destination_path);
throwFromErrno("Cannot stat " + destination_path, ErrorCodes::CANNOT_STAT);
if (source_descr.st_ino != destination_descr.st_ino)
throwFromErrno("Destination file " + destination_path + " is already exist and have different inode.", 0, link_errno);
throwFromErrno("Destination file " + destination_path + " is already exist and have different inode.", ErrorCodes::CANNOT_LINK, link_errno);
}
else
throwFromErrno("Cannot link " + source_path + " to " + destination_path);
throwFromErrno("Cannot link " + source_path + " to " + destination_path, ErrorCodes::CANNOT_LINK);
}
}

4
dbms/src/Common/setThreadName.cpp
View File

@ -32,7 +32,7 @@ void setThreadName(const char * name)
#else
if (0 != prctl(PR_SET_NAME, name, 0, 0, 0))
#endif
DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME, ...)");
DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME, ...)", DB::ErrorCodes::PTHREAD_ERROR);
}
std::string getThreadName()
@ -48,7 +48,7 @@ std::string getThreadName()
// throw DB::Exception("Cannot get thread name with pthread_get_name_np()", DB::ErrorCodes::PTHREAD_ERROR);
#else
if (0 != prctl(PR_GET_NAME, name.data(), 0, 0, 0))
DB::throwFromErrno("Cannot get thread name with prctl(PR_GET_NAME)");
DB::throwFromErrno("Cannot get thread name with prctl(PR_GET_NAME)", DB::ErrorCodes::PTHREAD_ERROR);
#endif
name.resize(std::strlen(name.data()));

48
dbms/src/Common/tests/CMakeLists.txt
View File

@ -1,73 +1,73 @@
add_executable (hashes_test hashes_test.cpp)
target_link_libraries (hashes_test dbms)
target_link_libraries (hashes_test PRIVATE dbms)
add_executable (sip_hash sip_hash.cpp)
target_link_libraries (sip_hash clickhouse_common_io)
target_link_libraries (sip_hash PRIVATE clickhouse_common_io)
add_executable (sip_hash_perf sip_hash_perf.cpp)
target_link_libraries (sip_hash_perf clickhouse_common_io)
target_link_libraries (sip_hash_perf PRIVATE clickhouse_common_io)
add_executable (auto_array auto_array.cpp)
target_link_libraries (auto_array clickhouse_common_io)
target_link_libraries (auto_array PRIVATE clickhouse_common_io)
add_executable (lru_cache lru_cache.cpp)
target_link_libraries (lru_cache clickhouse_common_io)
target_link_libraries (lru_cache PRIVATE clickhouse_common_io)
add_executable (hash_table hash_table.cpp)
target_link_libraries (hash_table clickhouse_common_io)
target_link_libraries (hash_table PRIVATE clickhouse_common_io)
add_executable (small_table small_table.cpp)
target_link_libraries (small_table clickhouse_common_io)
target_link_libraries (small_table PRIVATE clickhouse_common_io)
add_executable (parallel_aggregation parallel_aggregation.cpp)
target_link_libraries (parallel_aggregation clickhouse_common_io)
target_link_libraries (parallel_aggregation PRIVATE clickhouse_common_io)
add_executable (parallel_aggregation2 parallel_aggregation2.cpp)
target_link_libraries (parallel_aggregation2 clickhouse_common_io)
target_link_libraries (parallel_aggregation2 PRIVATE clickhouse_common_io)
add_executable (int_hashes_perf int_hashes_perf.cpp AvalancheTest.cpp Random.cpp)
target_link_libraries (int_hashes_perf clickhouse_common_io)
target_link_libraries (int_hashes_perf PRIVATE clickhouse_common_io)
add_executable (simple_cache simple_cache.cpp)
target_include_directories (simple_cache PRIVATE ${DBMS_INCLUDE_DIR})
target_link_libraries (simple_cache common)
target_link_libraries (simple_cache PRIVATE common)
add_executable (compact_array compact_array.cpp)
target_link_libraries (compact_array clickhouse_common_io ${Boost_FILESYSTEM_LIBRARY})
target_link_libraries (compact_array PRIVATE clickhouse_common_io ${Boost_FILESYSTEM_LIBRARY})
add_executable (radix_sort radix_sort.cpp)
target_link_libraries (radix_sort clickhouse_common_io)
target_link_libraries (radix_sort PRIVATE clickhouse_common_io)
add_executable (shell_command_test shell_command_test.cpp)
target_link_libraries (shell_command_test clickhouse_common_io)
target_link_libraries (shell_command_test PRIVATE clickhouse_common_io)
add_executable (arena_with_free_lists arena_with_free_lists.cpp)
target_link_libraries (arena_with_free_lists clickhouse_common_io)
target_link_libraries (arena_with_free_lists PRIVATE clickhouse_common_io)
add_executable (pod_array pod_array.cpp)
target_link_libraries (pod_array clickhouse_common_io)
target_link_libraries (pod_array PRIVATE clickhouse_common_io)
add_executable (thread_creation_latency thread_creation_latency.cpp)
target_link_libraries (thread_creation_latency clickhouse_common_io)
target_link_libraries (thread_creation_latency PRIVATE clickhouse_common_io)
add_executable (thread_pool thread_pool.cpp)
target_link_libraries (thread_pool clickhouse_common_io)
target_link_libraries (thread_pool PRIVATE clickhouse_common_io)
add_executable (array_cache array_cache.cpp)
target_link_libraries (array_cache clickhouse_common_io)
target_link_libraries (array_cache PRIVATE clickhouse_common_io)
add_executable (space_saving space_saving.cpp)
target_link_libraries (space_saving clickhouse_common_io)
target_link_libraries (space_saving PRIVATE clickhouse_common_io)
add_executable (integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp)
target_include_directories (integer_hash_tables_and_hashes SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
target_link_libraries (integer_hash_tables_and_hashes clickhouse_common_io)
target_link_libraries (integer_hash_tables_and_hashes PRIVATE clickhouse_common_io)
add_executable (allocator allocator.cpp)
target_link_libraries (allocator clickhouse_common_io)
target_link_libraries (allocator PRIVATE clickhouse_common_io)
add_executable (cow_columns cow_columns.cpp)
target_link_libraries (cow_columns clickhouse_common_io)
target_link_libraries (cow_columns PRIVATE clickhouse_common_io)
add_executable (stopwatch stopwatch.cpp)
target_link_libraries (stopwatch clickhouse_common_io)
target_link_libraries (stopwatch PRIVATE clickhouse_common_io)

10
dbms/src/Common/tests/arena_with_free_lists.cpp
View File

@ -25,6 +25,14 @@
using namespace DB;
namespace DB
{
namespace ErrorCodes
{
extern const int SYSTEM_ERROR;
}
}
/// Implementation of ArenaWithFreeLists, which contains a bug. Used to reproduce the bug.
#if USE_BAD_ARENA
@ -237,7 +245,7 @@ int main(int argc, char ** argv)
rusage resource_usage;
if (0 != getrusage(RUSAGE_SELF, &resource_usage))
throwFromErrno("Cannot getrusage");
throwFromErrno("Cannot getrusage", ErrorCodes::SYSTEM_ERROR);
size_t allocated_bytes = resource_usage.ru_maxrss * 1024;
std::cerr << "Current memory usage: " << allocated_bytes << " bytes.\n";

13
dbms/src/Common/tests/thread_creation_latency.cpp
View File

@ -22,6 +22,15 @@ void f() { ++x; }
void * g(void *) { f(); return {}; }
namespace DB
{
namespace ErrorCodes
{
extern const int PTHREAD_ERROR;
}
}
template <typename F>
void test(size_t n, const char * name, F && kernel)
{
@ -80,9 +89,9 @@ int main(int argc, char ** argv)
{
pthread_t thread;
if (pthread_create(&thread, nullptr, g, nullptr))
DB::throwFromErrno("Cannot create thread.");
DB::throwFromErrno("Cannot create thread.", DB::ErrorCodes::PTHREAD_ERROR);
if (pthread_join(thread, nullptr))
DB::throwFromErrno("Cannot join thread.");
DB::throwFromErrno("Cannot join thread.", DB::ErrorCodes::PTHREAD_ERROR);
});
test(n, "Create and destroy std::thread each iteration", []

3
dbms/src/Core/Defines.h
View File

@ -62,6 +62,9 @@
#define DEFAULT_HTTP_READ_BUFFER_TIMEOUT 1800
#define DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT 1
/// Maximum namber of http-connections between two endpoints
/// the number is unmotivated
#define DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT 15
// more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html

75
dbms/src/Core/Field.h
View File

@ -578,43 +578,54 @@ template <> struct TypeName<Array> { static std::string get() { return "Array";
template <> struct TypeName<Tuple> { static std::string get() { return "Tuple"; } };
template <typename T> struct NearestFieldType;
template <typename T> struct NearestFieldTypeImpl;
template <> struct NearestFieldType<UInt8> { using Type = UInt64; };
template <> struct NearestFieldType<UInt16> { using Type = UInt64; };
template <> struct NearestFieldType<UInt32> { using Type = UInt64; };
template <> struct NearestFieldType<UInt64> { using Type = UInt64; };
#ifdef __APPLE__
template <> struct NearestFieldType<time_t> { using Type = UInt64; };
template <> struct NearestFieldType<size_t> { using Type = UInt64; };
#endif
template <> struct NearestFieldType<DayNum> { using Type = UInt64; };
template <> struct NearestFieldType<UInt128> { using Type = UInt128; };
template <> struct NearestFieldType<UUID> { using Type = UInt128; };
template <> struct NearestFieldType<Int8> { using Type = Int64; };
template <> struct NearestFieldType<Int16> { using Type = Int64; };
template <> struct NearestFieldType<Int32> { using Type = Int64; };
template <> struct NearestFieldType<Int64> { using Type = Int64; };
template <> struct NearestFieldType<Int128> { using Type = Int128; };
template <> struct NearestFieldType<Decimal32> { using Type = DecimalField<Decimal32>; };
template <> struct NearestFieldType<Decimal64> { using Type = DecimalField<Decimal64>; };
template <> struct NearestFieldType<Decimal128> { using Type = DecimalField<Decimal128>; };
template <> struct NearestFieldType<DecimalField<Decimal32>> { using Type = DecimalField<Decimal32>; };
template <> struct NearestFieldType<DecimalField<Decimal64>> { using Type = DecimalField<Decimal64>; };
template <> struct NearestFieldType<DecimalField<Decimal128>> { using Type = DecimalField<Decimal128>; };
template <> struct NearestFieldType<Float32> { using Type = Float64; };
template <> struct NearestFieldType<Float64> { using Type = Float64; };
template <> struct NearestFieldType<const char*> { using Type = String; };
template <> struct NearestFieldType<String> { using Type = String; };
template <> struct NearestFieldType<Array> { using Type = Array; };
template <> struct NearestFieldType<Tuple> { using Type = Tuple; };
template <> struct NearestFieldType<bool> { using Type = UInt64; };
template <> struct NearestFieldType<Null> { using Type = Null; };
/// char may be signed or unsigned, and behave identically to signed char or unsigned char,
/// but they are always three different types.
/// signedness of char is different in Linux on x86 and Linux on ARM.
template <> struct NearestFieldTypeImpl<char> { using Type = std::conditional_t<std::is_signed_v<char>, Int64, UInt64>; };
template <> struct NearestFieldTypeImpl<signed char> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<unsigned char> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<UInt16> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<UInt32> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<DayNum> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<UInt128> { using Type = UInt128; };
template <> struct NearestFieldTypeImpl<UUID> { using Type = UInt128; };
template <> struct NearestFieldTypeImpl<Int16> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<Int32> { using Type = Int64; };
/// long and long long are always different types that may behave identically or not.
/// This is different on Linux and Mac.
template <> struct NearestFieldTypeImpl<long> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<long long> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<unsigned long> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<unsigned long long> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<Int128> { using Type = Int128; };
template <> struct NearestFieldTypeImpl<Decimal32> { using Type = DecimalField<Decimal32>; };
template <> struct NearestFieldTypeImpl<Decimal64> { using Type = DecimalField<Decimal64>; };
template <> struct NearestFieldTypeImpl<Decimal128> { using Type = DecimalField<Decimal128>; };
template <> struct NearestFieldTypeImpl<DecimalField<Decimal32>> { using Type = DecimalField<Decimal32>; };
template <> struct NearestFieldTypeImpl<DecimalField<Decimal64>> { using Type = DecimalField<Decimal64>; };
template <> struct NearestFieldTypeImpl<DecimalField<Decimal128>> { using Type = DecimalField<Decimal128>; };
template <> struct NearestFieldTypeImpl<Float32> { using Type = Float64; };
template <> struct NearestFieldTypeImpl<Float64> { using Type = Float64; };
template <> struct NearestFieldTypeImpl<const char *> { using Type = String; };
template <> struct NearestFieldTypeImpl<String> { using Type = String; };
template <> struct NearestFieldTypeImpl<Array> { using Type = Array; };
template <> struct NearestFieldTypeImpl<Tuple> { using Type = Tuple; };
template <> struct NearestFieldTypeImpl<bool> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<Null> { using Type = Null; };
template <typename T>
using NearestFieldType = typename NearestFieldTypeImpl<T>::Type;
template <typename T>
decltype(auto) nearestFieldType(T && x)
{
using U = typename NearestFieldType<std::decay_t<T>>::Type;
using U = NearestFieldType<std::decay_t<T>>;
if constexpr (std::is_same_v<std::decay_t<T>, U>)
return std::forward<T>(x);
else

2
dbms/src/Core/Names.h
View File

@ -2,6 +2,7 @@
#include <vector>
#include <string>
#include <set>
#include <unordered_set>
#include <unordered_map>
@ -11,6 +12,7 @@ namespace DB
using Names = std::vector<std::string>;
using NameSet = std::unordered_set<std::string>;
using NameOrderedSet = std::set<std::string>;
using NameToNameMap = std::unordered_map<std::string, std::string>;
using NameToNameSetMap = std::unordered_map<std::string, NameSet>;

13
dbms/src/Core/tests/CMakeLists.txt
View File

@ -1,18 +1,15 @@
add_executable (exception exception.cpp)
target_link_libraries (exception clickhouse_common_io)
add_executable (string_pool string_pool.cpp)
target_link_libraries (string_pool clickhouse_common_io)
target_link_libraries (string_pool PRIVATE clickhouse_common_io)
target_include_directories (string_pool SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
add_executable (field field.cpp)
target_link_libraries (field dbms)
target_link_libraries (field PRIVATE dbms)
add_executable (move_field move_field.cpp)
target_link_libraries (move_field clickhouse_common_io)
target_link_libraries (move_field PRIVATE clickhouse_common_io)
add_executable (rvo_test rvo_test.cpp)
target_link_libraries (rvo_test Threads::Threads)
target_link_libraries (rvo_test PRIVATE Threads::Threads)
add_executable (string_ref_hash string_ref_hash.cpp)
target_link_libraries (string_ref_hash clickhouse_common_io)
target_link_libraries (string_ref_hash PRIVATE clickhouse_common_io)

25
dbms/src/Core/tests/exception.cpp
View File

@ -1,25 +0,0 @@
#include <iostream>
#include <Poco/Net/NetException.h>
#include <Common/Exception.h>
int main(int, char **)
{
try
{
//throw Poco::Net::ConnectionRefusedException();
throw DB::Exception(Poco::Net::ConnectionRefusedException());
}
catch (const DB::Exception & e)
{
std::cerr << e.displayText() << std::endl;
}
catch (const Poco::Exception & e)
{
std::cerr << e.displayText() << std::endl;
}
return 0;
}

6
dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp
View File

@ -179,8 +179,9 @@ void MergingAggregatedMemoryEfficientBlockInputStream::start()
auto thread_group = CurrentThread::getGroup();
reading_pool->schedule([&child, thread_group]
{
CurrentThread::attachToIfDetached(thread_group);
setThreadName("MergeAggReadThr");
if (thread_group)
CurrentThread::attachToIfDetached(thread_group);
CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread};
child->readPrefix();
});
@ -485,8 +486,9 @@ MergingAggregatedMemoryEfficientBlockInputStream::BlocksToMerge MergingAggregate
auto thread_group = CurrentThread::getGroup();
reading_pool->schedule([&input, &read_from_input, thread_group]
{
CurrentThread::attachToIfDetached(thread_group);
setThreadName("MergeAggReadThr");
if (thread_group)
CurrentThread::attachToIfDetached(thread_group);
CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread};
read_from_input(input);
});

18
dbms/src/DataStreams/ParallelInputsProcessor.h
View File

@ -107,9 +107,27 @@ public:
active_threads = max_threads;
threads.reserve(max_threads);
auto thread_group = CurrentThread::getGroup();
try
{
for (size_t i = 0; i < max_threads; ++i)
threads.emplace_back([=] () { thread(thread_group, i); } );
}
catch (...)
{
cancel(false);
wait();
if (active_threads)
{
active_threads = 0;
/// handler.onFinish() is supposed to be called from one of the threads when the number of
/// finished threads reaches max_threads. But since we weren't able to launch all threads,
/// we have to call onFinish() manually here.
handler.onFinish();
}
throw;
}
}
/// Ask all sources to stop earlier than they run out.
void cancel(bool kill)

1
dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
View File

@ -91,6 +91,7 @@ void PushingToViewsBlockOutputStream::write(const Block & block)
pool.schedule([=]
{
setThreadName("PushingToViewsBlockOutputStream");
if (thread_group)
CurrentThread::attachToIfDetached(thread_group);
process(block, view_num);
});

10
dbms/src/DataStreams/tests/CMakeLists.txt
View File

@ -1,16 +1,16 @@
set(SRCS)
add_executable (expression_stream expression_stream.cpp ${SRCS})
target_link_libraries (expression_stream dbms clickhouse_storages_system)
target_link_libraries (expression_stream PRIVATE dbms clickhouse_storages_system clickhouse_parsers)
add_executable (filter_stream filter_stream.cpp ${SRCS})
target_link_libraries (filter_stream dbms clickhouse_storages_system)
target_link_libraries (filter_stream PRIVATE dbms clickhouse_storages_system clickhouse_parsers clickhouse_common_io)
add_executable (union_stream2 union_stream2.cpp ${SRCS})
target_link_libraries (union_stream2 dbms)
target_link_libraries (union_stream2 PRIVATE dbms)
add_executable (collapsing_sorted_stream collapsing_sorted_stream.cpp ${SRCS})
target_link_libraries (collapsing_sorted_stream dbms)
target_link_libraries (collapsing_sorted_stream PRIVATE dbms)
add_executable (finish_sorting_stream finish_sorting_stream.cpp ${SRCS})
target_link_libraries (finish_sorting_stream dbms)
target_link_libraries (finish_sorting_stream PRIVATE dbms)

Some files were not shown because too many files have changed in this diff Show More