diff --git a/.gitmodules b/.gitmodules index c43b754dba8..a9e50ab8f6d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -46,3 +46,6 @@ [submodule "contrib/unixodbc"] path = contrib/unixodbc url = https://github.com/ClickHouse-Extras/UnixODBC.git +[submodule "contrib/base64"] + path = contrib/base64 + url = https://github.com/aklomp/base64.git diff --git a/CHANGELOG.md b/CHANGELOG.md index 33c569d2a88..4e81432edd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,15 @@ +## ClickHouse release 18.14.15, 2018-11-21 + +### Bug fixes: +* The size of memory chunk was overestimated while deserializing the column of type `Array(String)` that leads to "Memory limit exceeded" errors. The issue appeared in version 18.12.13. [#3589](https://github.com/yandex/ClickHouse/issues/3589) + ## ClickHouse release 18.14.14, 2018-11-20 ### Bug fixes: -* Fixed `ON CLUSTER` queries when cluster configured as secure (flag ``). [#3599] +* Fixed `ON CLUSTER` queries when cluster configured as secure (flag ``). [#3599](https://github.com/yandex/ClickHouse/pull/3599) ### Build changes: -* Fixed problems (llvm-7 from system, macos) [#3582] +* Fixed problems (llvm-7 from system, macos) [#3582](https://github.com/yandex/ClickHouse/pull/3582) ## ClickHouse release 18.14.11, 2018-10-29 diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md index 3a4646ff5bc..216d80f100f 100644 --- a/CHANGELOG_RU.md +++ b/CHANGELOG_RU.md @@ -1,10 +1,15 @@ +## ClickHouse release 18.14.15, 2018-11-21 + +### Исправления ошибок: +* При чтении столбцов типа `Array(String)`, размер требуемого куска памяти оценивался слишком большим, что приводило к исключению "Memory limit exceeded" при выполнении запроса. Ошибка появилась в версии 18.12.13. [#3589](https://github.com/yandex/ClickHouse/issues/3589) + ## ClickHouse release 18.14.14, 2018-11-20 ### Исправления ошибок: -* Исправлена работа запросов `ON CLUSTER` в случае, когда в конфигурации кластера включено шифрование (флаг ``). [#3599] +* Исправлена работа запросов `ON CLUSTER` в случае, когда в конфигурации кластера включено шифрование (флаг ``). [#3599](https://github.com/yandex/ClickHouse/pull/3599) ### Улучшения процесса сборки ClickHouse: -* Испрпавлены проблемы сборки (llvm-7 из системы, macos) [#3582] +* Испрпавлены проблемы сборки (llvm-7 из системы, macos) [#3582](https://github.com/yandex/ClickHouse/pull/3582) ## ClickHouse release 18.14.13, 2018-11-08 diff --git a/CMakeLists.txt b/CMakeLists.txt index 552ec57c8fd..27f31c44a5b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -256,6 +256,7 @@ include (cmake/find_capnp.cmake) include (cmake/find_llvm.cmake) include (cmake/find_cpuid.cmake) include (cmake/find_consistent-hashing.cmake) +include (cmake/find_base64.cmake) if (ENABLE_TESTS) include (cmake/find_gtest.cmake) endif () diff --git a/README.md b/README.md index b248d5ee906..1f4542e6afe 100644 --- a/README.md +++ b/README.md @@ -7,4 +7,6 @@ ClickHouse is an open-source column-oriented database management system that all * [Official website](https://clickhouse.yandex/) has quick high-level overview of ClickHouse on main page. * [Tutorial](https://clickhouse.yandex/tutorial.html) shows how to set up and query small ClickHouse cluster. * [Documentation](https://clickhouse.yandex/docs/en/) provides more in-depth information. +* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events. * [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any. +* You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person. diff --git a/cmake/dbms_generate_function.cmake b/cmake/dbms_generate_function.cmake deleted file mode 100644 index ae701f798fe..00000000000 --- a/cmake/dbms_generate_function.cmake +++ /dev/null @@ -1,10 +0,0 @@ -function(generate_function_register FUNCTION_AREA) - foreach(FUNCTION IN LISTS ARGN) - configure_file (registerFunction.h.in ${FUNCTIONS_GENERATED_DIR}register${FUNCTION}.h) - configure_file (registerFunction.cpp.in ${FUNCTIONS_GENERATED_DIR}register${FUNCTION}.cpp) - set(REGISTER_HEADERS "${REGISTER_HEADERS}#include \"register${FUNCTION}.h\"\n") - set(REGISTER_FUNCTIONS "${REGISTER_FUNCTIONS} register${FUNCTION}(factory);\n") - endforeach() - - configure_file (registerFunctions_area.cpp.in ${FUNCTIONS_GENERATED_DIR}registerFunctions${FUNCTION_AREA}.cpp) -endfunction() diff --git a/cmake/find_base64.cmake b/cmake/find_base64.cmake new file mode 100644 index 00000000000..ad71d11c1b4 --- /dev/null +++ b/cmake/find_base64.cmake @@ -0,0 +1,12 @@ +option (ENABLE_BASE64 "Enable base64" ON) + +if (ENABLE_BASE64) + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64") + message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive") + else() + set (BASE64_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64/include) + set (BASE64_LIBRARY base64) + set (USE_BASE64 1) + endif() +endif () + diff --git a/cmake/find_odbc.cmake b/cmake/find_odbc.cmake index d89e3b532d8..32a410c6f11 100644 --- a/cmake/find_odbc.cmake +++ b/cmake/find_odbc.cmake @@ -71,10 +71,10 @@ if (ENABLE_ODBC) ) # MinGW find usually fails - if(MINGW) + if (MINGW) set(ODBC_INCLUDE_DIRECTORIES ".") set(ODBC_LIBRARIES odbc32) - endif() + endif () include(FindPackageHandleStandardArgs) find_package_handle_standard_args(ODBC @@ -82,6 +82,10 @@ if (ENABLE_ODBC) ODBC_INCLUDE_DIRECTORIES ODBC_LIBRARIES) + if (USE_STATIC_LIBRARIES) + list(APPEND ODBC_LIBRARIES ${LTDL_LIBRARY}) + endif () + mark_as_advanced(ODBC_FOUND ODBC_LIBRARIES ODBC_INCLUDE_DIRECTORIES) endif () endif () diff --git a/cmake/find_poco.cmake b/cmake/find_poco.cmake index d8468e5306d..012f269d48d 100644 --- a/cmake/find_poco.cmake +++ b/cmake/find_poco.cmake @@ -93,8 +93,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY) endif () if (OPENSSL_FOUND AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL)) - set (Poco_NetSSL_LIBRARY PocoNetSSL) - set (Poco_Crypto_LIBRARY PocoCrypto) + set (Poco_NetSSL_LIBRARY PocoNetSSL ${OPENSSL_LIBRARIES}) + set (Poco_Crypto_LIBRARY PocoCrypto ${OPENSSL_LIBRARIES}) endif () if (USE_STATIC_LIBRARIES AND USE_INTERNAL_ZLIB_LIBRARY) diff --git a/cmake/lib_name.cmake b/cmake/lib_name.cmake index b49276fc279..5c919b263e6 100644 --- a/cmake/lib_name.cmake +++ b/cmake/lib_name.cmake @@ -1,5 +1,4 @@ set(DIVIDE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libdivide) -set(CITYHASH_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/cityhash102/include) set(COMMON_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/libs/libcommon/include ${ClickHouse_BINARY_DIR}/libs/libcommon/include) set(DBMS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/dbms/src ${ClickHouse_BINARY_DIR}/dbms/src) set(DOUBLE_CONVERSION_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/double-conversion) diff --git a/cmake/print_include_directories.cmake b/cmake/print_include_directories.cmake index 41c4773cfa3..c4c5d00c54f 100644 --- a/cmake/print_include_directories.cmake +++ b/cmake/print_include_directories.cmake @@ -10,6 +10,9 @@ list(APPEND dirs ${dirs1}) get_property (dirs1 TARGET common PROPERTY INCLUDE_DIRECTORIES) list(APPEND dirs ${dirs1}) +get_property (dirs1 TARGET cityhash PROPERTY INCLUDE_DIRECTORIES) +list(APPEND dirs ${dirs1}) + if (USE_INTERNAL_BOOST_LIBRARY) get_property (dirs1 TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY} PROPERTY INCLUDE_DIRECTORIES) list(APPEND dirs ${dirs1}) diff --git a/cmake/test_cpu.cmake b/cmake/test_cpu.cmake index 6894c58703d..c360de5b962 100644 --- a/cmake/test_cpu.cmake +++ b/cmake/test_cpu.cmake @@ -45,6 +45,38 @@ if (HAVE_SSE42) set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}") endif () +set (TEST_FLAG "-mssse3") +set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0") +check_cxx_source_compiles(" + #include + int main() { + __m64 a = _mm_abs_pi8(__m64()); + (void)a; + return 0; + } +" HAVE_SSSE3) + +set (TEST_FLAG "-mavx") +set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0") +check_cxx_source_compiles(" + #include + int main() { + auto a = _mm256_insert_epi8(__m256i(), 0, 0); + (void)a; + return 0; + } +" HAVE_AVX) + +set (TEST_FLAG "-mavx2") +set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0") +check_cxx_source_compiles(" + #include + int main() { + auto a = _mm256_add_epi16(__m256i(), __m256i()); + (void)a; + return 0; + } +" HAVE_AVX2) # gcc -dM -E -mpopcnt - < /dev/null | sort > gcc-dump-popcnt #define __POPCNT__ 1 @@ -65,5 +97,3 @@ if (HAVE_POPCNT AND NOT ARCH_AARCH64) endif () cmake_pop_check_state () - -# TODO: add here sse3 test if you want use it diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 928c48e9334..b58d211af41 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -191,3 +191,6 @@ if (USE_INTERNAL_LLVM_LIBRARY) add_subdirectory (llvm/llvm) endif () +if (USE_BASE64) + add_subdirectory (base64-cmake) +endif() diff --git a/contrib/base64 b/contrib/base64 new file mode 160000 index 00000000000..a27c565d1b6 --- /dev/null +++ b/contrib/base64 @@ -0,0 +1 @@ +Subproject commit a27c565d1b6c676beaf297fe503c4518185666f7 diff --git a/contrib/base64-cmake/.gitignore b/contrib/base64-cmake/.gitignore new file mode 100644 index 00000000000..0e56cf2f8c1 --- /dev/null +++ b/contrib/base64-cmake/.gitignore @@ -0,0 +1 @@ +config.h diff --git a/contrib/base64-cmake/CMakeLists.txt b/contrib/base64-cmake/CMakeLists.txt new file mode 100644 index 00000000000..09abb3b02b9 --- /dev/null +++ b/contrib/base64-cmake/CMakeLists.txt @@ -0,0 +1,52 @@ +SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64) + +set(base64_compile_instructions "") +LIST(LENGTH base64_compile_instructions 0) +macro(cast_to_bool var instruction) + if (HAVE_${var}) + set(base64_${var} 1) + set(base64_${var}_opt ${instruction}) + else() + set(base64_${var} 0) + endif() +endmacro() + +cast_to_bool(SSSE3 "-mssse3") +cast_to_bool(SSE41 "-msse4.1") +cast_to_bool(SSE42 "-msse4.2") +cast_to_bool(AVX "-mavx") +cast_to_bool(AVX2 "-mavx2") + +# write config.h file, to include it in application +file(READ config-header.tpl header) +file(WRITE config.h ${header}) +file(APPEND config.h "#define HAVE_SSSE3 ${base64_SSSE3}\n") +file(APPEND config.h "#define HAVE_SSE41 ${base64_SSE41}\n") +file(APPEND config.h "#define HAVE_SSE42 ${base64_SSE42}\n") +file(APPEND config.h "#define HAVE_AVX ${base64_AVX}\n") +file(APPEND config.h "#define HAVE_AVX2 ${base64_AVX2}\n") + +set(HAVE_FAST_UNALIGNED_ACCESS 0) +if (${base64_SSSE3} OR ${base64_SSE41} OR ${base64_SSE42} OR ${base64_AVX} OR ${base64_AVX2}) + set(HAVE_FAST_UNALIGNED_ACCESS 1) +endif () + +file(APPEND config.h "#define HAVE_FAST_UNALIGNED_ACCESS " ${HAVE_FAST_UNALIGNED_ACCESS} "\n") + +add_library(base64 ${LINK_MODE} + ${LIBRARY_DIR}/lib/lib.c + ${LIBRARY_DIR}/lib/codec_choose.c + ${LIBRARY_DIR}/lib/arch/avx/codec.c + ${LIBRARY_DIR}/lib/arch/avx2/codec.c + ${LIBRARY_DIR}/lib/arch/generic/codec.c + ${LIBRARY_DIR}/lib/arch/neon32/codec.c + ${LIBRARY_DIR}/lib/arch/neon64/codec.c + ${LIBRARY_DIR}/lib/arch/sse41/codec.c + ${LIBRARY_DIR}/lib/arch/sse42/codec.c + ${LIBRARY_DIR}/lib/arch/ssse3/codec.c + + ${LIBRARY_DIR}/lib/codecs.h + config.h) + +target_compile_options(base64 PRIVATE ${base64_SSSE3_opt} ${base64_SSE41_opt} ${base64_SSE42_opt} ${base64_AVX_opt} ${base64_AVX2_opt}) +target_include_directories(base64 PRIVATE ${LIBRARY_DIR}/include .) \ No newline at end of file diff --git a/contrib/base64-cmake/config-header.tpl b/contrib/base64-cmake/config-header.tpl new file mode 100644 index 00000000000..c978ca0c082 --- /dev/null +++ b/contrib/base64-cmake/config-header.tpl @@ -0,0 +1,2 @@ +#define HAVE_NEON32 0 +#define HAVE_NEON64 0 diff --git a/contrib/cityhash102/CMakeLists.txt b/contrib/cityhash102/CMakeLists.txt index eafa6f2025f..c3f53a8f878 100644 --- a/contrib/cityhash102/CMakeLists.txt +++ b/contrib/cityhash102/CMakeLists.txt @@ -1,9 +1,8 @@ add_library(cityhash src/city.cc - include/citycrc.h include/city.h src/config.h) -target_include_directories (cityhash BEFORE PUBLIC include) -target_include_directories (cityhash PRIVATE src) +target_include_directories(cityhash BEFORE PUBLIC include) +target_include_directories(cityhash PRIVATE src) diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 3edcbbee5f3..eebd3504192 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -119,7 +119,7 @@ endif () if (USE_EMBEDDED_COMPILER) llvm_libs_all(REQUIRED_LLVM_LIBRARIES) - target_link_libraries (dbms ${REQUIRED_LLVM_LIBRARIES}) + target_link_libraries (dbms PRIVATE ${REQUIRED_LLVM_LIBRARIES}) target_include_directories (dbms SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) endif () @@ -150,33 +150,48 @@ if (NOT ARCH_ARM AND CPUID_LIBRARY) endif() target_link_libraries (clickhouse_common_io + PUBLIC common + PRIVATE string_utils widechar_width ${LINK_LIBRARIES_ONLY_ON_X86_64} ${LZ4_LIBRARY} ${ZSTD_LIBRARY} ${DOUBLE_CONVERSION_LIBRARIES} + pocoext + PUBLIC ${Poco_Net_LIBRARY} ${Poco_Util_LIBRARY} ${Poco_Foundation_LIBRARY} + ${RE2_LIBRARY} + ${RE2_ST_LIBRARY} + ${CITYHASH_LIBRARIES} + PRIVATE ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} ${ELF_LIBRARY} + PUBLIC ${Boost_SYSTEM_LIBRARY} + PRIVATE apple_rt ${CMAKE_DL_LIBS} ) target_link_libraries (dbms + PRIVATE clickhouse_parsers clickhouse_common_config + PUBLIC clickhouse_common_io + pocoext + PUBLIC ${MYSQLXX_LIBRARY} - ${RE2_LIBRARY} - ${RE2_ST_LIBRARY} + PRIVATE ${BTRIE_LIBRARIES} ${Boost_PROGRAM_OPTIONS_LIBRARY} + PUBLIC + ${Boost_SYSTEM_LIBRARY} ) if (NOT USE_INTERNAL_RE2_LIBRARY) @@ -193,8 +208,8 @@ if (Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY) endif() if (USE_POCO_SQLODBC) - target_link_libraries (clickhouse_common_io ${Poco_SQL_LIBRARY}) - target_link_libraries (dbms ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY}) + target_link_libraries (clickhouse_common_io PRIVATE ${Poco_SQL_LIBRARY}) + target_link_libraries (dbms PRIVATE ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY}) if (NOT USE_INTERNAL_POCO_LIBRARY) target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIR}) target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR} PUBLIC ${Poco_SQL_INCLUDE_DIR}) @@ -208,48 +223,44 @@ if (Poco_Data_FOUND) endif() if (USE_POCO_DATAODBC) - target_link_libraries (clickhouse_common_io ${Poco_Data_LIBRARY}) - target_link_libraries (dbms ${Poco_DataODBC_LIBRARY}) + target_link_libraries (clickhouse_common_io PRIVATE ${Poco_Data_LIBRARY}) + target_link_libraries (dbms PRIVATE ${Poco_DataODBC_LIBRARY}) if (NOT USE_INTERNAL_POCO_LIBRARY) target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR}) endif() endif() if (USE_POCO_MONGODB) - target_link_libraries (dbms ${Poco_MongoDB_LIBRARY}) + target_link_libraries (dbms PRIVATE ${Poco_MongoDB_LIBRARY}) endif() if (USE_POCO_NETSSL) - target_link_libraries (clickhouse_common_io ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) - target_link_libraries (dbms ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) + target_link_libraries (clickhouse_common_io PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) + target_link_libraries (dbms PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) endif() -target_link_libraries (dbms ${Poco_Foundation_LIBRARY}) +target_link_libraries (dbms PRIVATE ${Poco_Foundation_LIBRARY}) if (USE_ICU) - target_link_libraries (dbms ${ICU_LIBS}) + target_link_libraries (dbms PRIVATE ${ICU_LIBS}) target_include_directories (dbms SYSTEM PRIVATE ${ICU_INCLUDE_DIR}) endif () if (USE_CAPNP) - target_link_libraries (dbms ${CAPNP_LIBRARY}) + target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARY}) if (NOT USE_INTERNAL_CAPNP_LIBRARY) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR}) endif () endif () if (USE_RDKAFKA) - target_link_libraries (dbms ${RDKAFKA_LIBRARY}) + target_link_libraries (dbms PRIVATE ${RDKAFKA_LIBRARY}) if (NOT USE_INTERNAL_RDKAFKA_LIBRARY) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR}) endif () endif () -target_link_libraries(dbms ${OPENSSL_CRYPTO_LIBRARY}) - -target_link_libraries (dbms - Threads::Threads -) +target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY} Threads::Threads) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR}) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) @@ -285,6 +296,6 @@ if (ENABLE_TESTS) # attach all dbms gtest sources grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms dbms_gtest_sources) add_executable(unit_tests_dbms ${dbms_gtest_sources}) - target_link_libraries(unit_tests_dbms gtest_main dbms) + target_link_libraries(unit_tests_dbms PRIVATE gtest_main dbms clickhouse_common_zookeeper) add_check(unit_tests_dbms) endif () diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index e62a8e4796c..9f623279b99 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -22,3 +22,5 @@ endif () set (VERSION_NAME "${PROJECT_NAME}" CACHE STRING "") set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}" CACHE STRING "") set (VERSION_SO "${VERSION_STRING}" CACHE STRING "") + +math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000") diff --git a/dbms/programs/CMakeLists.txt b/dbms/programs/CMakeLists.txt index 136616ca44b..9d7c6f2cda1 100644 --- a/dbms/programs/CMakeLists.txt +++ b/dbms/programs/CMakeLists.txt @@ -48,45 +48,45 @@ else () link_directories (${LLVM_LIBRARY_DIRS}) endif () add_executable (clickhouse main.cpp) - target_link_libraries (clickhouse clickhouse_common_io) + target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils) target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR}) target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) if (ENABLE_CLICKHOUSE_SERVER) - target_link_libraries (clickhouse clickhouse-server-lib) + target_link_libraries (clickhouse PRIVATE clickhouse-server-lib) endif () if (ENABLE_CLICKHOUSE_CLIENT) - target_link_libraries (clickhouse clickhouse-client-lib) + target_link_libraries (clickhouse PRIVATE clickhouse-client-lib) endif () if (ENABLE_CLICKHOUSE_LOCAL) - target_link_libraries (clickhouse clickhouse-local-lib) + target_link_libraries (clickhouse PRIVATE clickhouse-local-lib) endif () if (ENABLE_CLICKHOUSE_BENCHMARK) - target_link_libraries (clickhouse clickhouse-benchmark-lib) + target_link_libraries (clickhouse PRIVATE clickhouse-benchmark-lib) endif () if (ENABLE_CLICKHOUSE_PERFORMANCE) - target_link_libraries (clickhouse clickhouse-performance-test-lib) + target_link_libraries (clickhouse PRIVATE clickhouse-performance-test-lib) endif () if (ENABLE_CLICKHOUSE_COPIER) - target_link_libraries (clickhouse clickhouse-copier-lib) + target_link_libraries (clickhouse PRIVATE clickhouse-copier-lib) endif () if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG) - target_link_libraries (clickhouse clickhouse-extract-from-config-lib) + target_link_libraries (clickhouse PRIVATE clickhouse-extract-from-config-lib) endif () if (ENABLE_CLICKHOUSE_COMPRESSOR) - target_link_libraries (clickhouse clickhouse-compressor-lib) + target_link_libraries (clickhouse PRIVATE clickhouse-compressor-lib) endif () if (ENABLE_CLICKHOUSE_FORMAT) - target_link_libraries (clickhouse clickhouse-format-lib) + target_link_libraries (clickhouse PRIVATE clickhouse-format-lib) endif () if (ENABLE_CLICKHOUSE_OBFUSCATOR) - target_link_libraries (clickhouse clickhouse-obfuscator-lib) + target_link_libraries (clickhouse PRIVATE clickhouse-obfuscator-lib) endif () if (USE_EMBEDDED_COMPILER) - target_link_libraries (clickhouse clickhouse-compiler-lib) + target_link_libraries (clickhouse PRIVATE clickhouse-compiler-lib) endif () if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) - target_link_libraries (clickhouse clickhouse-odbc-bridge-lib) + target_link_libraries (clickhouse PRIVATE clickhouse-odbc-bridge-lib) endif() set (CLICKHOUSE_BUNDLE) diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index 161321f4d8c..8931774bac1 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -42,10 +42,8 @@ namespace DB namespace ErrorCodes { - extern const int POCO_EXCEPTION; - extern const int STD_EXCEPTION; - extern const int UNKNOWN_EXCEPTION; extern const int BAD_ARGUMENTS; + extern const int EMPTY_DATA_PASSED; } class Benchmark @@ -170,7 +168,7 @@ private: } if (queries.empty()) - throw Exception("Empty list of queries."); + throw Exception("Empty list of queries.", ErrorCodes::EMPTY_DATA_PASSED); std::cerr << "Loaded " << queries.size() << " queries.\n"; } diff --git a/dbms/programs/benchmark/CMakeLists.txt b/dbms/programs/benchmark/CMakeLists.txt index 85af0750715..c41c46edeb8 100644 --- a/dbms/programs/benchmark/CMakeLists.txt +++ b/dbms/programs/benchmark/CMakeLists.txt @@ -1,8 +1,8 @@ add_library (clickhouse-benchmark-lib ${LINK_MODE} Benchmark.cpp) -target_link_libraries (clickhouse-benchmark-lib clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_link_libraries (clickhouse-benchmark-lib PRIVATE clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) if (CLICKHOUSE_SPLIT_BINARY) add_executable (clickhouse-benchmark clickhouse-benchmark.cpp) - target_link_libraries (clickhouse-benchmark clickhouse-benchmark-lib clickhouse_aggregate_functions) + target_link_libraries (clickhouse-benchmark PRIVATE clickhouse-benchmark-lib clickhouse_aggregate_functions) endif () diff --git a/dbms/programs/clang/CMakeLists.txt b/dbms/programs/clang/CMakeLists.txt index ca06f27546d..dec21ac611e 100644 --- a/dbms/programs/clang/CMakeLists.txt +++ b/dbms/programs/clang/CMakeLists.txt @@ -6,9 +6,9 @@ if (CLICKHOUSE_SPLIT_BINARY) if (USE_EMBEDDED_COMPILER) link_directories (${LLVM_LIBRARY_DIRS}) add_executable (clickhouse-clang clickhouse-clang.cpp) - target_link_libraries (clickhouse-clang clickhouse-compiler-lib) + target_link_libraries (clickhouse-clang PRIVATE clickhouse-compiler-lib) add_executable (clickhouse-lld clickhouse-lld.cpp) - target_link_libraries (clickhouse-lld clickhouse-compiler-lib) + target_link_libraries (clickhouse-lld PRIVATE clickhouse-compiler-lib) install (TARGETS clickhouse-clang clickhouse-lld RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) endif () endif () diff --git a/dbms/programs/client/CMakeLists.txt b/dbms/programs/client/CMakeLists.txt index 65353094c26..f3dd518e891 100644 --- a/dbms/programs/client/CMakeLists.txt +++ b/dbms/programs/client/CMakeLists.txt @@ -1,13 +1,12 @@ add_library (clickhouse-client-lib ${LINK_MODE} Client.cpp) -target_link_libraries (clickhouse-client-lib clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_link_libraries (clickhouse-client-lib PRIVATE clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY}) if (READLINE_INCLUDE_DIR) target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR}) endif () if (CLICKHOUSE_SPLIT_BINARY) add_executable (clickhouse-client clickhouse-client.cpp) - target_link_libraries (clickhouse-client clickhouse-client-lib) + target_link_libraries (clickhouse-client PRIVATE clickhouse-client-lib) endif () install (FILES clickhouse-client.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-client COMPONENT clickhouse-client RENAME config.xml) - diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 516547de5d1..ee57b322b94 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include #include @@ -86,9 +86,6 @@ namespace DB namespace ErrorCodes { - extern const int POCO_EXCEPTION; - extern const int STD_EXCEPTION; - extern const int UNKNOWN_EXCEPTION; extern const int NETWORK_ERROR; extern const int NO_DATA_TO_INSERT; extern const int BAD_ARGUMENTS; @@ -529,7 +526,7 @@ private: if (max_client_network_bandwidth) { - ThrottlerPtr throttler = std::make_shared(max_client_network_bandwidth, 0, ""); + ThrottlerPtr throttler = std::make_shared(max_client_network_bandwidth, 0, ""); connection->setThrottler(throttler); } diff --git a/dbms/programs/client/Suggest.h b/dbms/programs/client/Suggest.h index b93086d3b00..617e2bb520e 100644 --- a/dbms/programs/client/Suggest.h +++ b/dbms/programs/client/Suggest.h @@ -184,7 +184,7 @@ public: } catch (...) { - std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false) << "\n"; + std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n"; } /// Note that keyword suggestions are available even if we cannot load data from server. diff --git a/dbms/programs/compressor/CMakeLists.txt b/dbms/programs/compressor/CMakeLists.txt index 5c9c11072c9..7aa2cad5708 100644 --- a/dbms/programs/compressor/CMakeLists.txt +++ b/dbms/programs/compressor/CMakeLists.txt @@ -1,8 +1,8 @@ add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp) -target_link_libraries (clickhouse-compressor-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) if (CLICKHOUSE_SPLIT_BINARY) # Also in utils add_executable (clickhouse-compressor clickhouse-compressor.cpp) - target_link_libraries (clickhouse-compressor clickhouse-compressor-lib) + target_link_libraries (clickhouse-compressor PRIVATE clickhouse-compressor-lib) endif () diff --git a/dbms/programs/copier/CMakeLists.txt b/dbms/programs/copier/CMakeLists.txt index e8583dba440..ed3e55208aa 100644 --- a/dbms/programs/copier/CMakeLists.txt +++ b/dbms/programs/copier/CMakeLists.txt @@ -1,5 +1,5 @@ add_library (clickhouse-copier-lib ${LINK_MODE} ClusterCopier.cpp) -target_link_libraries (clickhouse-copier-lib clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions) +target_link_libraries (clickhouse-copier-lib PRIVATE clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions daemon) if (CLICKHOUSE_SPLIT_BINARY) add_executable (clickhouse-copier clickhouse-copier.cpp) diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp index 06f826d58d5..02d371957ef 100644 --- a/dbms/programs/copier/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -323,7 +323,7 @@ struct TaskTable struct TaskCluster { TaskCluster(const String & task_zookeeper_path_, const String & default_local_database_) - : task_zookeeper_path(task_zookeeper_path_), default_local_database(default_local_database_) {} + : task_zookeeper_path(task_zookeeper_path_), default_local_database(default_local_database_) {} void loadTasks(const Poco::Util::AbstractConfiguration & config, const String & base_key = ""); @@ -410,8 +410,7 @@ BlockInputStreamPtr squashStreamIntoOneBlock(const BlockInputStreamPtr & stream) return std::make_shared( stream, std::numeric_limits::max(), - std::numeric_limits::max() - ); + std::numeric_limits::max()); } Block getBlockWithAllStreamData(const BlockInputStreamPtr & stream) diff --git a/dbms/programs/extract-from-config/CMakeLists.txt b/dbms/programs/extract-from-config/CMakeLists.txt index c31b0e8cec9..62253649368 100644 --- a/dbms/programs/extract-from-config/CMakeLists.txt +++ b/dbms/programs/extract-from-config/CMakeLists.txt @@ -1,7 +1,7 @@ add_library (clickhouse-extract-from-config-lib ${LINK_MODE} ExtractFromConfig.cpp) -target_link_libraries (clickhouse-extract-from-config-lib clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_link_libraries (clickhouse-extract-from-config-lib PRIVATE clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) if (CLICKHOUSE_SPLIT_BINARY) add_executable (clickhouse-extract-from-config clickhouse-extract-from-config.cpp) - target_link_libraries (clickhouse-extract-from-config clickhouse-extract-from-config-lib) + target_link_libraries (clickhouse-extract-from-config PRIVATE clickhouse-extract-from-config-lib) endif () diff --git a/dbms/programs/format/CMakeLists.txt b/dbms/programs/format/CMakeLists.txt index f53cd11bee7..53d09e82621 100644 --- a/dbms/programs/format/CMakeLists.txt +++ b/dbms/programs/format/CMakeLists.txt @@ -1,6 +1,6 @@ add_library (clickhouse-format-lib ${LINK_MODE} Format.cpp) -target_link_libraries (clickhouse-format-lib dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_link_libraries (clickhouse-format-lib PRIVATE dbms clickhouse_common_io clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY}) if (CLICKHOUSE_SPLIT_BINARY) add_executable (clickhouse-format clickhouse-format.cpp) - target_link_libraries (clickhouse-format clickhouse-format-lib) + target_link_libraries (clickhouse-format PRIVATE clickhouse-format-lib) endif () diff --git a/dbms/programs/local/CMakeLists.txt b/dbms/programs/local/CMakeLists.txt index 9680ceaf1c7..07729d68563 100644 --- a/dbms/programs/local/CMakeLists.txt +++ b/dbms/programs/local/CMakeLists.txt @@ -1,7 +1,7 @@ add_library (clickhouse-local-lib ${LINK_MODE} LocalServer.cpp) -target_link_libraries (clickhouse-local-lib clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_link_libraries (clickhouse-local-lib PRIVATE clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY}) if (CLICKHOUSE_SPLIT_BINARY) add_executable (clickhouse-local clickhouse-local.cpp) - target_link_libraries (clickhouse-local clickhouse-local-lib) + target_link_libraries (clickhouse-local PRIVATE clickhouse-local-lib) endif () diff --git a/dbms/programs/main.cpp b/dbms/programs/main.cpp index adffde3979e..29d64213d9c 100644 --- a/dbms/programs/main.cpp +++ b/dbms/programs/main.cpp @@ -1,7 +1,7 @@ #include #include #include -#include /// pair +#include /// pair #if __has_include("config_tools.h") #include "config_tools.h" diff --git a/dbms/programs/obfuscator/CMakeLists.txt b/dbms/programs/obfuscator/CMakeLists.txt index 5ee6ace5a02..73c3f01e9cb 100644 --- a/dbms/programs/obfuscator/CMakeLists.txt +++ b/dbms/programs/obfuscator/CMakeLists.txt @@ -1,8 +1,8 @@ add_library (clickhouse-obfuscator-lib ${LINK_MODE} Obfuscator.cpp) -target_link_libraries (clickhouse-obfuscator-lib dbms ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_link_libraries (clickhouse-obfuscator-lib PRIVATE dbms ${Boost_PROGRAM_OPTIONS_LIBRARY}) if (CLICKHOUSE_SPLIT_BINARY) add_executable (clickhouse-obfuscator clickhouse-obfuscator.cpp) set_target_properties(clickhouse-obfuscator PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) - target_link_libraries (clickhouse-obfuscator clickhouse-obfuscator-lib) + target_link_libraries (clickhouse-obfuscator PRIVATE clickhouse-obfuscator-lib) endif () diff --git a/dbms/programs/odbc-bridge/CMakeLists.txt b/dbms/programs/odbc-bridge/CMakeLists.txt index d13a2866e77..a57c8c9c8cf 100644 --- a/dbms/programs/odbc-bridge/CMakeLists.txt +++ b/dbms/programs/odbc-bridge/CMakeLists.txt @@ -9,23 +9,23 @@ add_library (clickhouse-odbc-bridge-lib ${LINK_MODE} validateODBCConnectionString.cpp ) -target_link_libraries (clickhouse-odbc-bridge-lib clickhouse_common_io daemon dbms) +target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE clickhouse_common_io daemon dbms) target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include) if (USE_POCO_SQLODBC) - target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQLODBC_LIBRARY}) + target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQLODBC_LIBRARY}) target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR}) endif () if (Poco_SQL_FOUND) - target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQL_LIBRARY}) + target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQL_LIBRARY}) endif () if (USE_POCO_DATAODBC) - target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_DataODBC_LIBRARY}) + target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_DataODBC_LIBRARY}) target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR}) endif() if (Poco_Data_FOUND) - target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_Data_LIBRARY}) + target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_Data_LIBRARY}) endif () @@ -35,5 +35,5 @@ endif () if (CLICKHOUSE_SPLIT_BINARY) add_executable (clickhouse-odbc-bridge odbc-bridge.cpp) - target_link_libraries (clickhouse-odbc-bridge clickhouse-odbc-bridge-lib) + target_link_libraries (clickhouse-odbc-bridge PRIVATE clickhouse-odbc-bridge-lib) endif () diff --git a/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp b/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp index f59abd5f587..997ef9cf216 100644 --- a/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -124,9 +124,9 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques auto identifier_quote = getIdentifierQuote(hdbc); if (identifier_quote.length() == 0) settings.identifier_quoting_style = IdentifierQuotingStyle::None; - else if(identifier_quote[0] == '`') + else if (identifier_quote[0] == '`') settings.identifier_quoting_style = IdentifierQuotingStyle::Backticks; - else if(identifier_quote[0] == '"') + else if (identifier_quote[0] == '"') settings.identifier_quoting_style = IdentifierQuotingStyle::DoubleQuotes; else throw Exception("Can not map quote identifier '" + identifier_quote + "' to IdentifierQuotingStyle value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); diff --git a/dbms/programs/odbc-bridge/HandlerFactory.cpp b/dbms/programs/odbc-bridge/HandlerFactory.cpp index 8a0ff06268f..a6422db268c 100644 --- a/dbms/programs/odbc-bridge/HandlerFactory.cpp +++ b/dbms/programs/odbc-bridge/HandlerFactory.cpp @@ -25,7 +25,7 @@ Poco::Net::HTTPRequestHandler * HandlerFactory::createRequestHandler(const Poco: #else return nullptr; #endif - else if(uri.getPath() == "/identifier_quote") + else if (uri.getPath() == "/identifier_quote") #if USE_POCO_SQLODBC || USE_POCO_DATAODBC return new IdentifierQuoteHandler(keep_alive_timeout, context); #else diff --git a/dbms/programs/odbc-bridge/tests/CMakeLists.txt b/dbms/programs/odbc-bridge/tests/CMakeLists.txt index 5240a917429..5211c39d111 100644 --- a/dbms/programs/odbc-bridge/tests/CMakeLists.txt +++ b/dbms/programs/odbc-bridge/tests/CMakeLists.txt @@ -1,2 +1,2 @@ add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp) -target_link_libraries (validate-odbc-connection-string clickhouse-odbc-bridge-lib) +target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse-odbc-bridge-lib clickhouse_common_io) diff --git a/dbms/programs/odbc-bridge/validateODBCConnectionString.cpp b/dbms/programs/odbc-bridge/validateODBCConnectionString.cpp index a817a01c288..a64d7bb0b7c 100644 --- a/dbms/programs/odbc-bridge/validateODBCConnectionString.cpp +++ b/dbms/programs/odbc-bridge/validateODBCConnectionString.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include "validateODBCConnectionString.h" diff --git a/dbms/programs/performance-test/CMakeLists.txt b/dbms/programs/performance-test/CMakeLists.txt index 31796cd9d74..adad45025c3 100644 --- a/dbms/programs/performance-test/CMakeLists.txt +++ b/dbms/programs/performance-test/CMakeLists.txt @@ -1,8 +1,8 @@ add_library (clickhouse-performance-test-lib ${LINK_MODE} PerformanceTest.cpp) -target_link_libraries (clickhouse-performance-test-lib clickhouse_common_io dbms ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_link_libraries (clickhouse-performance-test-lib PRIVATE dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) if (CLICKHOUSE_SPLIT_BINARY) add_executable (clickhouse-performance-test clickhouse-performance-test.cpp) - target_link_libraries (clickhouse-performance-test clickhouse-performance-test-lib dbms) + target_link_libraries (clickhouse-performance-test PRIVATE clickhouse-performance-test-lib) endif () diff --git a/dbms/programs/performance-test/PerformanceTest.cpp b/dbms/programs/performance-test/PerformanceTest.cpp index e63d40e2db0..ebebedd8769 100644 --- a/dbms/programs/performance-test/PerformanceTest.cpp +++ b/dbms/programs/performance-test/PerformanceTest.cpp @@ -49,10 +49,10 @@ namespace DB { namespace ErrorCodes { - extern const int POCO_EXCEPTION; - extern const int STD_EXCEPTION; - extern const int UNKNOWN_EXCEPTION; extern const int NOT_IMPLEMENTED; + extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; + extern const int FILE_DOESNT_EXIST; } static String pad(size_t padding) @@ -156,7 +156,7 @@ struct StopConditionsSet else if (key == "average_speed_not_changing_for_ms") average_speed_not_changing_for_ms.value = stop_conditions_view->getUInt64(key); else - throw DB::Exception("Met unkown stop condition: " + key); + throw DB::Exception("Met unkown stop condition: " + key, DB::ErrorCodes::LOGICAL_ERROR); ++initialized_count; } @@ -521,7 +521,7 @@ public: { if (input_files.size() < 1) { - throw DB::Exception("No tests were specified", 0); + throw DB::Exception("No tests were specified", DB::ErrorCodes::BAD_ARGUMENTS); } std::string name; @@ -694,7 +694,7 @@ private: size_t ram_size_needed = config->getUInt64("preconditions.ram_size"); size_t actual_ram = getMemoryAmount(); if (!actual_ram) - throw DB::Exception("ram_size precondition not available on this platform", ErrorCodes::NOT_IMPLEMENTED); + throw DB::Exception("ram_size precondition not available on this platform", DB::ErrorCodes::NOT_IMPLEMENTED); if (ram_size_needed > actual_ram) { @@ -868,12 +868,12 @@ private: if (!test_config->has("query") && !test_config->has("query_file")) { - throw DB::Exception("Missing query fields in test's config: " + test_name); + throw DB::Exception("Missing query fields in test's config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS); } if (test_config->has("query") && test_config->has("query_file")) { - throw DB::Exception("Found both query and query_file fields. Choose only one"); + throw DB::Exception("Found both query and query_file fields. Choose only one", DB::ErrorCodes::BAD_ARGUMENTS); } if (test_config->has("query")) @@ -885,7 +885,7 @@ private: { const String filename = test_config->getString("query_file"); if (filename.empty()) - throw DB::Exception("Empty file name"); + throw DB::Exception("Empty file name", DB::ErrorCodes::BAD_ARGUMENTS); bool tsv = fs::path(filename).extension().string() == ".tsv"; @@ -909,7 +909,7 @@ private: if (queries.empty()) { - throw DB::Exception("Did not find any query to execute: " + test_name); + throw DB::Exception("Did not find any query to execute: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS); } if (test_config->has("substitutions")) @@ -929,7 +929,7 @@ private: if (!test_config->has("type")) { - throw DB::Exception("Missing type property in config: " + test_name); + throw DB::Exception("Missing type property in config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS); } String config_exec_type = test_config->getString("type"); @@ -938,7 +938,7 @@ private: else if (config_exec_type == "once") exec_type = ExecutionType::Once; else - throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name); + throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name, DB::ErrorCodes::BAD_ARGUMENTS); times_to_run = test_config->getUInt("times_to_run", 1); @@ -951,7 +951,7 @@ private: } if (stop_conditions_template.empty()) - throw DB::Exception("No termination conditions were found in config"); + throw DB::Exception("No termination conditions were found in config", DB::ErrorCodes::BAD_ARGUMENTS); for (size_t i = 0; i < times_to_run * queries.size(); ++i) stop_conditions_by_run.push_back(stop_conditions_template); @@ -978,7 +978,7 @@ private: else { if (lite_output) - throw DB::Exception("Specify main_metric for lite output"); + throw DB::Exception("Specify main_metric for lite output", DB::ErrorCodes::BAD_ARGUMENTS); } if (metrics.size() > 0) @@ -1023,22 +1023,14 @@ private: if (exec_type == ExecutionType::Loop) { for (const String & metric : metrics) - { if (std::find(non_loop_metrics.begin(), non_loop_metrics.end(), metric) != non_loop_metrics.end()) - { - throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")"); - } - } + throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS); } else { for (const String & metric : metrics) - { if (std::find(loop_metrics.begin(), loop_metrics.end(), metric) != loop_metrics.end()) - { - throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")"); - } - } + throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS); } } @@ -1439,7 +1431,7 @@ try if (input_files.empty()) { std::cerr << std::endl; - throw DB::Exception("Did not find any xml files"); + throw DB::Exception("Did not find any xml files", DB::ErrorCodes::BAD_ARGUMENTS); } else std::cerr << " found " << input_files.size() << " files." << std::endl; @@ -1454,7 +1446,7 @@ try fs::path file(filename); if (!fs::exists(file)) - throw DB::Exception("File '" + filename + "' does not exist"); + throw DB::Exception("File '" + filename + "' does not exist", DB::ErrorCodes::FILE_DOESNT_EXIST); if (fs::is_directory(file)) { @@ -1463,7 +1455,7 @@ try else { if (file.extension().string() != ".xml") - throw DB::Exception("File '" + filename + "' does not have .xml extension"); + throw DB::Exception("File '" + filename + "' does not have .xml extension", DB::ErrorCodes::BAD_ARGUMENTS); collected_files.push_back(filename); } } diff --git a/dbms/programs/server/CMakeLists.txt b/dbms/programs/server/CMakeLists.txt index bc6683d6e92..d8caa07b743 100644 --- a/dbms/programs/server/CMakeLists.txt +++ b/dbms/programs/server/CMakeLists.txt @@ -10,12 +10,16 @@ add_library (clickhouse-server-lib ${LINK_MODE} TCPHandler.cpp ) -target_link_libraries (clickhouse-server-lib clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions) +target_link_libraries (clickhouse-server-lib PRIVATE clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY}) +if (USE_POCO_NETSSL) + target_link_libraries (clickhouse-server-lib PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) +endif () + target_include_directories (clickhouse-server-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include) if (CLICKHOUSE_SPLIT_BINARY) add_executable (clickhouse-server clickhouse-server.cpp) - target_link_libraries (clickhouse-server clickhouse-server-lib) + target_link_libraries (clickhouse-server PRIVATE clickhouse-server-lib) install (TARGETS clickhouse-server ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) endif () diff --git a/dbms/programs/server/InterserverIOHTTPHandler.cpp b/dbms/programs/server/InterserverIOHTTPHandler.cpp index 3895b2d8994..3c93ee1989a 100644 --- a/dbms/programs/server/InterserverIOHTTPHandler.cpp +++ b/dbms/programs/server/InterserverIOHTTPHandler.cpp @@ -19,9 +19,6 @@ namespace DB namespace ErrorCodes { extern const int ABORTED; - extern const int POCO_EXCEPTION; - extern const int STD_EXCEPTION; - extern const int UNKNOWN_EXCEPTION; extern const int TOO_MANY_SIMULTANEOUS_QUERIES; } diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 0937cf4b84a..03d9a8268c0 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -55,6 +55,7 @@ namespace CurrentMetrics { extern const Metric Revision; + extern const Metric VersionInteger; } namespace DB @@ -66,6 +67,8 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; extern const int ARGUMENT_OUT_OF_BOUND; extern const int EXCESSIVE_ELEMENT_IN_CONFIG; + extern const int INVALID_CONFIG_PARAMETER; + extern const int SYSTEM_ERROR; } @@ -73,7 +76,7 @@ static std::string getCanonicalPath(std::string && path) { Poco::trimInPlace(path); if (path.empty()) - throw Exception("path configuration parameter is empty"); + throw Exception("path configuration parameter is empty", ErrorCodes::INVALID_CONFIG_PARAMETER); if (path.back() != '/') path += '/'; return std::move(path); @@ -108,6 +111,7 @@ int Server::main(const std::vector & /*args*/) registerStorages(); CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get()); + CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger()); /** Context contains all that query execution is dependent: * settings, available functions, data types, aggregate functions, databases... @@ -141,7 +145,7 @@ int Server::main(const std::vector & /*args*/) { LOG_TRACE(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds."); if (0 != mlockall(MCL_CURRENT)) - LOG_WARNING(log, "Failed mlockall: " + errnoToString()); + LOG_WARNING(log, "Failed mlockall: " + errnoToString(ErrorCodes::SYSTEM_ERROR)); else LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed"); } @@ -555,10 +559,10 @@ int Server::main(const std::vector & /*args*/) socket.setReceiveTimeout(settings.receive_timeout); socket.setSendTimeout(settings.send_timeout); servers.emplace_back(new Poco::Net::TCPServer( - new TCPHandlerFactory(*this, /* secure= */ true ), - server_pool, - socket, - new Poco::Net::TCPServerParams)); + new TCPHandlerFactory(*this, /* secure= */ true), + server_pool, + socket, + new Poco::Net::TCPServerParams)); LOG_INFO(log, "Listening tcp_secure: " + address.toString()); #else throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", @@ -633,7 +637,7 @@ int Server::main(const std::vector & /*args*/) message << "Available RAM = " << formatReadableSizeWithBinarySuffix(memory_amount) << ";" << " physical cores = " << getNumberOfPhysicalCPUCores() << ";" // on ARM processors it can show only enabled at current moment cores - << " threads = " << std::thread::hardware_concurrency() << "."; + << " threads = " << std::thread::hardware_concurrency() << "."; LOG_INFO(log, message.str()); } diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 5c9b3a2d86d..834369582af 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -181,7 +181,7 @@ void TCPHandler::runImpl() /// Reset the input stream, as we received an empty block while receiving external table data. /// So, the stream has been marked as cancelled and we can't read from it anymore. state.block_in.reset(); - state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker. + state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker. }); /// Processing Query @@ -718,7 +718,7 @@ bool TCPHandler::receiveData() { NamesAndTypesList columns = block.getNamesAndTypesList(); storage = StorageMemory::create(external_table_name, - ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}); + ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, ColumnComments{}}); storage->startup(); query_context.addExternalTable(external_table_name, storage); } diff --git a/dbms/programs/server/TCPHandler.h b/dbms/programs/server/TCPHandler.h index af422921f07..098019e86da 100644 --- a/dbms/programs/server/TCPHandler.h +++ b/dbms/programs/server/TCPHandler.h @@ -89,7 +89,7 @@ public: , connection_context(server.context()) , query_context(server.context()) { - server_display_name = server.config().getString("display_name", getFQDNOrHostName()); + server_display_name = server.config().getString("display_name", getFQDNOrHostName()); } void run(); diff --git a/dbms/programs/server/config.xml b/dbms/programs/server/config.xml index 051f6f7fb2b..514a081eaca 100644 --- a/dbms/programs/server/config.xml +++ b/dbms/programs/server/config.xml @@ -164,6 +164,20 @@ + + + + localhost + 9000 + + + + + localhost + 9000 + + + diff --git a/dbms/programs/server/users.xml b/dbms/programs/server/users.xml index 6f746baf2a9..979b2d3fc17 100644 --- a/dbms/programs/server/users.xml +++ b/dbms/programs/server/users.xml @@ -56,7 +56,7 @@ Each element of list has one of the following forms: IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0 - 2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::. + 2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::. Hostname. Example: server01.yandex.ru. To check access, DNS query is performed, and all received addresses compared to peer address. Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$ diff --git a/dbms/scripts/gen-bias-data.py b/dbms/scripts/gen-bias-data.py index 7edc9948e76..034cfcca7dd 100755 --- a/dbms/scripts/gen-bias-data.py +++ b/dbms/scripts/gen-bias-data.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3.4 +#!/usr/bin/python3 # -*- coding: utf-8 -*- import sys diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp b/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp index 842fe5c3e14..51bc04ea39c 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp @@ -55,10 +55,10 @@ static AggregateFunctionPtr createAggregateFunctionGroupArray(const std::string else if (parameters.size() == 1) { auto type = parameters[0].getType(); - if (type != Field::Types::Int64 && type != Field::Types::UInt64) + if (type != Field::Types::Int64 && type != Field::Types::UInt64) throw Exception("Parameter for aggregate function " + name + " should be positive number", ErrorCodes::BAD_ARGUMENTS); - if ((type == Field::Types::Int64 && parameters[0].get() < 0) || + if ((type == Field::Types::Int64 && parameters[0].get() < 0) || (type == Field::Types::UInt64 && parameters[0].get() == 0)) throw Exception("Parameter for aggregate function " + name + " should be positive number", ErrorCodes::BAD_ARGUMENTS); diff --git a/dbms/src/AggregateFunctions/AggregateFunctionStatistics.h b/dbms/src/AggregateFunctions/AggregateFunctionStatistics.h index 75ccf50e44b..82d34fc2954 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionStatistics.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionStatistics.h @@ -274,7 +274,7 @@ public: ++count; - left_mean += left_delta / count; + left_mean += left_delta / count; right_mean += right_delta / count; co_moment += (left_val - left_mean) * (right_val - old_right_mean); diff --git a/dbms/src/AggregateFunctions/AggregateFunctionUniq.cpp b/dbms/src/AggregateFunctions/AggregateFunctionUniq.cpp index 77b6c9cfb97..6b63a719b8f 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionUniq.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionUniq.cpp @@ -130,9 +130,6 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory & factory) factory.registerFunction("uniqExact", createAggregateFunctionUniq>); - - factory.registerFunction("uniqCombined", - createAggregateFunctionUniq>); } } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionUniq.h b/dbms/src/AggregateFunctions/AggregateFunctionUniq.h index 140928959a3..980d62b40ec 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionUniq.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionUniq.h @@ -22,7 +22,6 @@ #include #include -#include #include @@ -124,46 +123,6 @@ struct AggregateFunctionUniqExactData static String getName() { return "uniqExact"; } }; -template -struct AggregateFunctionUniqCombinedData -{ - using Key = UInt32; - using Set = CombinedCardinalityEstimator< - Key, - HashSet>, - 16, - 14, - 17, - TrivialHash, - UInt32, - HyperLogLogBiasEstimator, - HyperLogLogMode::FullFeatured>; - - Set set; - - static String getName() { return "uniqCombined"; } -}; - -template <> -struct AggregateFunctionUniqCombinedData -{ - using Key = UInt64; - using Set = CombinedCardinalityEstimator< - Key, - HashSet>, - 16, - 14, - 17, - TrivialHash, - UInt64, - HyperLogLogBiasEstimator, - HyperLogLogMode::FullFeatured>; - - Set set; - - static String getName() { return "uniqCombined"; } -}; - namespace detail { @@ -199,39 +158,6 @@ template <> struct AggregateFunctionUniqTraits } }; -/** Hash function for uniqCombined. - */ -template struct AggregateFunctionUniqCombinedTraits -{ - static UInt32 hash(T x) { return static_cast(intHash64(x)); } -}; - -template <> struct AggregateFunctionUniqCombinedTraits -{ - static UInt32 hash(UInt128 x) - { - return sipHash64(x); - } -}; - -template <> struct AggregateFunctionUniqCombinedTraits -{ - static UInt32 hash(Float32 x) - { - UInt64 res = ext::bit_cast(x); - return static_cast(intHash64(res)); - } -}; - -template <> struct AggregateFunctionUniqCombinedTraits -{ - static UInt32 hash(Float64 x) - { - UInt64 res = ext::bit_cast(x); - return static_cast(intHash64(res)); - } -}; - /** The structure for the delegation work to add one element to the `uniq` aggregate functions. * Used for partial specialization to add strings. @@ -255,19 +181,6 @@ struct OneAdder data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size)); } } - else if constexpr (std::is_same_v>) - { - if constexpr (!std::is_same_v) - { - const auto & value = static_cast &>(column).getData()[row_num]; - data.set.insert(AggregateFunctionUniqCombinedTraits::hash(value)); - } - else - { - StringRef value = column.getDataAt(row_num); - data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size)); - } - } else if constexpr (std::is_same_v>) { if constexpr (!std::is_same_v) @@ -387,5 +300,4 @@ public: const char * getHeaderFilePath() const override { return __FILE__; } }; - } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp b/dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp new file mode 100644 index 00000000000..90b84d3b927 --- /dev/null +++ b/dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp @@ -0,0 +1,127 @@ +#include + +#include +#include + +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ARGUMENT_OUT_OF_BOUND; +} + +namespace +{ + template + struct WithK + { + template + using AggregateFunction = AggregateFunctionUniqCombined; + + template + using AggregateFunctionVariadic = AggregateFunctionUniqCombinedVariadic; + }; + + template + AggregateFunctionPtr createAggregateFunctionWithK(const DataTypes & argument_types) + { + /// We use exact hash function if the arguments are not contiguous in memory, because only exact hash function has support for this case. + bool use_exact_hash_function = !isAllArgumentsContiguousInMemory(argument_types); + + if (argument_types.size() == 1) + { + const IDataType & argument_type = *argument_types[0]; + + AggregateFunctionPtr res(createWithNumericType::template AggregateFunction>(*argument_types[0])); + + WhichDataType which(argument_type); + if (res) + return res; + else if (which.isDate()) + return std::make_shared::template AggregateFunction>(); + else if (which.isDateTime()) + return std::make_shared::template AggregateFunction>(); + else if (which.isStringOrFixedString()) + return std::make_shared::template AggregateFunction>(); + else if (which.isUUID()) + return std::make_shared::template AggregateFunction>(); + else if (which.isTuple()) + { + if (use_exact_hash_function) + return std::make_shared::template AggregateFunctionVariadic>(argument_types); + else + return std::make_shared::template AggregateFunctionVariadic>(argument_types); + } + } + + /// "Variadic" method also works as a fallback generic case for a single argument. + if (use_exact_hash_function) + return std::make_shared::template AggregateFunctionVariadic>(argument_types); + else + return std::make_shared::template AggregateFunctionVariadic>(argument_types); + } + + AggregateFunctionPtr createAggregateFunctionUniqCombined( + const std::string & name, const DataTypes & argument_types, const Array & params) + { + /// log2 of the number of cells in HyperLogLog. + /// Reasonable default value, selected to be comparable in quality with "uniq" aggregate function. + UInt8 precision = 17; + + if (!params.empty()) + { + if (params.size() != 1) + throw Exception( + "Aggregate function " + name + " requires one parameter or less.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + UInt64 precision_param = applyVisitor(FieldVisitorConvertToNumber(), params[0]); + + // This range is hardcoded below + if (precision_param > 20 || precision_param < 12) + throw Exception( + "Parameter for aggregate function " + name + "is out or range: [12, 20].", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + precision = precision_param; + } + + if (argument_types.empty()) + throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + switch (precision) + { + case 12: + return createAggregateFunctionWithK<12>(argument_types); + case 13: + return createAggregateFunctionWithK<13>(argument_types); + case 14: + return createAggregateFunctionWithK<14>(argument_types); + case 15: + return createAggregateFunctionWithK<15>(argument_types); + case 16: + return createAggregateFunctionWithK<16>(argument_types); + case 17: + return createAggregateFunctionWithK<17>(argument_types); + case 18: + return createAggregateFunctionWithK<18>(argument_types); + case 19: + return createAggregateFunctionWithK<19>(argument_types); + case 20: + return createAggregateFunctionWithK<20>(argument_types); + } + + __builtin_unreachable(); + } + +} // namespace + +void registerAggregateFunctionUniqCombined(AggregateFunctionFactory & factory) +{ + factory.registerFunction("uniqCombined", createAggregateFunctionUniqCombined); +} + +} // namespace DB diff --git a/dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.h b/dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.h new file mode 100644 index 00000000000..99cc3a93890 --- /dev/null +++ b/dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.h @@ -0,0 +1,229 @@ +#pragma once + +#include + +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include + +namespace DB +{ +namespace detail +{ + /** Hash function for uniqCombined. + */ + template + struct AggregateFunctionUniqCombinedTraits + { + static UInt32 hash(T x) + { + return static_cast(intHash64(x)); + } + }; + + template <> + struct AggregateFunctionUniqCombinedTraits + { + static UInt32 hash(UInt128 x) + { + return sipHash64(x); + } + }; + + template <> + struct AggregateFunctionUniqCombinedTraits + { + static UInt32 hash(Float32 x) + { + UInt64 res = ext::bit_cast(x); + return static_cast(intHash64(res)); + } + }; + + template <> + struct AggregateFunctionUniqCombinedTraits + { + static UInt32 hash(Float64 x) + { + UInt64 res = ext::bit_cast(x); + return static_cast(intHash64(res)); + } + }; + +} // namespace detail + + +template +struct AggregateFunctionUniqCombinedDataWithKey +{ + // TODO(ilezhankin): pre-generate values for |UniqCombinedBiasData|, + // at the moment gen-bias-data.py script doesn't work. + + // We want to migrate from |HashSet| to |HyperLogLogCounter| when the sizes in memory become almost equal. + // The size per element in |HashSet| is sizeof(Key)*2 bytes, and the overall size of |HyperLogLogCounter| is 2^K * 6 bits. + // For Key=UInt32 we can calculate: 2^X * 4 * 2 ≤ 2^(K-3) * 6 ⇒ X ≤ K-4. + using Set = CombinedCardinalityEstimator>, 16, K - 4, K, TrivialHash, Key>; + + Set set; +}; + +template +struct AggregateFunctionUniqCombinedDataWithKey +{ + using Set = CombinedCardinalityEstimator>, + 16, + 13, + 17, + TrivialHash, + Key, + HyperLogLogBiasEstimator, + HyperLogLogMode::FullFeatured>; + + Set set; +}; + + +template +struct AggregateFunctionUniqCombinedData : public AggregateFunctionUniqCombinedDataWithKey +{ +}; + + +template +struct AggregateFunctionUniqCombinedData : public AggregateFunctionUniqCombinedDataWithKey +{ +}; + + +template +class AggregateFunctionUniqCombined final + : public IAggregateFunctionDataHelper, AggregateFunctionUniqCombined> +{ +public: + String getName() const override + { + return "uniqCombined"; + } + + DataTypePtr getReturnType() const override + { + return std::make_shared(); + } + + void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override + { + if constexpr (!std::is_same_v) + { + const auto & value = static_cast &>(*columns[0]).getData()[row_num]; + this->data(place).set.insert(detail::AggregateFunctionUniqCombinedTraits::hash(value)); + } + else + { + StringRef value = columns[0]->getDataAt(row_num); + this->data(place).set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size)); + } + } + + void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override + { + this->data(place).set.merge(this->data(rhs).set); + } + + void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override + { + this->data(place).set.write(buf); + } + + void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override + { + this->data(place).set.read(buf); + } + + void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + { + static_cast(to).getData().push_back(this->data(place).set.size()); + } + + const char * getHeaderFilePath() const override + { + return __FILE__; + } +}; + +/** For multiple arguments. To compute, hashes them. + * You can pass multiple arguments as is; You can also pass one argument - a tuple. + * But (for the possibility of efficient implementation), you can not pass several arguments, among which there are tuples. + */ +template +class AggregateFunctionUniqCombinedVariadic final : public IAggregateFunctionDataHelper, + AggregateFunctionUniqCombinedVariadic> +{ +private: + size_t num_args = 0; + +public: + explicit AggregateFunctionUniqCombinedVariadic(const DataTypes & arguments) + { + if (argument_is_tuple) + num_args = typeid_cast(*arguments[0]).getElements().size(); + else + num_args = arguments.size(); + } + + String getName() const override + { + return "uniqCombined"; + } + + DataTypePtr getReturnType() const override + { + return std::make_shared(); + } + + void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override + { + this->data(place).set.insert(typename AggregateFunctionUniqCombinedData::Set::value_type( + UniqVariadicHash::apply(num_args, columns, row_num))); + } + + void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override + { + this->data(place).set.merge(this->data(rhs).set); + } + + void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override + { + this->data(place).set.write(buf); + } + + void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override + { + this->data(place).set.read(buf); + } + + void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + { + static_cast(to).getData().push_back(this->data(place).set.size()); + } + + const char * getHeaderFilePath() const override + { + return __FILE__; + } +}; + +} // namespace DB diff --git a/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h b/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h index c9a4e6b32a7..317637b1b69 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h @@ -16,10 +16,12 @@ namespace DB { + namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; } struct ComparePairFirst final @@ -191,7 +193,7 @@ public: const auto time_arg = arguments.front().get(); if (!WhichDataType(time_arg).isDateTime() && !WhichDataType(time_arg).isUInt32()) throw Exception{"Illegal type " + time_arg->getName() + " of first argument of aggregate function " + getName() - + ", must be DateTime or UInt32"}; + + ", must be DateTime or UInt32", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; for (const auto i : ext::range(1, arguments.size())) { diff --git a/dbms/src/AggregateFunctions/CMakeLists.txt b/dbms/src/AggregateFunctions/CMakeLists.txt index ef2665d0d3f..56cc66d3913 100644 --- a/dbms/src/AggregateFunctions/CMakeLists.txt +++ b/dbms/src/AggregateFunctions/CMakeLists.txt @@ -20,5 +20,5 @@ list(REMOVE_ITEM clickhouse_aggregate_functions_headers ) add_library(clickhouse_aggregate_functions ${LINK_MODE} ${clickhouse_aggregate_functions_sources}) -target_link_libraries(clickhouse_aggregate_functions dbms) +target_link_libraries(clickhouse_aggregate_functions PRIVATE dbms) target_include_directories (clickhouse_aggregate_functions BEFORE PRIVATE ${COMMON_INCLUDE_DIR}) diff --git a/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp b/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp index 3517ad57a73..800beda1d53 100644 --- a/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp +++ b/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp @@ -21,6 +21,7 @@ void registerAggregateFunctionsStatisticsSimple(AggregateFunctionFactory &); void registerAggregateFunctionSum(AggregateFunctionFactory &); void registerAggregateFunctionSumMap(AggregateFunctionFactory &); void registerAggregateFunctionsUniq(AggregateFunctionFactory &); +void registerAggregateFunctionUniqCombined(AggregateFunctionFactory &); void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &); void registerAggregateFunctionTopK(AggregateFunctionFactory &); void registerAggregateFunctionsBitwise(AggregateFunctionFactory &); @@ -55,6 +56,7 @@ void registerAggregateFunctions() registerAggregateFunctionSum(factory); registerAggregateFunctionSumMap(factory); registerAggregateFunctionsUniq(factory); + registerAggregateFunctionUniqCombined(factory); registerAggregateFunctionUniqUpTo(factory); registerAggregateFunctionTopK(factory); registerAggregateFunctionsBitwise(factory); diff --git a/dbms/src/Client/CMakeLists.txt b/dbms/src/Client/CMakeLists.txt index 907d9787bf8..cc8b3f8eda8 100644 --- a/dbms/src/Client/CMakeLists.txt +++ b/dbms/src/Client/CMakeLists.txt @@ -4,3 +4,5 @@ #add_library(clickhouse_client ${LINK_MODE} ${clickhouse_client_headers} ${clickhouse_client_sources}) #target_link_libraries (clickhouse_client clickhouse_common_io ${Poco_Net_LIBRARY}) #target_include_directories (clickhouse_client PRIVATE ${DBMS_INCLUDE_DIR}) + +add_subdirectory(tests) diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index ce6246fba3a..60fd070064b 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -56,7 +56,7 @@ void Connection::connect() disconnect(); LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << (default_database.empty() ? "(not specified)" : default_database) << ". User: " << user - << (static_cast(secure) ? ". Secure" : "") << (static_cast(compression) ? "" : ". Uncompressed") ); + << (static_cast(secure) ? ". Secure" : "") << (static_cast(compression) ? "" : ". Uncompressed")); if (static_cast(secure)) { @@ -108,14 +108,14 @@ void Connection::connect() disconnect(); /// Add server address to exception. Also Exception will remember stack trace. It's a pity that more precise exception type is lost. - throw NetException(e.displayText(), "(" + getDescription() + ")", ErrorCodes::NETWORK_ERROR); + throw NetException(e.displayText() + " (" + getDescription() + ")", ErrorCodes::NETWORK_ERROR); } catch (Poco::TimeoutException & e) { disconnect(); /// Add server address to exception. Also Exception will remember stack trace. It's a pity that more precise exception type is lost. - throw NetException(e.displayText(), "(" + getDescription() + ")", ErrorCodes::SOCKET_TIMEOUT); + throw NetException(e.displayText() + " (" + getDescription() + ")", ErrorCodes::SOCKET_TIMEOUT); } } @@ -696,7 +696,7 @@ void Connection::setDescription() { auto resolved_address = getResolvedAddress(); description = host + ":" + toString(resolved_address.port()); - auto ip_address = resolved_address.host().toString(); + auto ip_address = resolved_address.host().toString(); if (host != ip_address) description += ", " + ip_address; diff --git a/dbms/src/Client/ConnectionPoolWithFailover.cpp b/dbms/src/Client/ConnectionPoolWithFailover.cpp index d71d1f08339..7687742169f 100644 --- a/dbms/src/Client/ConnectionPoolWithFailover.cpp +++ b/dbms/src/Client/ConnectionPoolWithFailover.cpp @@ -203,7 +203,7 @@ ConnectionPoolWithFailover::tryGetEntry( LOG_TRACE( log, "Server " << result.entry->getDescription() << " has unacceptable replica delay " << "for table " << table_to_check->database << "." << table_to_check->table - << ": " << delay); + << ": " << delay); ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica); } } diff --git a/dbms/src/Client/tests/CMakeLists.txt b/dbms/src/Client/tests/CMakeLists.txt new file mode 100644 index 00000000000..f4471136a8a --- /dev/null +++ b/dbms/src/Client/tests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(test-connect test_connect.cpp) +target_link_libraries (test-connect dbms) diff --git a/dbms/src/Client/tests/test_connect.cpp b/dbms/src/Client/tests/test_connect.cpp new file mode 100644 index 00000000000..75eb606cc97 --- /dev/null +++ b/dbms/src/Client/tests/test_connect.cpp @@ -0,0 +1,59 @@ +#include +#include +#include + +#include +#include +#include +#include + + +/** In a loop it connects to the server and immediately breaks the connection. +  * Using the SO_LINGER option, we ensure that the connection is terminated by sending a RST packet (not FIN). +  * This behavior causes a bug in the TCPServer implementation in the Poco library. +  */ +int main(int argc, char ** argv) +try +{ + for (size_t i = 0, num_iters = argc >= 2 ? DB::parse(argv[1]) : 1; i < num_iters; ++i) + { + std::cerr << "."; + + Poco::Net::SocketAddress address("localhost", 9000); + + int fd = socket(PF_INET, SOCK_STREAM, IPPROTO_IP); + + if (fd < 0) + DB::throwFromErrno("Cannot create socket", 0); + + linger linger_value; + linger_value.l_onoff = 1; + linger_value.l_linger = 0; + + if (0 != setsockopt(fd, SOL_SOCKET, SO_LINGER, &linger_value, sizeof(linger_value))) + DB::throwFromErrno("Cannot set linger", 0); + + try + { + int res = connect(fd, address.addr(), address.length()); + + if (res != 0 && errno != EINPROGRESS && errno != EWOULDBLOCK) + { + close(fd); + DB::throwFromErrno("Cannot connect", 0); + } + + close(fd); + } + catch (const Poco::Exception & e) + { + std::cerr << e.displayText() << "\n"; + } + } + + std::cerr << "\n"; +} +catch (const Poco::Exception & e) +{ + std::cerr << e.displayText() << "\n"; +} diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index b556e608a86..4a9b54b08d5 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -437,11 +437,11 @@ ColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hin auto res = ColumnArray::create(data->cloneEmpty()); const ColumnString & src_string = typeid_cast(*data); - const ColumnString::Chars_t & src_chars = src_string.getChars(); + const ColumnString::Chars & src_chars = src_string.getChars(); const Offsets & src_string_offsets = src_string.getOffsets(); const Offsets & src_offsets = getOffsets(); - ColumnString::Chars_t & res_chars = typeid_cast(res->getData()).getChars(); + ColumnString::Chars & res_chars = typeid_cast(res->getData()).getChars(); Offsets & res_string_offsets = typeid_cast(res->getData()).getOffsets(); Offsets & res_offsets = res->getOffsets(); @@ -697,20 +697,20 @@ ColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const if (replicate_offsets.empty()) return cloneEmpty(); - if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); - if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); - if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); - if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); - if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); - if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); - if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); - if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); - if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); - if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); - if (typeid_cast(data.get())) return replicateString(replicate_offsets); - if (typeid_cast(data.get())) return replicateConst(replicate_offsets); - if (typeid_cast(data.get())) return replicateNullable(replicate_offsets); - if (typeid_cast(data.get())) return replicateTuple(replicate_offsets); + if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); + if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); + if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); + if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); + if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); + if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); + if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); + if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); + if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); + if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); + if (typeid_cast(data.get())) return replicateString(replicate_offsets); + if (typeid_cast(data.get())) return replicateConst(replicate_offsets); + if (typeid_cast(data.get())) return replicateNullable(replicate_offsets); + if (typeid_cast(data.get())) return replicateTuple(replicate_offsets); return replicateGeneric(replicate_offsets); } @@ -781,11 +781,11 @@ ColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const ColumnArray & res_ = static_cast(*res); const ColumnString & src_string = typeid_cast(*data); - const ColumnString::Chars_t & src_chars = src_string.getChars(); + const ColumnString::Chars & src_chars = src_string.getChars(); const Offsets & src_string_offsets = src_string.getOffsets(); const Offsets & src_offsets = getOffsets(); - ColumnString::Chars_t & res_chars = typeid_cast(res_.getData()).getChars(); + ColumnString::Chars & res_chars = typeid_cast(res_.getData()).getChars(); Offsets & res_string_offsets = typeid_cast(res_.getData()).getOffsets(); Offsets & res_offsets = res_.getOffsets(); diff --git a/dbms/src/Columns/ColumnDecimal.cpp b/dbms/src/Columns/ColumnDecimal.cpp index e2cb798360e..e6124f6f6d3 100644 --- a/dbms/src/Columns/ColumnDecimal.cpp +++ b/dbms/src/Columns/ColumnDecimal.cpp @@ -21,7 +21,7 @@ namespace ErrorCodes } template -int ColumnDecimal::compareAt(size_t n, size_t m, const IColumn & rhs_, int ) const +int ColumnDecimal::compareAt(size_t n, size_t m, const IColumn & rhs_, int) const { auto other = static_cast(rhs_); const T & a = data[n]; diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp index 130eb155337..b91c27becd7 100644 --- a/dbms/src/Columns/ColumnFixedString.cpp +++ b/dbms/src/Columns/ColumnFixedString.cpp @@ -248,7 +248,7 @@ ColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) con auto res = ColumnFixedString::create(n); - Chars_t & res_chars = res->chars; + Chars & res_chars = res->chars; res_chars.resize(n * limit); @@ -274,7 +274,7 @@ ColumnPtr ColumnFixedString::indexImpl(const PaddedPODArray & indexes, siz auto res = ColumnFixedString::create(n); - Chars_t & res_chars = res->chars; + Chars & res_chars = res->chars; res_chars.resize(n * limit); @@ -296,7 +296,7 @@ ColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const if (0 == col_size) return res; - Chars_t & res_chars = res->chars; + Chars & res_chars = res->chars; res_chars.resize(n * offsets.back()); Offset curr_offset = 0; diff --git a/dbms/src/Columns/ColumnFixedString.h b/dbms/src/Columns/ColumnFixedString.h index e8ce153e1a1..fcfed7481e8 100644 --- a/dbms/src/Columns/ColumnFixedString.h +++ b/dbms/src/Columns/ColumnFixedString.h @@ -17,14 +17,14 @@ class ColumnFixedString final : public COWPtrHelper public: friend class COWPtrHelper; - using Chars_t = PaddedPODArray; + using Chars = PaddedPODArray; private: /// Bytes of rows, laid in succession. The strings are stored without a trailing zero byte. /** NOTE It is required that the offset and type of chars in the object be the same as that of `data in ColumnUInt8`. * Used in `packFixed` function (AggregationCommon.h) */ - Chars_t chars; + Chars chars; /// The size of the rows. const size_t n; @@ -138,8 +138,8 @@ public: /// Specialized part of interface, not from IColumn. - Chars_t & getChars() { return chars; } - const Chars_t & getChars() const { return chars; } + Chars & getChars() { return chars; } + const Chars & getChars() const { return chars; } size_t getN() const { return n; } }; diff --git a/dbms/src/Columns/ColumnFunction.cpp b/dbms/src/Columns/ColumnFunction.cpp index 852f6f03adf..8bc239eb657 100644 --- a/dbms/src/Columns/ColumnFunction.cpp +++ b/dbms/src/Columns/ColumnFunction.cpp @@ -183,7 +183,7 @@ void ColumnFunction::appendArgument(const ColumnWithTypeAndName & column) auto index = captured_columns.size(); if (!column.type->equals(*argumnet_types[index])) throw Exception("Cannot capture column " + std::to_string(argumnet_types.size()) + - "because it has incompatible type: got " + column.type->getName() + + " because it has incompatible type: got " + column.type->getName() + ", but " + argumnet_types[index]->getName() + " is expected.", ErrorCodes::LOGICAL_ERROR); captured_columns.push_back(column); diff --git a/dbms/src/Columns/ColumnLowCardinality.cpp b/dbms/src/Columns/ColumnLowCardinality.cpp index 873ecfa0613..da6bcfa6597 100644 --- a/dbms/src/Columns/ColumnLowCardinality.cpp +++ b/dbms/src/Columns/ColumnLowCardinality.cpp @@ -604,7 +604,7 @@ void ColumnLowCardinality::Index::check(size_t /*max_dictionary_size*/) void ColumnLowCardinality::Index::checkSizeOfType() { if (size_of_type != getSizeOfIndexType(*positions, size_of_type)) - throw Exception("Invalid size of type. Expected " + toString(8 * size_of_type) + + throw Exception("Invalid size of type. Expected " + toString(8 * size_of_type) + ", but positions are " + positions->getName(), ErrorCodes::LOGICAL_ERROR); } diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp index 64806d56d72..a7e1cb1576d 100644 --- a/dbms/src/Columns/ColumnString.cpp +++ b/dbms/src/Columns/ColumnString.cpp @@ -101,7 +101,7 @@ ColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) co auto res = ColumnString::create(); - Chars_t & res_chars = res->chars; + Chars & res_chars = res->chars; Offsets & res_offsets = res->offsets; filterArraysImpl(chars, offsets, res_chars, res_offsets, filt, result_size_hint); @@ -126,7 +126,7 @@ ColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const auto res = ColumnString::create(); - Chars_t & res_chars = res->chars; + Chars & res_chars = res->chars; Offsets & res_offsets = res->offsets; if (limit == size) @@ -202,7 +202,7 @@ ColumnPtr ColumnString::indexImpl(const PaddedPODArray & indexes, size_t l auto res = ColumnString::create(); - Chars_t & res_chars = res->chars; + Chars & res_chars = res->chars; Offsets & res_offsets = res->offsets; size_t new_chars_size = 0; @@ -287,7 +287,7 @@ ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const if (0 == col_size) return res; - Chars_t & res_chars = res->chars; + Chars & res_chars = res->chars; Offsets & res_offsets = res->offsets; res_chars.reserve(chars.size() / col_size * replicate_offsets.back()); res_offsets.reserve(replicate_offsets.back()); diff --git a/dbms/src/Columns/ColumnString.h b/dbms/src/Columns/ColumnString.h index 767d67a7385..bdf9be13773 100644 --- a/dbms/src/Columns/ColumnString.h +++ b/dbms/src/Columns/ColumnString.h @@ -19,7 +19,7 @@ namespace DB class ColumnString final : public COWPtrHelper { public: - using Chars_t = PaddedPODArray; + using Chars = PaddedPODArray; private: friend class COWPtrHelper; @@ -29,7 +29,7 @@ private: /// Bytes of strings, placed contiguously. /// For convenience, every string ends with terminating zero byte. Note that strings could contain zero bytes in the middle. - Chars_t chars; + Chars chars; size_t ALWAYS_INLINE offsetAt(size_t i) const { return i == 0 ? 0 : offsets[i - 1]; } @@ -245,8 +245,8 @@ public: bool canBeInsideNullable() const override { return true; } - Chars_t & getChars() { return chars; } - const Chars_t & getChars() const { return chars; } + Chars & getChars() { return chars; } + const Chars & getChars() const { return chars; } Offsets & getOffsets() { return offsets; } const Offsets & getOffsets() const { return offsets; } diff --git a/dbms/src/Columns/ColumnUnique.h b/dbms/src/Columns/ColumnUnique.h index be5d71c3e29..7fb01620fbd 100644 --- a/dbms/src/Columns/ColumnUnique.h +++ b/dbms/src/Columns/ColumnUnique.h @@ -81,12 +81,14 @@ public: { return column_holder->allocatedBytes() + index.allocatedBytes() - + (cached_null_mask ? cached_null_mask->allocatedBytes() : 0); + + (nested_null_mask ? nested_null_mask->allocatedBytes() : 0); } void forEachSubcolumn(IColumn::ColumnCallback callback) override { callback(column_holder); index.setColumn(getRawColumnPtr()); + if (is_nullable) + nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask); } const UInt64 * tryGetSavedHash() const override { return index.tryGetSavedHash(); } @@ -100,8 +102,8 @@ private: ReverseIndex index; /// For DataTypeNullable, stores null map. - mutable ColumnPtr cached_null_mask; - mutable ColumnPtr cached_column_nullable; + ColumnPtr nested_null_mask; + ColumnPtr nested_column_nullable; class IncrementalHash { @@ -118,6 +120,9 @@ private: mutable IncrementalHash hash; + void createNullMask(); + void updateNullMask(); + static size_t numSpecialValues(bool is_nullable) { return is_nullable ? 2 : 1; } size_t numSpecialValues() const { return numSpecialValues(is_nullable); } @@ -148,6 +153,7 @@ ColumnUnique::ColumnUnique(const ColumnUnique & other) , index(numSpecialValues(is_nullable), 0) { index.setColumn(getRawColumnPtr()); + createNullMask(); } template @@ -158,6 +164,7 @@ ColumnUnique::ColumnUnique(const IDataType & type) const auto & holder_type = is_nullable ? *static_cast(type).getNestedType() : type; column_holder = holder_type.createColumn()->cloneResized(numSpecialValues()); index.setColumn(getRawColumnPtr()); + createNullMask(); } template @@ -172,32 +179,51 @@ ColumnUnique::ColumnUnique(MutableColumnPtr && holder, bool is_nulla throw Exception("Holder column for ColumnUnique can't be nullable.", ErrorCodes::ILLEGAL_COLUMN); index.setColumn(getRawColumnPtr()); + createNullMask(); +} + +template +void ColumnUnique::createNullMask() +{ + if (is_nullable) + { + size_t size = getRawColumnPtr()->size(); + if (!nested_null_mask) + { + ColumnUInt8::MutablePtr null_mask = ColumnUInt8::create(size, UInt8(0)); + null_mask->getData()[getNullValueIndex()] = 1; + nested_null_mask = std::move(null_mask); + nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask); + } + else + throw Exception("Null mask for ColumnUnique is already created.", ErrorCodes::LOGICAL_ERROR); + } +} + +template +void ColumnUnique::updateNullMask() +{ + if (is_nullable) + { + if (!nested_null_mask) + throw Exception("Null mask for ColumnUnique is was not created.", ErrorCodes::LOGICAL_ERROR); + + size_t size = getRawColumnPtr()->size(); + + if (nested_null_mask->size() != size) + { + IColumn & null_mask = nested_null_mask->assumeMutableRef(); + static_cast(null_mask).getData().resize_fill(size); + } + } } template const ColumnPtr & ColumnUnique::getNestedColumn() const { if (is_nullable) - { - size_t size = getRawColumnPtr()->size(); - if (!cached_null_mask) - { - ColumnUInt8::MutablePtr null_mask = ColumnUInt8::create(size, UInt8(0)); - null_mask->getData()[getNullValueIndex()] = 1; - cached_null_mask = std::move(null_mask); - cached_column_nullable = ColumnNullable::create(column_holder, cached_null_mask); - } + return nested_column_nullable; - if (cached_null_mask->size() != size) - { - MutableColumnPtr null_mask = (*std::move(cached_null_mask)).mutate(); - static_cast(*null_mask).getData().resize_fill(size); - cached_null_mask = std::move(null_mask); - cached_column_nullable = ColumnNullable::create(column_holder, cached_null_mask); - } - - return cached_column_nullable; - } return column_holder; } @@ -227,6 +253,8 @@ size_t ColumnUnique::uniqueInsert(const Field & x) if (pos != prev_size) column->popBack(1); + updateNullMask(); + return pos; } @@ -260,6 +288,8 @@ size_t ColumnUnique::uniqueInsertData(const char * pos, size_t lengt index.insertFromLastRow(); } + updateNullMask(); + return insertion_point; } @@ -288,6 +318,8 @@ size_t ColumnUnique::uniqueInsertDataWithTerminatingZero(const char if (position != prev_size) column->popBack(1); + updateNullMask(); + return static_cast(position); } @@ -305,7 +337,7 @@ StringRef ColumnUnique::serializeValueIntoArena(size_t n, Arena & ar size_t nested_size = 0; - if (n == getNullValueIndex()) + if (n != getNullValueIndex()) nested_size = column_holder->serializeValueIntoArena(n, arena, begin).size; return StringRef(pos, sizeof(null_flag) + nested_size); @@ -343,6 +375,8 @@ size_t ColumnUnique::uniqueDeserializeAndInsertFromArena(const char if (index_pos != prev_size) column->popBack(1); + updateNullMask(); + return static_cast(index_pos); } @@ -533,6 +567,8 @@ MutableColumnPtr ColumnUnique::uniqueInsertRangeFrom(const IColumn & if (!positions_column) throw Exception("Can't find index type for ColumnUnique", ErrorCodes::LOGICAL_ERROR); + updateNullMask(); + return positions_column; } @@ -577,6 +613,8 @@ IColumnUnique::IndexesWithOverflow ColumnUnique::uniqueInsertRangeWi if (!positions_column) throw Exception("Can't find index type for ColumnUnique", ErrorCodes::LOGICAL_ERROR); + updateNullMask(); + IColumnUnique::IndexesWithOverflow indexes_with_overflow; indexes_with_overflow.indexes = std::move(positions_column); indexes_with_overflow.overflowed_keys = std::move(overflowed_keys); diff --git a/dbms/src/Columns/FilterDescription.cpp b/dbms/src/Columns/FilterDescription.cpp index 104599ba974..f9fb39a9f47 100644 --- a/dbms/src/Columns/FilterDescription.cpp +++ b/dbms/src/Columns/FilterDescription.cpp @@ -72,7 +72,7 @@ FilterDescription::FilterDescription(const IColumn & column_) ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER); const NullMap & null_map = nullable_column->getNullMapData(); - IColumn::Filter & res = concrete_column->getData(); + IColumn::Filter & res = concrete_column->getData(); size_t size = res.size(); for (size_t i = 0; i < size; ++i) diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h index 82a68662588..088be128e8d 100644 --- a/dbms/src/Columns/IColumn.h +++ b/dbms/src/Columns/IColumn.h @@ -377,7 +377,7 @@ protected: }; using ColumnPtr = IColumn::Ptr; -using MutableColumnPtr = IColumn::MutablePtr; +using MutableColumnPtr = IColumn::MutablePtr; using Columns = std::vector; using MutableColumns = std::vector; diff --git a/dbms/src/Columns/ReverseIndex.h b/dbms/src/Columns/ReverseIndex.h index cf735343167..a2b791fed6e 100644 --- a/dbms/src/Columns/ReverseIndex.h +++ b/dbms/src/Columns/ReverseIndex.h @@ -83,7 +83,7 @@ namespace template - struct ReverseIndexHashTableCell + struct ReverseIndexHashTableCell : public HashTableCell> { using Base = HashTableCell>; @@ -272,7 +272,7 @@ public: auto hash = calcHashes(); ptr = &hash->getData()[0]; UInt64 * expected = nullptr; - if(saved_hash_ptr.compare_exchange_strong(expected, ptr)) + if (saved_hash_ptr.compare_exchange_strong(expected, ptr)) saved_hash = std::move(hash); else ptr = expected; diff --git a/dbms/src/Common/ClickHouseRevision.cpp b/dbms/src/Common/ClickHouseRevision.cpp index ea48dcdc54f..5513922a655 100644 --- a/dbms/src/Common/ClickHouseRevision.cpp +++ b/dbms/src/Common/ClickHouseRevision.cpp @@ -4,4 +4,5 @@ namespace ClickHouseRevision { unsigned get() { return VERSION_REVISION; } + unsigned getVersionInteger() { return VERSION_INTEGER; } } diff --git a/dbms/src/Common/ClickHouseRevision.h b/dbms/src/Common/ClickHouseRevision.h index 7c147fe6f8b..1d097a5bf89 100644 --- a/dbms/src/Common/ClickHouseRevision.h +++ b/dbms/src/Common/ClickHouseRevision.h @@ -3,4 +3,5 @@ namespace ClickHouseRevision { unsigned get(); + unsigned getVersionInteger(); } diff --git a/dbms/src/Common/Config/CMakeLists.txt b/dbms/src/Common/Config/CMakeLists.txt index 614e70e757b..a1bb2790fdf 100644 --- a/dbms/src/Common/Config/CMakeLists.txt +++ b/dbms/src/Common/Config/CMakeLists.txt @@ -1,9 +1,8 @@ - include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) add_headers_and_sources(clickhouse_common_config .) add_library(clickhouse_common_config ${LINK_MODE} ${clickhouse_common_config_headers} ${clickhouse_common_config_sources}) -target_link_libraries (clickhouse_common_config clickhouse_common_zookeeper string_utils ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY}) -target_include_directories (clickhouse_common_config PRIVATE ${DBMS_INCLUDE_DIR}) +target_link_libraries(clickhouse_common_config PUBLIC common PRIVATE clickhouse_common_zookeeper string_utils PUBLIC ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY}) +target_include_directories(clickhouse_common_config PUBLIC ${DBMS_INCLUDE_DIR}) diff --git a/dbms/src/Common/Config/ConfigProcessor.cpp b/dbms/src/Common/Config/ConfigProcessor.cpp index eebf9b75a35..cc7660f9641 100644 --- a/dbms/src/Common/Config/ConfigProcessor.cpp +++ b/dbms/src/Common/Config/ConfigProcessor.cpp @@ -260,10 +260,10 @@ void ConfigProcessor::doIncludesRecursive( /// Substitute for the number extracted from the hostname only if there is an /// empty tag without attributes in the original file. - if ( node->nodeName() == "layer" && - !node->hasAttributes() && - !node->hasChildNodes() && - node->nodeValue().empty()) + if (node->nodeName() == "layer" + && !node->hasAttributes() + && !node->hasChildNodes() + && node->nodeValue().empty()) { NodePtr new_node = config->createTextNode(layerFromHost()); node->appendChild(new_node); diff --git a/dbms/src/Common/CounterInFile.h b/dbms/src/Common/CounterInFile.h index 6ea34362a59..2c831e33302 100644 --- a/dbms/src/Common/CounterInFile.h +++ b/dbms/src/Common/CounterInFile.h @@ -18,7 +18,15 @@ #include -#define SMALL_READ_WRITE_BUFFER_SIZE 16 +namespace DB +{ + namespace ErrorCodes + { + extern const int CANNOT_OPEN_FILE; + extern const int CANNOT_READ_ALL_DATA; + extern const int ATTEMPT_TO_READ_AFTER_EOF; + } +} /** Stores a number in the file. @@ -26,6 +34,9 @@ */ class CounterInFile { +private: + static inline constexpr size_t SMALL_READ_WRITE_BUFFER_SIZE = 16; + public: /// path - the name of the file, including the path CounterInFile(const std::string & path_) : path(path_) {} @@ -56,13 +67,13 @@ public: int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0666); if (-1 == fd) - DB::throwFromErrno("Cannot open file " + path); + DB::throwFromErrno("Cannot open file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE); try { int flock_ret = flock(fd, LOCK_EX); if (-1 == flock_ret) - DB::throwFromErrno("Cannot lock file " + path); + DB::throwFromErrno("Cannot lock file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE); if (!file_doesnt_exists) { @@ -130,7 +141,7 @@ public: int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0666); if (-1 == fd) - DB::throwFromErrno("Cannot open file " + path); + DB::throwFromErrno("Cannot open file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE); try { @@ -178,6 +189,3 @@ private: std::string path; std::mutex mutex; }; - - -#undef SMALL_READ_WRITE_BUFFER_SIZE diff --git a/dbms/src/Common/CurrentMetrics.cpp b/dbms/src/Common/CurrentMetrics.cpp index 59b26f4b5e1..2f8346d554e 100644 --- a/dbms/src/Common/CurrentMetrics.cpp +++ b/dbms/src/Common/CurrentMetrics.cpp @@ -39,7 +39,8 @@ M(StorageBufferRows, "Number of rows in buffers of Buffer tables") \ M(StorageBufferBytes, "Number of bytes in buffers of Buffer tables") \ M(DictCacheRequests, "Number of requests in fly to data sources of dictionaries of cache type.") \ - M(Revision, "Revision of the server. It is a number incremented for every release or release candidate.") \ + M(Revision, "Revision of the server. It is a number incremented for every release or release candidate except patch releases.") \ + M(VersionInteger, "Version of the server in a single integer number in base-1000. For example, version 11.22.33 is translated to 11022033.") \ M(RWLockWaitingReaders, "Number of threads waiting for read on a table RWLock.") \ M(RWLockWaitingWriters, "Number of threads waiting for write on a table RWLock.") \ M(RWLockActiveReaders, "Number of threads holding read lock in a table RWLock.") \ diff --git a/dbms/src/Common/DNSResolver.cpp b/dbms/src/Common/DNSResolver.cpp index c82f302d7b2..bba3305684c 100644 --- a/dbms/src/Common/DNSResolver.cpp +++ b/dbms/src/Common/DNSResolver.cpp @@ -107,7 +107,7 @@ Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host_an Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host, UInt16 port) { - return !impl->disable_cache ? Poco::Net::SocketAddress(impl->cache_host(host), port) : Poco::Net::SocketAddress(host, port); + return !impl->disable_cache ? Poco::Net::SocketAddress(impl->cache_host(host), port) : Poco::Net::SocketAddress(host, port); } void DNSResolver::dropCache() diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index 3560bcee9af..719e593e3b1 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -397,6 +397,10 @@ namespace ErrorCodes extern const int CANNOT_UPDATE_COLUMN = 420; extern const int CANNOT_ADD_DIFFERENT_AGGREGATE_STATES = 421; extern const int UNSUPPORTED_URI_SCHEME = 422; + extern const int CANNOT_GETTIMEOFDAY = 423; + extern const int CANNOT_LINK = 424; + extern const int SYSTEM_ERROR = 425; + extern const int NULL_POINTER_DEREFERENCE = 426; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Common/Exception.cpp b/dbms/src/Common/Exception.cpp index 606f180e9d7..a7bfbd64424 100644 --- a/dbms/src/Common/Exception.cpp +++ b/dbms/src/Common/Exception.cpp @@ -52,7 +52,7 @@ std::string errnoToString(int code, int e) void throwFromErrno(const std::string & s, int code, int e) { - throw ErrnoException(s + ", " + errnoToString(code, e)); + throw ErrnoException(s + ", " + errnoToString(code, e), code, e); } void tryLogCurrentException(const char * log_name, const std::string & start_of_message) diff --git a/dbms/src/Common/Exception.h b/dbms/src/Common/Exception.h index 4e3bdc8bafe..d5e9d3112e1 100644 --- a/dbms/src/Common/Exception.h +++ b/dbms/src/Common/Exception.h @@ -14,19 +14,25 @@ namespace Poco { class Logger; } namespace DB { +namespace ErrorCodes +{ + extern const int POCO_EXCEPTION; +} + class Exception : public Poco::Exception { public: Exception() {} /// For deferred initialization. - Exception(const std::string & msg, int code = 0) : Poco::Exception(msg, code) {} - Exception(const std::string & msg, const std::string & arg, int code = 0) : Poco::Exception(msg, arg, code) {} - Exception(const std::string & msg, const Exception & exc, int code = 0) : Poco::Exception(msg, exc, code), trace(exc.trace) {} - explicit Exception(const Poco::Exception & exc) : Poco::Exception(exc.displayText()) {} + Exception(const std::string & msg, int code) : Poco::Exception(msg, code) {} + Exception(const std::string & msg, const Exception & nested_exception, int code) + : Poco::Exception(msg, nested_exception, code), trace(nested_exception.trace) {} - const char * name() const throw() override { return "DB::Exception"; } - const char * className() const throw() override { return "DB::Exception"; } - DB::Exception * clone() const override { return new DB::Exception(*this); } + enum CreateFromPocoTag { CreateFromPoco }; + Exception(CreateFromPocoTag, const Poco::Exception & exc) : Poco::Exception(exc.displayText(), ErrorCodes::POCO_EXCEPTION) {} + + Exception * clone() const override { return new Exception(*this); } void rethrow() const override { throw *this; } + const char * name() const throw() override { return "DB::Exception"; } /// Add something to the existing message. void addMessage(const std::string & arg) { extendedMessage(arg); } @@ -35,6 +41,8 @@ public: private: StackTrace trace; + + const char * className() const throw() override { return "DB::Exception"; } }; @@ -42,25 +50,27 @@ private: class ErrnoException : public Exception { public: - ErrnoException(const std::string & msg, int code = 0, int saved_errno_ = 0) + ErrnoException(const std::string & msg, int code, int saved_errno_) : Exception(msg, code), saved_errno(saved_errno_) {} - ErrnoException(const std::string & msg, const std::string & arg, int code = 0, int saved_errno_ = 0) - : Exception(msg, arg, code), saved_errno(saved_errno_) {} - ErrnoException(const std::string & msg, const Exception & exc, int code = 0, int saved_errno_ = 0) - : Exception(msg, exc, code), saved_errno(saved_errno_) {} + + ErrnoException * clone() const override { return new ErrnoException(*this); } + void rethrow() const override { throw *this; } int getErrno() const { return saved_errno; } private: int saved_errno; + + const char * name() const throw() override { return "DB::ErrnoException"; } + const char * className() const throw() override { return "DB::ErrnoException"; } }; using Exceptions = std::vector; -std::string errnoToString(int code = 0, int the_errno = errno); -[[noreturn]] void throwFromErrno(const std::string & s, int code = 0, int the_errno = errno); +std::string errnoToString(int code, int the_errno = errno); +[[noreturn]] void throwFromErrno(const std::string & s, int code, int the_errno = errno); /** Try to write an exception to the log (and forget about it). diff --git a/dbms/src/Common/FieldVisitors.h b/dbms/src/Common/FieldVisitors.h index 6b5c0730f93..19b4e583338 100644 --- a/dbms/src/Common/FieldVisitors.h +++ b/dbms/src/Common/FieldVisitors.h @@ -18,7 +18,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -UInt128 stringToUUID(const String & ); +UInt128 stringToUUID(const String &); /** StaticVisitor (and its descendants) - class with overloaded operator() for all types of fields. @@ -38,16 +38,16 @@ typename std::decay_t::ResultType applyVisitor(Visitor && visitor, F && { switch (field.getType()) { - case Field::Types::Null: return visitor(field.template get()); - case Field::Types::UInt64: return visitor(field.template get()); + case Field::Types::Null: return visitor(field.template get()); + case Field::Types::UInt64: return visitor(field.template get()); case Field::Types::UInt128: return visitor(field.template get()); - case Field::Types::Int64: return visitor(field.template get()); + case Field::Types::Int64: return visitor(field.template get()); case Field::Types::Float64: return visitor(field.template get()); - case Field::Types::String: return visitor(field.template get()); - case Field::Types::Array: return visitor(field.template get()); - case Field::Types::Tuple: return visitor(field.template get()); - case Field::Types::Decimal32: return visitor(field.template get>()); - case Field::Types::Decimal64: return visitor(field.template get>()); + case Field::Types::String: return visitor(field.template get()); + case Field::Types::Array: return visitor(field.template get()); + case Field::Types::Tuple: return visitor(field.template get()); + case Field::Types::Decimal32: return visitor(field.template get>()); + case Field::Types::Decimal64: return visitor(field.template get>()); case Field::Types::Decimal128: return visitor(field.template get>()); default: diff --git a/dbms/src/Common/HashTable/HashTable.h b/dbms/src/Common/HashTable/HashTable.h index b35d3b41acc..b78ce798ba2 100644 --- a/dbms/src/Common/HashTable/HashTable.h +++ b/dbms/src/Common/HashTable/HashTable.h @@ -229,8 +229,8 @@ public: void setHasZero() { has_zero = true; } void clearHasZero() { has_zero = false; } - Cell * zeroValue() { return reinterpret_cast(&zero_value_storage); } - const Cell * zeroValue() const { return reinterpret_cast(&zero_value_storage); } + Cell * zeroValue() { return reinterpret_cast(&zero_value_storage); } + const Cell * zeroValue() const { return reinterpret_cast(&zero_value_storage); } }; template @@ -240,8 +240,8 @@ struct ZeroValueStorage void setHasZero() { throw DB::Exception("HashTable: logical error", DB::ErrorCodes::LOGICAL_ERROR); } void clearHasZero() {} - Cell * zeroValue() { return nullptr; } - const Cell * zeroValue() const { return nullptr; } + Cell * zeroValue() { return nullptr; } + const Cell * zeroValue() const { return nullptr; } }; diff --git a/dbms/src/Common/HashTable/SmallTable.h b/dbms/src/Common/HashTable/SmallTable.h index 73c2b358652..27dc8c00332 100644 --- a/dbms/src/Common/HashTable/SmallTable.h +++ b/dbms/src/Common/HashTable/SmallTable.h @@ -3,6 +3,15 @@ #include +namespace DB +{ + namespace ErrorCodes + { + extern const int INCORRECT_DATA; + } +} + + /** Replacement of the hash table for a small number (<10) of keys. * Implemented as an array with linear search. * The array is located inside the object. @@ -13,7 +22,6 @@ * you should check if the table is not full, * and do a `fallback` in this case (for example, use a real hash table). */ - template < typename Key, @@ -86,7 +94,7 @@ public: DB::readVarUInt(size, in); if (size > capacity) - throw DB::Exception("Illegal size"); + throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA); is_initialized = true; } @@ -306,7 +314,7 @@ public: DB::readVarUInt(new_size, rb); if (new_size > capacity) - throw DB::Exception("Illegal size"); + throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA); for (size_t i = 0; i < new_size; ++i) buf[i].read(rb); @@ -324,7 +332,7 @@ public: DB::readText(new_size, rb); if (new_size > capacity) - throw DB::Exception("Illegal size"); + throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA); for (size_t i = 0; i < new_size; ++i) { diff --git a/dbms/src/Common/HyperLogLogCounter.h b/dbms/src/Common/HyperLogLogCounter.h index d457a9342b9..d85da31e7fd 100644 --- a/dbms/src/Common/HyperLogLogCounter.h +++ b/dbms/src/Common/HyperLogLogCounter.h @@ -310,9 +310,9 @@ public: { /// Normalizing factor for harmonic mean. static constexpr double alpha_m = - bucket_count == 2 ? 0.351 : - bucket_count == 4 ? 0.532 : - bucket_count == 8 ? 0.626 : + bucket_count == 2 ? 0.351 : + bucket_count == 4 ? 0.532 : + bucket_count == 8 ? 0.626 : bucket_count == 16 ? 0.673 : bucket_count == 32 ? 0.697 : bucket_count == 64 ? 0.709 : 0.7213 / (1 + 1.079 / bucket_count); diff --git a/dbms/src/Common/NetException.h b/dbms/src/Common/NetException.h index b2b12e85648..ff7d7c5c5fe 100644 --- a/dbms/src/Common/NetException.h +++ b/dbms/src/Common/NetException.h @@ -6,20 +6,17 @@ namespace DB { -class NetException : public DB::Exception +class NetException : public Exception { public: - NetException(const std::string & msg, int code = 0) : DB::Exception(msg, code) {} - NetException(const std::string & msg, const std::string & arg, int code = 0) : DB::Exception(msg, arg, code) {} - NetException(const std::string & msg, const DB::Exception & exc, int code = 0) : DB::Exception(msg, exc, code) {} + NetException(const std::string & msg, int code) : Exception(msg, code) {} - explicit NetException(const DB::Exception & exc) : DB::Exception(exc) {} - explicit NetException(const Poco::Exception & exc) : DB::Exception(exc.displayText()) {} + NetException * clone() const override { return new NetException(*this); } + void rethrow() const override { throw *this; } +private: const char * name() const throw() override { return "DB::NetException"; } const char * className() const throw() override { return "DB::NetException"; } - DB::NetException * clone() const override { return new DB::NetException(*this); } - void rethrow() const override { throw *this; } }; } diff --git a/dbms/src/Common/PoolWithFailoverBase.h b/dbms/src/Common/PoolWithFailoverBase.h index 63420f12ac1..dc986a44ffa 100644 --- a/dbms/src/Common/PoolWithFailoverBase.h +++ b/dbms/src/Common/PoolWithFailoverBase.h @@ -305,7 +305,7 @@ void PoolWithFailoverBase::reportError(const Entry & entry) return; } } - throw DB::Exception("Can't find pool to report error."); + throw DB::Exception("Can't find pool to report error", DB::ErrorCodes::LOGICAL_ERROR); } template diff --git a/dbms/src/Common/ProfileEvents.h b/dbms/src/Common/ProfileEvents.h index ac4ae504f50..78b4ebbf42e 100644 --- a/dbms/src/Common/ProfileEvents.h +++ b/dbms/src/Common/ProfileEvents.h @@ -37,7 +37,7 @@ namespace ProfileEvents /// Global level static initializer Counters(Counter * allocated_counters) - : counters(allocated_counters), parent(nullptr), level(VariableContext::Global) {} + : counters(allocated_counters), parent(nullptr), level(VariableContext::Global) {} Counter & operator[] (Event event) { diff --git a/dbms/src/Common/RWLockFIFO.cpp b/dbms/src/Common/RWLockFIFO.cpp index 2d293164f3f..d511685111d 100644 --- a/dbms/src/Common/RWLockFIFO.cpp +++ b/dbms/src/Common/RWLockFIFO.cpp @@ -134,7 +134,7 @@ RWLockFIFO::LockHandler RWLockFIFO::getLock(RWLockFIFO::Type type, RWLockFIFO::C } /// Wait a notification - it_group->cv.wait(lock, [&] () { return it_group == queue.begin(); } ); + it_group->cv.wait(lock, [&] () { return it_group == queue.begin(); }); it_client->start_time = time(nullptr); finalize_metrics(); diff --git a/dbms/src/Common/ShellCommand.cpp b/dbms/src/Common/ShellCommand.cpp index c9e625810ca..675dd8391f1 100644 --- a/dbms/src/Common/ShellCommand.cpp +++ b/dbms/src/Common/ShellCommand.cpp @@ -4,9 +4,11 @@ #include #include #include +#include #include #include #include +#include namespace DB @@ -75,13 +77,27 @@ namespace namespace DB { +ShellCommand::ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd, bool terminate_in_destructor_) + : pid(pid) + , terminate_in_destructor(terminate_in_destructor_) + , log(&Poco::Logger::get("ShellCommand")) + , in(in_fd) + , out(out_fd) + , err(err_fd) {} + ShellCommand::~ShellCommand() { - if (!wait_called) + if (terminate_in_destructor) + { + int retcode = kill(pid, SIGTERM); + if (retcode != 0) + LOG_WARNING(log, "Cannot kill pid " << pid << " errno '" << errnoToString(retcode) << "'"); + } + else if (!wait_called) tryWait(); } -std::unique_ptr ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only) +std::unique_ptr ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor) { /** Here it is written that with a normal call `vfork`, there is a chance of deadlock in multithreaded programs, * because of the resolving of characters in the shared library @@ -128,7 +144,7 @@ std::unique_ptr ShellCommand::executeImpl(const char * filename, c _exit(int(ReturnCodes::CANNOT_EXEC)); } - std::unique_ptr res(new ShellCommand(pid, pipe_stdin.write_fd, pipe_stdout.read_fd, pipe_stderr.read_fd)); + std::unique_ptr res(new ShellCommand(pid, pipe_stdin.write_fd, pipe_stdout.read_fd, pipe_stderr.read_fd, terminate_in_destructor)); /// Now the ownership of the file descriptors is passed to the result. pipe_stdin.write_fd = -1; @@ -139,7 +155,7 @@ std::unique_ptr ShellCommand::executeImpl(const char * filename, c } -std::unique_ptr ShellCommand::execute(const std::string & command, bool pipe_stdin_only) +std::unique_ptr ShellCommand::execute(const std::string & command, bool pipe_stdin_only, bool terminate_in_destructor) { /// Arguments in non-constant chunks of memory (as required for `execv`). /// Moreover, their copying must be done before calling `vfork`, so after `vfork` do a minimum of things. @@ -149,11 +165,11 @@ std::unique_ptr ShellCommand::execute(const std::string & command, char * const argv[] = { argv0.data(), argv1.data(), argv2.data(), nullptr }; - return executeImpl("/bin/sh", argv, pipe_stdin_only); + return executeImpl("/bin/sh", argv, pipe_stdin_only, terminate_in_destructor); } -std::unique_ptr ShellCommand::executeDirect(const std::string & path, const std::vector & arguments) +std::unique_ptr ShellCommand::executeDirect(const std::string & path, const std::vector & arguments, bool terminate_in_destructor) { size_t argv_sum_size = path.size() + 1; for (const auto & arg : arguments) @@ -174,7 +190,7 @@ std::unique_ptr ShellCommand::executeDirect(const std::string & pa argv[arguments.size() + 1] = nullptr; - return executeImpl(path.data(), argv.data(), false); + return executeImpl(path.data(), argv.data(), false, terminate_in_destructor); } diff --git a/dbms/src/Common/ShellCommand.h b/dbms/src/Common/ShellCommand.h index a439568fe06..3d1308272e9 100644 --- a/dbms/src/Common/ShellCommand.h +++ b/dbms/src/Common/ShellCommand.h @@ -28,11 +28,13 @@ class ShellCommand private: pid_t pid; bool wait_called = false; + bool terminate_in_destructor; - ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd) - : pid(pid), in(in_fd), out(out_fd), err(err_fd) {} + Poco::Logger * log; - static std::unique_ptr executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only); + ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd, bool terminate_in_destructor_); + + static std::unique_ptr executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor); public: WriteBufferFromFile in; /// If the command reads from stdin, do not forget to call in.close() after writing all the data there. @@ -41,11 +43,13 @@ public: ~ShellCommand(); - /// Run the command using /bin/sh -c - static std::unique_ptr execute(const std::string & command, bool pipe_stdin_only = false); + /// Run the command using /bin/sh -c. + /// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process. + static std::unique_ptr execute(const std::string & command, bool pipe_stdin_only = false, bool terminate_in_destructor = false); /// Run the executable with the specified arguments. `arguments` - without argv[0]. - static std::unique_ptr executeDirect(const std::string & path, const std::vector & arguments); + /// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process. + static std::unique_ptr executeDirect(const std::string & path, const std::vector & arguments, bool terminate_in_destructor = false); /// Wait for the process to end, throw an exception if the code is not 0 or if the process was not completed by itself. void wait(); diff --git a/dbms/src/Common/StatusFile.cpp b/dbms/src/Common/StatusFile.cpp index 84b1edc922b..afe42262b55 100644 --- a/dbms/src/Common/StatusFile.cpp +++ b/dbms/src/Common/StatusFile.cpp @@ -20,6 +20,14 @@ namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_OPEN_FILE; + extern const int CANNOT_CLOSE_FILE; + extern const int CANNOT_TRUNCATE_FILE; + extern const int CANNOT_SEEK_THROUGH_FILE; +} + StatusFile::StatusFile(const std::string & path_) : path(path_) @@ -43,7 +51,7 @@ StatusFile::StatusFile(const std::string & path_) fd = ::open(path.c_str(), O_WRONLY | O_CREAT, 0666); if (-1 == fd) - throwFromErrno("Cannot open file " + path); + throwFromErrno("Cannot open file " + path, ErrorCodes::CANNOT_OPEN_FILE); try { @@ -51,16 +59,16 @@ StatusFile::StatusFile(const std::string & path_) if (-1 == flock_ret) { if (errno == EWOULDBLOCK) - throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running."); + throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running.", ErrorCodes::CANNOT_OPEN_FILE); else - throwFromErrno("Cannot lock file " + path); + throwFromErrno("Cannot lock file " + path, ErrorCodes::CANNOT_OPEN_FILE); } if (0 != ftruncate(fd, 0)) - throwFromErrno("Cannot ftruncate " + path); + throwFromErrno("Cannot ftruncate " + path, ErrorCodes::CANNOT_TRUNCATE_FILE); if (0 != lseek(fd, 0, SEEK_SET)) - throwFromErrno("Cannot lseek " + path); + throwFromErrno("Cannot lseek " + path, ErrorCodes::CANNOT_SEEK_THROUGH_FILE); /// Write information about current server instance to the file. { @@ -82,10 +90,10 @@ StatusFile::StatusFile(const std::string & path_) StatusFile::~StatusFile() { if (0 != close(fd)) - LOG_ERROR(&Logger::get("StatusFile"), "Cannot close file " << path << ", " << errnoToString()); + LOG_ERROR(&Logger::get("StatusFile"), "Cannot close file " << path << ", " << errnoToString(ErrorCodes::CANNOT_CLOSE_FILE)); if (0 != unlink(path.c_str())) - LOG_ERROR(&Logger::get("StatusFile"), "Cannot unlink file " << path << ", " << errnoToString()); + LOG_ERROR(&Logger::get("StatusFile"), "Cannot unlink file " << path << ", " << errnoToString(ErrorCodes::CANNOT_CLOSE_FILE)); } } diff --git a/dbms/src/Common/Stopwatch.h b/dbms/src/Common/Stopwatch.h index c1ec623e100..d6508b23b44 100644 --- a/dbms/src/Common/Stopwatch.h +++ b/dbms/src/Common/Stopwatch.h @@ -28,15 +28,15 @@ public: */ Stopwatch(clockid_t clock_type_ = CLOCK_MONOTONIC) : clock_type(clock_type_) { start(); } - void start() { start_ns = nanoseconds(); is_running = true; } - void stop() { stop_ns = nanoseconds(); is_running = false; } - void reset() { start_ns = 0; stop_ns = 0; is_running = false; } - void restart() { start(); } - UInt64 elapsed() const { return elapsedNanoseconds(); } - UInt64 elapsedNanoseconds() const { return is_running ? nanoseconds() - start_ns : stop_ns - start_ns; } - UInt64 elapsedMicroseconds() const { return elapsedNanoseconds() / 1000U; } - UInt64 elapsedMilliseconds() const { return elapsedNanoseconds() / 1000000UL; } - double elapsedSeconds() const { return static_cast(elapsedNanoseconds()) / 1000000000ULL; } + void start() { start_ns = nanoseconds(); is_running = true; } + void stop() { stop_ns = nanoseconds(); is_running = false; } + void reset() { start_ns = 0; stop_ns = 0; is_running = false; } + void restart() { start(); } + UInt64 elapsed() const { return elapsedNanoseconds(); } + UInt64 elapsedNanoseconds() const { return is_running ? nanoseconds() - start_ns : stop_ns - start_ns; } + UInt64 elapsedMicroseconds() const { return elapsedNanoseconds() / 1000U; } + UInt64 elapsedMilliseconds() const { return elapsedNanoseconds() / 1000000UL; } + double elapsedSeconds() const { return static_cast(elapsedNanoseconds()) / 1000000000ULL; } private: UInt64 start_ns = 0; @@ -53,10 +53,10 @@ class AtomicStopwatch public: AtomicStopwatch(clockid_t clock_type_ = CLOCK_MONOTONIC) : clock_type(clock_type_) { restart(); } - void restart() { start_ns = nanoseconds(); } - UInt64 elapsed() const { return nanoseconds() - start_ns; } - UInt64 elapsedMilliseconds() const { return elapsed() / 1000000UL; } - double elapsedSeconds() const { return static_cast(elapsed()) / 1000000000ULL; } + void restart() { start_ns = nanoseconds(); } + UInt64 elapsed() const { return nanoseconds() - start_ns; } + UInt64 elapsedMilliseconds() const { return elapsed() / 1000000UL; } + double elapsedSeconds() const { return static_cast(elapsed()) / 1000000000ULL; } /** If specified amount of time has passed, then restarts timer and returns true. * Otherwise returns false. @@ -140,10 +140,10 @@ struct StopwatchRUsage { StopwatchRUsage() = default; - void start() { start_ts = Timestamp::current(); is_running = true; } - void stop() { stop_ts = Timestamp::current(); is_running = false; } - void reset() { start_ts = Timestamp(); stop_ts = Timestamp(); is_running = false; } - void restart() { start(); } + void start() { start_ts = Timestamp::current(); is_running = true; } + void stop() { stop_ts = Timestamp::current(); is_running = false; } + void reset() { start_ts = Timestamp(); stop_ts = Timestamp(); is_running = false; } + void restart() { start(); } UInt64 elapsed(bool count_user = true, bool count_sys = true) const { diff --git a/dbms/src/Common/ThreadProfileEvents.h b/dbms/src/Common/ThreadProfileEvents.h index 233d4eec047..3be784f135e 100644 --- a/dbms/src/Common/ThreadProfileEvents.h +++ b/dbms/src/Common/ThreadProfileEvents.h @@ -141,9 +141,9 @@ struct TasksStatsCounters if (curr.stat.version < TASKSTATS_VERSION) return; - profile_events.increment(ProfileEvents::OSReadChars, safeDiff(prev.stat.read_char, curr.stat.read_char)); + profile_events.increment(ProfileEvents::OSReadChars, safeDiff(prev.stat.read_char, curr.stat.read_char)); profile_events.increment(ProfileEvents::OSWriteChars, safeDiff(prev.stat.write_char, curr.stat.write_char)); - profile_events.increment(ProfileEvents::OSReadBytes, safeDiff(prev.stat.read_bytes, curr.stat.read_bytes)); + profile_events.increment(ProfileEvents::OSReadBytes, safeDiff(prev.stat.read_bytes, curr.stat.read_bytes)); profile_events.increment(ProfileEvents::OSWriteBytes, safeDiff(prev.stat.write_bytes, curr.stat.write_bytes)); } diff --git a/dbms/src/Common/XDBCBridgeHelper.h b/dbms/src/Common/XDBCBridgeHelper.h index efdf7b401dd..3ff91c902f5 100644 --- a/dbms/src/Common/XDBCBridgeHelper.h +++ b/dbms/src/Common/XDBCBridgeHelper.h @@ -68,6 +68,7 @@ protected: public: using Configuration = Poco::Util::AbstractConfiguration; + Context & context; const Configuration & config; static constexpr inline auto DEFAULT_HOST = "localhost"; @@ -78,8 +79,8 @@ public: static constexpr inline auto IDENTIFIER_QUOTE_HANDLER = "/identifier_quote"; static constexpr inline auto PING_OK_ANSWER = "Ok."; - XDBCBridgeHelper(const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_) - : http_timeout(http_timeout_), connection_string(connection_string_), config(config_) + XDBCBridgeHelper(Context & global_context_, const Poco::Timespan & http_timeout_, const std::string & connection_string_) + : http_timeout(http_timeout_), connection_string(connection_string_), context(global_context_), config(context.getConfigRef()) { size_t bridge_port = config.getUInt(BridgeHelperMixin::configPrefix() + ".port", DEFAULT_PORT); std::string bridge_host = config.getString(BridgeHelperMixin::configPrefix() + ".host", DEFAULT_HOST); @@ -210,7 +211,8 @@ private: /* Contains logic for instantiation of the bridge instance */ void startBridge() const { - BridgeHelperMixin::startBridge(config, log, http_timeout); + auto cmd = BridgeHelperMixin::startBridge(config, log, http_timeout); + context.addXDBCBridgeCommand(std::move(cmd)); } }; @@ -230,7 +232,7 @@ struct JDBCBridgeMixin return "JDBC"; } - static void startBridge(const Poco::Util::AbstractConfiguration &, const Poco::Logger *, const Poco::Timespan &) + static std::unique_ptr startBridge(const Poco::Util::AbstractConfiguration &, const Poco::Logger *, const Poco::Timespan &) { throw Exception("jdbc-bridge is not running. Please, start it manually", ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING); } @@ -253,11 +255,13 @@ struct ODBCBridgeMixin return "ODBC"; } - static void startBridge(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout) + static std::unique_ptr startBridge(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout) { /// Path to executable folder Poco::Path path{config.getString("application.dir", "/usr/bin")}; + + std::vector cmd_args; path.setFileName( #if CLICKHOUSE_SPLIT_BINARY "clickhouse-odbc-bridge" @@ -268,34 +272,35 @@ struct ODBCBridgeMixin std::stringstream command; - command << path.toString() << -#if CLICKHOUSE_SPLIT_BINARY - " " -#else - " odbc-bridge " +#if !CLICKHOUSE_SPLIT_BINARY + cmd_args.push_back("odbc-bridge"); #endif - ; - command << "--http-port " << config.getUInt(configPrefix() + ".port", DEFAULT_PORT) << ' '; - command << "--listen-host " << config.getString(configPrefix() + ".listen_host", XDBCBridgeHelper::DEFAULT_HOST) - << ' '; - command << "--http-timeout " << http_timeout.totalMicroseconds() << ' '; + cmd_args.push_back("--http-port"); + cmd_args.push_back(std::to_string(config.getUInt(configPrefix() + ".port", DEFAULT_PORT))); + cmd_args.push_back("--listen-host"); + cmd_args.push_back(config.getString(configPrefix() + ".listen_host", XDBCBridgeHelper::DEFAULT_HOST)); + cmd_args.push_back("--http-timeout"); + cmd_args.push_back(std::to_string(http_timeout.totalMicroseconds())); if (config.has("logger." + configPrefix() + "_log")) - command << "--log-path " << config.getString("logger." + configPrefix() + "_log") << ' '; + { + cmd_args.push_back("--log-path"); + cmd_args.push_back(config.getString("logger." + configPrefix() + "_log")); + } if (config.has("logger." + configPrefix() + "_errlog")) - command << "--err-log-path " << config.getString("logger." + configPrefix() + "_errlog") << ' '; + { + cmd_args.push_back("--err-log-path"); + cmd_args.push_back(config.getString("logger." + configPrefix() + "_errlog")); + } if (config.has("logger." + configPrefix() + "_level")) - command << "--log-level " << config.getString("logger." + configPrefix() + "_level") << ' '; - command << "&"; /// we don't want to wait this process + { + cmd_args.push_back("--log-level"); + cmd_args.push_back(config.getString("logger." + configPrefix() + "_level")); + } - auto command_str = command.str(); + LOG_TRACE(log, "Starting " + serviceAlias()); - std::cerr << command_str << std::endl; - - LOG_TRACE(log, "Starting " + serviceAlias() + " with command: " << command_str); - - auto cmd = ShellCommand::execute(command_str); - cmd->wait(); + return ShellCommand::executeDirect(path.toString(), cmd_args, true); } }; } diff --git a/dbms/src/Common/ZooKeeper/CMakeLists.txt b/dbms/src/Common/ZooKeeper/CMakeLists.txt index 2969d9a91e9..1f69f0af1ec 100644 --- a/dbms/src/Common/ZooKeeper/CMakeLists.txt +++ b/dbms/src/Common/ZooKeeper/CMakeLists.txt @@ -4,7 +4,8 @@ add_headers_and_sources(clickhouse_common_zookeeper .) add_library(clickhouse_common_zookeeper ${LINK_MODE} ${clickhouse_common_zookeeper_headers} ${clickhouse_common_zookeeper_sources}) -target_link_libraries (clickhouse_common_zookeeper clickhouse_common_io) +target_link_libraries (clickhouse_common_zookeeper PUBLIC clickhouse_common_io common PRIVATE string_utils PUBLIC ${Poco_Util_LIBRARY}) +target_include_directories(clickhouse_common_zookeeper PUBLIC ${DBMS_INCLUDE_DIR}) if (ENABLE_TESTS) add_subdirectory (tests) diff --git a/dbms/src/Common/ZooKeeper/IKeeper.cpp b/dbms/src/Common/ZooKeeper/IKeeper.cpp index e6713cca3de..114b14d0118 100644 --- a/dbms/src/Common/ZooKeeper/IKeeper.cpp +++ b/dbms/src/Common/ZooKeeper/IKeeper.cpp @@ -91,30 +91,30 @@ const char * errorMessage(int32_t code) { switch (code) { - case ZOK: return "Ok"; - case ZSYSTEMERROR: return "System error"; - case ZRUNTIMEINCONSISTENCY: return "Run time inconsistency"; - case ZDATAINCONSISTENCY: return "Data inconsistency"; - case ZCONNECTIONLOSS: return "Connection loss"; - case ZMARSHALLINGERROR: return "Marshalling error"; - case ZUNIMPLEMENTED: return "Unimplemented"; - case ZOPERATIONTIMEOUT: return "Operation timeout"; - case ZBADARGUMENTS: return "Bad arguments"; - case ZINVALIDSTATE: return "Invalid zhandle state"; - case ZAPIERROR: return "API error"; - case ZNONODE: return "No node"; - case ZNOAUTH: return "Not authenticated"; - case ZBADVERSION: return "Bad version"; - case ZNOCHILDRENFOREPHEMERALS: return "No children for ephemerals"; - case ZNODEEXISTS: return "Node exists"; - case ZNOTEMPTY: return "Not empty"; - case ZSESSIONEXPIRED: return "Session expired"; - case ZINVALIDCALLBACK: return "Invalid callback"; - case ZINVALIDACL: return "Invalid ACL"; - case ZAUTHFAILED: return "Authentication failed"; - case ZCLOSING: return "ZooKeeper is closing"; - case ZNOTHING: return "(not error) no server responses to process"; - case ZSESSIONMOVED: return "Session moved to another server, so operation is ignored"; + case ZOK: return "Ok"; + case ZSYSTEMERROR: return "System error"; + case ZRUNTIMEINCONSISTENCY: return "Run time inconsistency"; + case ZDATAINCONSISTENCY: return "Data inconsistency"; + case ZCONNECTIONLOSS: return "Connection loss"; + case ZMARSHALLINGERROR: return "Marshalling error"; + case ZUNIMPLEMENTED: return "Unimplemented"; + case ZOPERATIONTIMEOUT: return "Operation timeout"; + case ZBADARGUMENTS: return "Bad arguments"; + case ZINVALIDSTATE: return "Invalid zhandle state"; + case ZAPIERROR: return "API error"; + case ZNONODE: return "No node"; + case ZNOAUTH: return "Not authenticated"; + case ZBADVERSION: return "Bad version"; + case ZNOCHILDRENFOREPHEMERALS: return "No children for ephemerals"; + case ZNODEEXISTS: return "Node exists"; + case ZNOTEMPTY: return "Not empty"; + case ZSESSIONEXPIRED: return "Session expired"; + case ZINVALIDCALLBACK: return "Invalid callback"; + case ZINVALIDACL: return "Invalid ACL"; + case ZAUTHFAILED: return "Authentication failed"; + case ZCLOSING: return "ZooKeeper is closing"; + case ZNOTHING: return "(not error) no server responses to process"; + case ZSESSIONMOVED: return "Session moved to another server, so operation is ignored"; } if (code > 0) return strerror(code); diff --git a/dbms/src/Common/ZooKeeper/ZooKeeperHolder.cpp b/dbms/src/Common/ZooKeeper/ZooKeeperHolder.cpp index 1b49c22cf46..9343bb4addc 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeperHolder.cpp +++ b/dbms/src/Common/ZooKeeper/ZooKeeperHolder.cpp @@ -1,5 +1,15 @@ #include "ZooKeeperHolder.h" + +namespace DB +{ + namespace ErrorCodes + { + extern const int NULL_POINTER_DEREFERENCE; + } +} + + using namespace zkutil; ZooKeeperHolder::UnstorableZookeeperHandler ZooKeeperHolder::getZooKeeper() @@ -47,7 +57,7 @@ ZooKeeperHolder::UnstorableZookeeperHandler::UnstorableZookeeperHandler(ZooKeepe ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->() { if (zk_ptr == nullptr) - throw DB::Exception(nullptr_exception_message); + throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE); return zk_ptr.get(); } @@ -55,20 +65,20 @@ ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->() const ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->() const { if (zk_ptr == nullptr) - throw DB::Exception(nullptr_exception_message); + throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE); return zk_ptr.get(); } ZooKeeper & ZooKeeperHolder::UnstorableZookeeperHandler::operator*() { if (zk_ptr == nullptr) - throw DB::Exception(nullptr_exception_message); + throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE); return *zk_ptr; } const ZooKeeper & ZooKeeperHolder::UnstorableZookeeperHandler::operator*() const { if (zk_ptr == nullptr) - throw DB::Exception(nullptr_exception_message); + throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE); return *zk_ptr; } diff --git a/dbms/src/Common/ZooKeeper/tests/CMakeLists.txt b/dbms/src/Common/ZooKeeper/tests/CMakeLists.txt index a24948240f6..06716e49918 100644 --- a/dbms/src/Common/ZooKeeper/tests/CMakeLists.txt +++ b/dbms/src/Common/ZooKeeper/tests/CMakeLists.txt @@ -1,23 +1,23 @@ add_executable(zkutil_test_commands zkutil_test_commands.cpp) -target_link_libraries(zkutil_test_commands clickhouse_common_zookeeper) +target_link_libraries(zkutil_test_commands PRIVATE clickhouse_common_zookeeper) add_executable(zkutil_test_commands_new_lib zkutil_test_commands_new_lib.cpp) -target_link_libraries(zkutil_test_commands_new_lib clickhouse_common_zookeeper) +target_link_libraries(zkutil_test_commands_new_lib PRIVATE clickhouse_common_zookeeper) add_executable(zkutil_test_lock zkutil_test_lock.cpp) -target_link_libraries(zkutil_test_lock clickhouse_common_zookeeper) +target_link_libraries(zkutil_test_lock PRIVATE clickhouse_common_zookeeper) add_executable(zkutil_expiration_test zkutil_expiration_test.cpp) -target_link_libraries(zkutil_expiration_test clickhouse_common_zookeeper) +target_link_libraries(zkutil_expiration_test PRIVATE clickhouse_common_zookeeper) add_executable(zkutil_test_async zkutil_test_async.cpp) -target_link_libraries(zkutil_test_async clickhouse_common_zookeeper) +target_link_libraries(zkutil_test_async PRIVATE clickhouse_common_zookeeper) add_executable(zkutil_zookeeper_holder zkutil_zookeeper_holder.cpp) -target_link_libraries(zkutil_zookeeper_holder clickhouse_common_zookeeper) +target_link_libraries(zkutil_zookeeper_holder PRIVATE clickhouse_common_zookeeper) add_executable (zk_many_watches_reconnect zk_many_watches_reconnect.cpp) -target_link_libraries (zk_many_watches_reconnect clickhouse_common_zookeeper clickhouse_common_config) +target_link_libraries (zk_many_watches_reconnect PRIVATE clickhouse_common_zookeeper clickhouse_common_config) add_executable (zookeeper_impl zookeeper_impl.cpp) -target_link_libraries (zookeeper_impl clickhouse_common_zookeeper) +target_link_libraries (zookeeper_impl PRIVATE clickhouse_common_zookeeper) diff --git a/dbms/src/Common/config.h.in b/dbms/src/Common/config.h.in index af12cc525f7..44aa9e9773d 100644 --- a/dbms/src/Common/config.h.in +++ b/dbms/src/Common/config.h.in @@ -15,3 +15,4 @@ #cmakedefine01 USE_POCO_MONGODB #cmakedefine01 USE_POCO_NETSSL #cmakedefine01 CLICKHOUSE_SPLIT_BINARY +#cmakedefine01 USE_BASE64 diff --git a/dbms/src/Common/config_version.h.in b/dbms/src/Common/config_version.h.in index 8f3e2f56c84..a90fd77b6a8 100644 --- a/dbms/src/Common/config_version.h.in +++ b/dbms/src/Common/config_version.h.in @@ -23,6 +23,7 @@ #cmakedefine VERSION_FULL "@VERSION_FULL@" #cmakedefine VERSION_DESCRIBE "@VERSION_DESCRIBE@" #cmakedefine VERSION_GITHASH "@VERSION_GITHASH@" +#cmakedefine VERSION_INTEGER @VERSION_INTEGER@ #if defined(VERSION_MAJOR) #define DBMS_VERSION_MAJOR VERSION_MAJOR diff --git a/dbms/src/Common/createHardLink.cpp b/dbms/src/Common/createHardLink.cpp index 06647cd437c..824b7e99086 100644 --- a/dbms/src/Common/createHardLink.cpp +++ b/dbms/src/Common/createHardLink.cpp @@ -8,6 +8,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_STAT; + extern const int CANNOT_LINK; +} + void createHardLink(const String & source_path, const String & destination_path) { if (0 != link(source_path.c_str(), destination_path.c_str())) @@ -20,16 +26,16 @@ void createHardLink(const String & source_path, const String & destination_path) struct stat destination_descr; if (0 != lstat(source_path.c_str(), &source_descr)) - throwFromErrno("Cannot stat " + source_path); + throwFromErrno("Cannot stat " + source_path, ErrorCodes::CANNOT_STAT); if (0 != lstat(destination_path.c_str(), &destination_descr)) - throwFromErrno("Cannot stat " + destination_path); + throwFromErrno("Cannot stat " + destination_path, ErrorCodes::CANNOT_STAT); if (source_descr.st_ino != destination_descr.st_ino) - throwFromErrno("Destination file " + destination_path + " is already exist and have different inode.", 0, link_errno); + throwFromErrno("Destination file " + destination_path + " is already exist and have different inode.", ErrorCodes::CANNOT_LINK, link_errno); } else - throwFromErrno("Cannot link " + source_path + " to " + destination_path); + throwFromErrno("Cannot link " + source_path + " to " + destination_path, ErrorCodes::CANNOT_LINK); } } diff --git a/dbms/src/Common/formatIPv6.cpp b/dbms/src/Common/formatIPv6.cpp index e2a295cb923..1b09a7e0e1d 100644 --- a/dbms/src/Common/formatIPv6.cpp +++ b/dbms/src/Common/formatIPv6.cpp @@ -7,7 +7,7 @@ namespace DB { -/// integer logarithm, return ceil(log(value, base)) (the smallest integer greater or equal than log(value, base) +/// integer logarithm, return ceil(log(value, base)) (the smallest integer greater or equal than log(value, base) static constexpr UInt32 intLog(const UInt32 value, const UInt32 base, const bool carry) { return value >= base ? 1 + intLog(value / base, base, value % base || carry) : value % base > 1 || carry; diff --git a/dbms/src/Common/isLocalAddress.h b/dbms/src/Common/isLocalAddress.h index ffa03977a3f..81039dff68e 100644 --- a/dbms/src/Common/isLocalAddress.h +++ b/dbms/src/Common/isLocalAddress.h @@ -22,7 +22,7 @@ namespace DB * - only the first address is taken for each network interface; * - the routing rules that affect which network interface we go to the specified address are not checked. */ - bool isLocalAddress(const Poco::Net::SocketAddress & address, UInt16 clickhouse_port); + bool isLocalAddress(const Poco::Net::SocketAddress & address, UInt16 clickhouse_port); bool isLocalAddress(const Poco::Net::SocketAddress & address); diff --git a/dbms/src/Common/parseAddress.cpp b/dbms/src/Common/parseAddress.cpp index 5cd6d353e42..1e266ad503f 100644 --- a/dbms/src/Common/parseAddress.cpp +++ b/dbms/src/Common/parseAddress.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include namespace DB diff --git a/dbms/src/Common/setThreadName.cpp b/dbms/src/Common/setThreadName.cpp index fc228d78ff4..d035822974a 100644 --- a/dbms/src/Common/setThreadName.cpp +++ b/dbms/src/Common/setThreadName.cpp @@ -32,7 +32,7 @@ void setThreadName(const char * name) #else if (0 != prctl(PR_SET_NAME, name, 0, 0, 0)) #endif - DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME, ...)"); + DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME, ...)", DB::ErrorCodes::PTHREAD_ERROR); } std::string getThreadName() @@ -48,7 +48,7 @@ std::string getThreadName() // throw DB::Exception("Cannot get thread name with pthread_get_name_np()", DB::ErrorCodes::PTHREAD_ERROR); #else if (0 != prctl(PR_GET_NAME, name.data(), 0, 0, 0)) - DB::throwFromErrno("Cannot get thread name with prctl(PR_GET_NAME)"); + DB::throwFromErrno("Cannot get thread name with prctl(PR_GET_NAME)", DB::ErrorCodes::PTHREAD_ERROR); #endif name.resize(std::strlen(name.data())); diff --git a/dbms/src/Common/tests/CMakeLists.txt b/dbms/src/Common/tests/CMakeLists.txt index f4d01e85bd2..05984c9d42e 100644 --- a/dbms/src/Common/tests/CMakeLists.txt +++ b/dbms/src/Common/tests/CMakeLists.txt @@ -1,73 +1,73 @@ add_executable (hashes_test hashes_test.cpp) -target_link_libraries (hashes_test dbms) +target_link_libraries (hashes_test PRIVATE dbms ${OPENSSL_CRYPTO_LIBRARY}) add_executable (sip_hash sip_hash.cpp) -target_link_libraries (sip_hash clickhouse_common_io) +target_link_libraries (sip_hash PRIVATE clickhouse_common_io) add_executable (sip_hash_perf sip_hash_perf.cpp) -target_link_libraries (sip_hash_perf clickhouse_common_io) +target_link_libraries (sip_hash_perf PRIVATE clickhouse_common_io) add_executable (auto_array auto_array.cpp) -target_link_libraries (auto_array clickhouse_common_io) +target_link_libraries (auto_array PRIVATE clickhouse_common_io) add_executable (lru_cache lru_cache.cpp) -target_link_libraries (lru_cache clickhouse_common_io) +target_link_libraries (lru_cache PRIVATE clickhouse_common_io) add_executable (hash_table hash_table.cpp) -target_link_libraries (hash_table clickhouse_common_io) +target_link_libraries (hash_table PRIVATE clickhouse_common_io) add_executable (small_table small_table.cpp) -target_link_libraries (small_table clickhouse_common_io) +target_link_libraries (small_table PRIVATE clickhouse_common_io) add_executable (parallel_aggregation parallel_aggregation.cpp) -target_link_libraries (parallel_aggregation clickhouse_common_io) +target_link_libraries (parallel_aggregation PRIVATE clickhouse_common_io) add_executable (parallel_aggregation2 parallel_aggregation2.cpp) -target_link_libraries (parallel_aggregation2 clickhouse_common_io) +target_link_libraries (parallel_aggregation2 PRIVATE clickhouse_common_io) add_executable (int_hashes_perf int_hashes_perf.cpp AvalancheTest.cpp Random.cpp) -target_link_libraries (int_hashes_perf clickhouse_common_io) +target_link_libraries (int_hashes_perf PRIVATE clickhouse_common_io) add_executable (simple_cache simple_cache.cpp) target_include_directories (simple_cache PRIVATE ${DBMS_INCLUDE_DIR}) -target_link_libraries (simple_cache common) +target_link_libraries (simple_cache PRIVATE common) add_executable (compact_array compact_array.cpp) -target_link_libraries (compact_array clickhouse_common_io ${Boost_FILESYSTEM_LIBRARY}) +target_link_libraries (compact_array PRIVATE clickhouse_common_io ${Boost_FILESYSTEM_LIBRARY}) add_executable (radix_sort radix_sort.cpp) -target_link_libraries (radix_sort clickhouse_common_io) +target_link_libraries (radix_sort PRIVATE clickhouse_common_io) add_executable (shell_command_test shell_command_test.cpp) -target_link_libraries (shell_command_test clickhouse_common_io) +target_link_libraries (shell_command_test PRIVATE clickhouse_common_io) add_executable (arena_with_free_lists arena_with_free_lists.cpp) -target_link_libraries (arena_with_free_lists clickhouse_common_io) +target_link_libraries (arena_with_free_lists PRIVATE clickhouse_common_io) add_executable (pod_array pod_array.cpp) -target_link_libraries (pod_array clickhouse_common_io) +target_link_libraries (pod_array PRIVATE clickhouse_common_io) add_executable (thread_creation_latency thread_creation_latency.cpp) -target_link_libraries (thread_creation_latency clickhouse_common_io) +target_link_libraries (thread_creation_latency PRIVATE clickhouse_common_io) add_executable (thread_pool thread_pool.cpp) -target_link_libraries (thread_pool clickhouse_common_io) +target_link_libraries (thread_pool PRIVATE clickhouse_common_io) add_executable (array_cache array_cache.cpp) -target_link_libraries (array_cache clickhouse_common_io) +target_link_libraries (array_cache PRIVATE clickhouse_common_io) add_executable (space_saving space_saving.cpp) -target_link_libraries (space_saving clickhouse_common_io) +target_link_libraries (space_saving PRIVATE clickhouse_common_io) add_executable (integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp) target_include_directories (integer_hash_tables_and_hashes SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (integer_hash_tables_and_hashes clickhouse_common_io) +target_link_libraries (integer_hash_tables_and_hashes PRIVATE clickhouse_common_io) add_executable (allocator allocator.cpp) -target_link_libraries (allocator clickhouse_common_io) +target_link_libraries (allocator PRIVATE clickhouse_common_io) add_executable (cow_columns cow_columns.cpp) -target_link_libraries (cow_columns clickhouse_common_io) +target_link_libraries (cow_columns PRIVATE clickhouse_common_io) add_executable (stopwatch stopwatch.cpp) -target_link_libraries (stopwatch clickhouse_common_io) +target_link_libraries (stopwatch PRIVATE clickhouse_common_io) diff --git a/dbms/src/Common/tests/arena_with_free_lists.cpp b/dbms/src/Common/tests/arena_with_free_lists.cpp index 0a16eff6c17..5091551b550 100644 --- a/dbms/src/Common/tests/arena_with_free_lists.cpp +++ b/dbms/src/Common/tests/arena_with_free_lists.cpp @@ -25,6 +25,14 @@ using namespace DB; +namespace DB +{ + namespace ErrorCodes + { + extern const int SYSTEM_ERROR; + } +} + /// Implementation of ArenaWithFreeLists, which contains a bug. Used to reproduce the bug. #if USE_BAD_ARENA @@ -237,7 +245,7 @@ int main(int argc, char ** argv) rusage resource_usage; if (0 != getrusage(RUSAGE_SELF, &resource_usage)) - throwFromErrno("Cannot getrusage"); + throwFromErrno("Cannot getrusage", ErrorCodes::SYSTEM_ERROR); size_t allocated_bytes = resource_usage.ru_maxrss * 1024; std::cerr << "Current memory usage: " << allocated_bytes << " bytes.\n"; diff --git a/dbms/src/Common/tests/thread_creation_latency.cpp b/dbms/src/Common/tests/thread_creation_latency.cpp index ccc499c9a1c..ef910a3e9f3 100644 --- a/dbms/src/Common/tests/thread_creation_latency.cpp +++ b/dbms/src/Common/tests/thread_creation_latency.cpp @@ -22,6 +22,15 @@ void f() { ++x; } void * g(void *) { f(); return {}; } +namespace DB +{ + namespace ErrorCodes + { + extern const int PTHREAD_ERROR; + } +} + + template void test(size_t n, const char * name, F && kernel) { @@ -80,9 +89,9 @@ int main(int argc, char ** argv) { pthread_t thread; if (pthread_create(&thread, nullptr, g, nullptr)) - DB::throwFromErrno("Cannot create thread."); + DB::throwFromErrno("Cannot create thread.", DB::ErrorCodes::PTHREAD_ERROR); if (pthread_join(thread, nullptr)) - DB::throwFromErrno("Cannot join thread."); + DB::throwFromErrno("Cannot join thread.", DB::ErrorCodes::PTHREAD_ERROR); }); test(n, "Create and destroy std::thread each iteration", [] diff --git a/dbms/src/Core/BackgroundSchedulePool.cpp b/dbms/src/Core/BackgroundSchedulePool.cpp index 3e24cfca922..5da499e5ae9 100644 --- a/dbms/src/Core/BackgroundSchedulePool.cpp +++ b/dbms/src/Core/BackgroundSchedulePool.cpp @@ -278,7 +278,7 @@ void BackgroundSchedulePool::delayExecutionThreadFunction() { std::unique_lock lock(delayed_tasks_mutex); - while(!shutdown) + while (!shutdown) { Poco::Timestamp min_time; diff --git a/dbms/src/Core/ExternalTable.cpp b/dbms/src/Core/ExternalTable.cpp index 9ab3a8e8d35..5bfdbb12e95 100644 --- a/dbms/src/Core/ExternalTable.cpp +++ b/dbms/src/Core/ExternalTable.cpp @@ -168,7 +168,7 @@ void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header, /// Write data data.first->readPrefix(); output->writePrefix(); - while(Block block = data.first->read()) + while (Block block = data.first->read()) output->write(block); data.first->readSuffix(); output->writeSuffix(); diff --git a/dbms/src/Core/QueryProcessingStage.h b/dbms/src/Core/QueryProcessingStage.h index a16b041ae35..6de65d32f93 100644 --- a/dbms/src/Core/QueryProcessingStage.h +++ b/dbms/src/Core/QueryProcessingStage.h @@ -12,9 +12,9 @@ namespace QueryProcessingStage /// Numbers matter - the later stage has a larger number. enum Enum { - FetchColumns = 0, /// Only read/have been read the columns specified in the query. - WithMergeableState = 1, /// Until the stage where the results of processing on different servers can be combined. - Complete = 2, /// Completely. + FetchColumns = 0, /// Only read/have been read the columns specified in the query. + WithMergeableState = 1, /// Until the stage where the results of processing on different servers can be combined. + Complete = 2, /// Completely. }; inline const char * toString(UInt64 stage) diff --git a/dbms/src/Core/tests/CMakeLists.txt b/dbms/src/Core/tests/CMakeLists.txt index 26e89341bbb..b5b08bb161d 100644 --- a/dbms/src/Core/tests/CMakeLists.txt +++ b/dbms/src/Core/tests/CMakeLists.txt @@ -1,18 +1,15 @@ -add_executable (exception exception.cpp) -target_link_libraries (exception clickhouse_common_io) - add_executable (string_pool string_pool.cpp) -target_link_libraries (string_pool clickhouse_common_io) +target_link_libraries (string_pool PRIVATE clickhouse_common_io) target_include_directories (string_pool SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) add_executable (field field.cpp) -target_link_libraries (field dbms) +target_link_libraries (field PRIVATE dbms) add_executable (move_field move_field.cpp) -target_link_libraries (move_field clickhouse_common_io) +target_link_libraries (move_field PRIVATE clickhouse_common_io) add_executable (rvo_test rvo_test.cpp) -target_link_libraries (rvo_test Threads::Threads) +target_link_libraries (rvo_test PRIVATE Threads::Threads) add_executable (string_ref_hash string_ref_hash.cpp) -target_link_libraries (string_ref_hash clickhouse_common_io) +target_link_libraries (string_ref_hash PRIVATE clickhouse_common_io) diff --git a/dbms/src/Core/tests/exception.cpp b/dbms/src/Core/tests/exception.cpp deleted file mode 100644 index d0e66f75ff7..00000000000 --- a/dbms/src/Core/tests/exception.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#include - -#include - -#include - - -int main(int, char **) -{ - try - { - //throw Poco::Net::ConnectionRefusedException(); - throw DB::Exception(Poco::Net::ConnectionRefusedException()); - } - catch (const DB::Exception & e) - { - std::cerr << e.displayText() << std::endl; - } - catch (const Poco::Exception & e) - { - std::cerr << e.displayText() << std::endl; - } - - return 0; -} diff --git a/dbms/src/DataStreams/DistinctSortedBlockInputStream.cpp b/dbms/src/DataStreams/DistinctSortedBlockInputStream.cpp index a14d32794f3..651b9738f73 100644 --- a/dbms/src/DataStreams/DistinctSortedBlockInputStream.cpp +++ b/dbms/src/DataStreams/DistinctSortedBlockInputStream.cpp @@ -150,7 +150,7 @@ ColumnRawPtrs DistinctSortedBlockInputStream::getClearingColumns(const Block & b { ColumnRawPtrs clearing_hint_columns; clearing_hint_columns.reserve(description.size()); - for(const auto & sort_column_description : description) + for (const auto & sort_column_description : description) { const auto sort_column_ptr = block.safeGetByPosition(sort_column_description.column_number).column.get(); const auto it = std::find(key_columns.cbegin(), key_columns.cend(), sort_column_ptr); diff --git a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp index 4cdc2655933..d85992f8e5d 100644 --- a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp @@ -198,7 +198,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::start() */ for (size_t i = 0; i < merging_threads; ++i) - pool.schedule([this, thread_group=CurrentThread::getGroup()] () { mergeThread(thread_group); } ); + pool.schedule([this, thread_group=CurrentThread::getGroup()] () { mergeThread(thread_group); }); } } diff --git a/dbms/src/DataStreams/ParallelInputsProcessor.h b/dbms/src/DataStreams/ParallelInputsProcessor.h index a1c4e2ac480..500561e0282 100644 --- a/dbms/src/DataStreams/ParallelInputsProcessor.h +++ b/dbms/src/DataStreams/ParallelInputsProcessor.h @@ -107,8 +107,26 @@ public: active_threads = max_threads; threads.reserve(max_threads); auto thread_group = CurrentThread::getGroup(); - for (size_t i = 0; i < max_threads; ++i) - threads.emplace_back([=] () { thread(thread_group, i); } ); + + try + { + for (size_t i = 0; i < max_threads; ++i) + threads.emplace_back([=] () { thread(thread_group, i); }); + } + catch (...) + { + cancel(false); + wait(); + if (active_threads) + { + active_threads = 0; + /// handler.onFinish() is supposed to be called from one of the threads when the number of + /// finished threads reaches max_threads. But since we weren't able to launch all threads, + /// we have to call onFinish() manually here. + handler.onFinish(); + } + throw; + } } /// Ask all sources to stop earlier than they run out. diff --git a/dbms/src/DataStreams/tests/CMakeLists.txt b/dbms/src/DataStreams/tests/CMakeLists.txt index 3dbbc0edbe3..3f6e154927f 100644 --- a/dbms/src/DataStreams/tests/CMakeLists.txt +++ b/dbms/src/DataStreams/tests/CMakeLists.txt @@ -1,16 +1,16 @@ -set(SRCS ) +set(SRCS) add_executable (expression_stream expression_stream.cpp ${SRCS}) -target_link_libraries (expression_stream dbms clickhouse_storages_system) +target_link_libraries (expression_stream PRIVATE dbms clickhouse_storages_system clickhouse_parsers) add_executable (filter_stream filter_stream.cpp ${SRCS}) -target_link_libraries (filter_stream dbms clickhouse_storages_system) +target_link_libraries (filter_stream PRIVATE dbms clickhouse_storages_system clickhouse_parsers clickhouse_common_io) add_executable (union_stream2 union_stream2.cpp ${SRCS}) -target_link_libraries (union_stream2 dbms) +target_link_libraries (union_stream2 PRIVATE dbms) add_executable (collapsing_sorted_stream collapsing_sorted_stream.cpp ${SRCS}) -target_link_libraries (collapsing_sorted_stream dbms) +target_link_libraries (collapsing_sorted_stream PRIVATE dbms) add_executable (finish_sorting_stream finish_sorting_stream.cpp ${SRCS}) -target_link_libraries (finish_sorting_stream dbms) +target_link_libraries (finish_sorting_stream PRIVATE dbms) diff --git a/dbms/src/DataStreams/tests/finish_sorting_stream.cpp b/dbms/src/DataStreams/tests/finish_sorting_stream.cpp index 7627d1712b2..861965e1e1c 100644 --- a/dbms/src/DataStreams/tests/finish_sorting_stream.cpp +++ b/dbms/src/DataStreams/tests/finish_sorting_stream.cpp @@ -13,6 +13,15 @@ using namespace DB; +namespace DB +{ + namespace ErrorCodes + { + extern const int LOGICAL_ERROR; + } +} + + int main(int argc, char ** argv) { srand(123456); @@ -75,7 +84,7 @@ int main(int argc, char ** argv) } if (res_block.rows() != n * m) - throw Exception("Result block size mismatch"); + throw Exception("Result block size mismatch", ErrorCodes::LOGICAL_ERROR); const auto & columns = res_block.getColumns(); @@ -86,7 +95,7 @@ int main(int argc, char ** argv) if (res < 0) break; else if (res > 0) - throw Exception("Result stream not sorted"); + throw Exception("Result stream not sorted", ErrorCodes::LOGICAL_ERROR); } stopwatch.stop(); diff --git a/dbms/src/DataTypes/DataTypeFixedString.cpp b/dbms/src/DataTypes/DataTypeFixedString.cpp index 82c0bec489f..f43de616e8d 100644 --- a/dbms/src/DataTypes/DataTypeFixedString.cpp +++ b/dbms/src/DataTypes/DataTypeFixedString.cpp @@ -64,7 +64,7 @@ void DataTypeFixedString::serializeBinary(const IColumn & column, size_t row_num void DataTypeFixedString::deserializeBinary(IColumn & column, ReadBuffer & istr) const { - ColumnFixedString::Chars_t & data = static_cast(column).getChars(); + ColumnFixedString::Chars & data = static_cast(column).getChars(); size_t old_size = data.size(); data.resize(old_size + n); try @@ -81,7 +81,7 @@ void DataTypeFixedString::deserializeBinary(IColumn & column, ReadBuffer & istr) void DataTypeFixedString::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { - const ColumnFixedString::Chars_t & data = typeid_cast(column).getChars(); + const ColumnFixedString::Chars & data = typeid_cast(column).getChars(); size_t size = data.size() / n; @@ -95,7 +95,7 @@ void DataTypeFixedString::serializeBinaryBulk(const IColumn & column, WriteBuffe void DataTypeFixedString::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double /*avg_value_size_hint*/) const { - ColumnFixedString::Chars_t & data = typeid_cast(column).getChars(); + ColumnFixedString::Chars & data = typeid_cast(column).getChars(); size_t initial_size = data.size(); size_t max_bytes = limit * n; @@ -126,7 +126,7 @@ void DataTypeFixedString::serializeTextEscaped(const IColumn & column, size_t ro template static inline void read(const DataTypeFixedString & self, IColumn & column, Reader && reader) { - ColumnFixedString::Chars_t & data = typeid_cast(column).getChars(); + ColumnFixedString::Chars & data = typeid_cast(column).getChars(); size_t prev_size = data.size(); try @@ -152,7 +152,7 @@ static inline void read(const DataTypeFixedString & self, IColumn & column, Read void DataTypeFixedString::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { - read(*this, column, [&istr](ColumnFixedString::Chars_t & data) { readEscapedStringInto(data, istr); }); + read(*this, column, [&istr](ColumnFixedString::Chars & data) { readEscapedStringInto(data, istr); }); } @@ -165,7 +165,7 @@ void DataTypeFixedString::serializeTextQuoted(const IColumn & column, size_t row void DataTypeFixedString::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { - read(*this, column, [&istr](ColumnFixedString::Chars_t & data) { readQuotedStringInto(data, istr); }); + read(*this, column, [&istr](ColumnFixedString::Chars & data) { readQuotedStringInto(data, istr); }); } @@ -178,7 +178,7 @@ void DataTypeFixedString::serializeTextJSON(const IColumn & column, size_t row_n void DataTypeFixedString::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { - read(*this, column, [&istr](ColumnFixedString::Chars_t & data) { readJSONStringInto(data, istr); }); + read(*this, column, [&istr](ColumnFixedString::Chars & data) { readJSONStringInto(data, istr); }); } @@ -198,7 +198,7 @@ void DataTypeFixedString::serializeTextCSV(const IColumn & column, size_t row_nu void DataTypeFixedString::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - read(*this, column, [&istr, &csv = settings.csv](ColumnFixedString::Chars_t & data) { readCSVStringInto(data, istr, csv); }); + read(*this, column, [&istr, &csv = settings.csv](ColumnFixedString::Chars & data) { readCSVStringInto(data, istr, csv); }); } diff --git a/dbms/src/DataTypes/DataTypeFixedString.h b/dbms/src/DataTypes/DataTypeFixedString.h index 34a3f02c3c1..abea529ad42 100644 --- a/dbms/src/DataTypes/DataTypeFixedString.h +++ b/dbms/src/DataTypes/DataTypeFixedString.h @@ -82,6 +82,7 @@ public: size_t getSizeOfValueInMemory() const override { return n; } bool isCategorial() const override { return true; } bool canBeInsideNullable() const override { return true; } + bool canBeInsideLowCardinality() const override { return true; } }; } diff --git a/dbms/src/DataTypes/DataTypeLowCardinality.cpp b/dbms/src/DataTypes/DataTypeLowCardinality.cpp index 1e973bc9c9b..cf38941b743 100644 --- a/dbms/src/DataTypes/DataTypeLowCardinality.cpp +++ b/dbms/src/DataTypes/DataTypeLowCardinality.cpp @@ -42,9 +42,7 @@ DataTypeLowCardinality::DataTypeLowCardinality(DataTypePtr dictionary_type_) if (dictionary_type->isNullable()) inner_type = static_cast(*dictionary_type).getNestedType(); - if (!isStringOrFixedString(inner_type) - && !isDateOrDateTime(inner_type) - && !isNumber(inner_type)) + if (!inner_type->canBeInsideLowCardinality()) throw Exception("DataTypeLowCardinality is supported only for numbers, strings, Date or DateTime, but got " + dictionary_type->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -58,7 +56,7 @@ void DataTypeLowCardinality::enumerateStreams(const StreamCallback & callback, S path.pop_back(); } -struct KeysSerializationVersion +struct KeysSerializationVersion { enum Value { diff --git a/dbms/src/DataTypes/DataTypeLowCardinality.h b/dbms/src/DataTypes/DataTypeLowCardinality.h index a51e009534b..86698ded893 100644 --- a/dbms/src/DataTypes/DataTypeLowCardinality.h +++ b/dbms/src/DataTypes/DataTypeLowCardinality.h @@ -96,7 +96,7 @@ public: serializeImpl(column, row_num, ostr, &IDataType::serializeText, settings); } - void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override + void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override { serializeImpl(column, row_num, ostr, &IDataType::serializeTextJSON, settings); } diff --git a/dbms/src/DataTypes/DataTypeNullable.cpp b/dbms/src/DataTypes/DataTypeNullable.cpp index ebd10fe9735..6322f5b2b17 100644 --- a/dbms/src/DataTypes/DataTypeNullable.cpp +++ b/dbms/src/DataTypes/DataTypeNullable.cpp @@ -161,7 +161,7 @@ void DataTypeNullable::deserializeBinary(IColumn & column, ReadBuffer & istr) co { safeDeserialize(column, [&istr] { bool is_null = 0; readBinary(is_null, istr); return is_null; }, - [this, &istr] (IColumn & nested) { nested_data_type->deserializeBinary(nested, istr); } ); + [this, &istr] (IColumn & nested) { nested_data_type->deserializeBinary(nested, istr); }); } @@ -188,7 +188,7 @@ void DataTypeNullable::deserializeTextEscaped(IColumn & column, ReadBuffer & ist { safeDeserialize(column, [] { return false; }, - [this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeTextEscaped(nested, istr, settings); } ); + [this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeTextEscaped(nested, istr, settings); }); } else { @@ -248,7 +248,7 @@ void DataTypeNullable::deserializeTextQuoted(IColumn & column, ReadBuffer & istr { safeDeserialize(column, [&istr] { return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr); }, - [this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeTextQuoted(nested, istr, settings); } ); + [this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeTextQuoted(nested, istr, settings); }); } void DataTypeNullable::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const @@ -265,7 +265,7 @@ void DataTypeNullable::deserializeTextCSV(IColumn & column, ReadBuffer & istr, c { safeDeserialize(column, [&istr] { return checkStringByFirstCharacterAndAssertTheRest("\\N", istr); }, - [this, &settings, &istr] (IColumn & nested) { nested_data_type->deserializeTextCSV(nested, istr, settings); } ); + [this, &settings, &istr] (IColumn & nested) { nested_data_type->deserializeTextCSV(nested, istr, settings); }); } void DataTypeNullable::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const @@ -298,7 +298,7 @@ void DataTypeNullable::deserializeTextJSON(IColumn & column, ReadBuffer & istr, { safeDeserialize(column, [&istr] { return checkStringByFirstCharacterAndAssertTheRest("null", istr); }, - [this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeTextJSON(nested, istr, settings); } ); + [this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeTextJSON(nested, istr, settings); }); } void DataTypeNullable::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const diff --git a/dbms/src/DataTypes/DataTypeNullable.h b/dbms/src/DataTypes/DataTypeNullable.h index a8bb00862aa..c6c228d441d 100644 --- a/dbms/src/DataTypes/DataTypeNullable.h +++ b/dbms/src/DataTypes/DataTypeNullable.h @@ -91,6 +91,7 @@ public: bool isNullable() const override { return true; } size_t getSizeOfValueInMemory() const override; bool onlyNull() const override; + bool canBeInsideLowCardinality() const override { return nested_data_type->canBeInsideLowCardinality(); } const DataTypePtr & getNestedType() const { return nested_data_type; } diff --git a/dbms/src/DataTypes/DataTypeNumberBase.h b/dbms/src/DataTypes/DataTypeNumberBase.h index 2aac709090c..67da1c7602e 100644 --- a/dbms/src/DataTypes/DataTypeNumberBase.h +++ b/dbms/src/DataTypes/DataTypeNumberBase.h @@ -51,6 +51,7 @@ public: bool haveMaximumSizeOfValue() const override { return true; } size_t getSizeOfValueInMemory() const override { return sizeof(T); } bool isCategorial() const override { return isValueRepresentedByInteger(); } + bool canBeInsideLowCardinality() const override { return true; } }; } diff --git a/dbms/src/DataTypes/DataTypeString.cpp b/dbms/src/DataTypes/DataTypeString.cpp index 2023523aa41..e33cf5d064e 100644 --- a/dbms/src/DataTypes/DataTypeString.cpp +++ b/dbms/src/DataTypes/DataTypeString.cpp @@ -53,7 +53,7 @@ void DataTypeString::serializeBinary(const IColumn & column, size_t row_num, Wri void DataTypeString::deserializeBinary(IColumn & column, ReadBuffer & istr) const { ColumnString & column_string = static_cast(column); - ColumnString::Chars_t & data = column_string.getChars(); + ColumnString::Chars & data = column_string.getChars(); ColumnString::Offsets & offsets = column_string.getOffsets(); UInt64 size; @@ -81,7 +81,7 @@ void DataTypeString::deserializeBinary(IColumn & column, ReadBuffer & istr) cons void DataTypeString::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { const ColumnString & column_string = typeid_cast(column); - const ColumnString::Chars_t & data = column_string.getChars(); + const ColumnString::Chars & data = column_string.getChars(); const ColumnString::Offsets & offsets = column_string.getOffsets(); size_t size = column.size(); @@ -111,7 +111,7 @@ void DataTypeString::serializeBinaryBulk(const IColumn & column, WriteBuffer & o template -static NO_INLINE void deserializeBinarySSE2(ColumnString::Chars_t & data, ColumnString::Offsets & offsets, ReadBuffer & istr, size_t limit) +static NO_INLINE void deserializeBinarySSE2(ColumnString::Chars & data, ColumnString::Offsets & offsets, ReadBuffer & istr, size_t limit) { size_t offset = data.size(); for (size_t i = 0; i < limit; ++i) @@ -174,7 +174,7 @@ static NO_INLINE void deserializeBinarySSE2(ColumnString::Chars_t & data, Column void DataTypeString::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const { ColumnString & column_string = typeid_cast(column); - ColumnString::Chars_t & data = column_string.getChars(); + ColumnString::Chars & data = column_string.getChars(); ColumnString::Offsets & offsets = column_string.getOffsets(); double avg_chars_size = 1; /// By default reserve only for empty strings. @@ -235,7 +235,7 @@ template static inline void read(IColumn & column, Reader && reader) { ColumnString & column_string = static_cast(column); - ColumnString::Chars_t & data = column_string.getChars(); + ColumnString::Chars & data = column_string.getChars(); ColumnString::Offsets & offsets = column_string.getOffsets(); size_t old_chars_size = data.size(); @@ -258,7 +258,7 @@ static inline void read(IColumn & column, Reader && reader) void DataTypeString::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { - read(column, [&](ColumnString::Chars_t & data) { readEscapedStringInto(data, istr); }); + read(column, [&](ColumnString::Chars & data) { readEscapedStringInto(data, istr); }); } @@ -270,7 +270,7 @@ void DataTypeString::serializeTextQuoted(const IColumn & column, size_t row_num, void DataTypeString::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { - read(column, [&](ColumnString::Chars_t & data) { readQuotedStringInto(data, istr); }); + read(column, [&](ColumnString::Chars & data) { readQuotedStringInto(data, istr); }); } @@ -282,7 +282,7 @@ void DataTypeString::serializeTextJSON(const IColumn & column, size_t row_num, W void DataTypeString::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { - read(column, [&](ColumnString::Chars_t & data) { readJSONStringInto(data, istr); }); + read(column, [&](ColumnString::Chars & data) { readJSONStringInto(data, istr); }); } @@ -300,7 +300,7 @@ void DataTypeString::serializeTextCSV(const IColumn & column, size_t row_num, Wr void DataTypeString::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - read(column, [&](ColumnString::Chars_t & data) { readCSVStringInto(data, istr, settings.csv); }); + read(column, [&](ColumnString::Chars & data) { readCSVStringInto(data, istr, settings.csv); }); } diff --git a/dbms/src/DataTypes/DataTypeString.h b/dbms/src/DataTypes/DataTypeString.h index b2e36e30cff..d0a210dcbf7 100644 --- a/dbms/src/DataTypes/DataTypeString.h +++ b/dbms/src/DataTypes/DataTypeString.h @@ -61,6 +61,7 @@ public: bool isValueUnambiguouslyRepresentedInContiguousMemoryRegion() const override { return true; } bool isCategorial() const override { return true; } bool canBeInsideNullable() const override { return true; } + bool canBeInsideLowCardinality() const override { return true; } }; } diff --git a/dbms/src/DataTypes/DataTypesDecimal.cpp b/dbms/src/DataTypes/DataTypesDecimal.cpp index 9f81107eb68..0c5bd6c6559 100644 --- a/dbms/src/DataTypes/DataTypesDecimal.cpp +++ b/dbms/src/DataTypes/DataTypesDecimal.cpp @@ -123,7 +123,7 @@ void DataTypeDecimal::deserializeBinary(IColumn & column, ReadBuffer & istr) } template -void DataTypeDecimal::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double ) const +void DataTypeDecimal::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double) const { typename ColumnType::Container & x = typeid_cast(column).getData(); size_t initial_size = x.size(); diff --git a/dbms/src/DataTypes/DataTypesDecimal.h b/dbms/src/DataTypes/DataTypesDecimal.h index 863ba7bcae1..96319dccfa4 100644 --- a/dbms/src/DataTypes/DataTypesDecimal.h +++ b/dbms/src/DataTypes/DataTypesDecimal.h @@ -152,7 +152,7 @@ public: /// @returns multiplier for U to become T with correct scale template - T scaleFactorFor(const DataTypeDecimal & x, bool ) const + T scaleFactorFor(const DataTypeDecimal & x, bool) const { if (getScale() < x.getScale()) throw Exception("Decimal result's scale is less then argiment's one", ErrorCodes::ARGUMENT_OUT_OF_BOUND); diff --git a/dbms/src/DataTypes/FieldToDataType.cpp b/dbms/src/DataTypes/FieldToDataType.cpp index 10f083173e4..3086c9167c2 100644 --- a/dbms/src/DataTypes/FieldToDataType.cpp +++ b/dbms/src/DataTypes/FieldToDataType.cpp @@ -30,7 +30,7 @@ DataTypePtr FieldToDataType::operator() (const Null &) const DataTypePtr FieldToDataType::operator() (const UInt64 & x) const { - if (x <= std::numeric_limits::max()) return std::make_shared(); + if (x <= std::numeric_limits::max()) return std::make_shared(); if (x <= std::numeric_limits::max()) return std::make_shared(); if (x <= std::numeric_limits::max()) return std::make_shared(); return std::make_shared(); @@ -43,7 +43,7 @@ DataTypePtr FieldToDataType::operator() (const UInt128 &) const DataTypePtr FieldToDataType::operator() (const Int64 & x) const { - if (x <= std::numeric_limits::max() && x >= std::numeric_limits::min()) return std::make_shared(); + if (x <= std::numeric_limits::max() && x >= std::numeric_limits::min()) return std::make_shared(); if (x <= std::numeric_limits::max() && x >= std::numeric_limits::min()) return std::make_shared(); if (x <= std::numeric_limits::max() && x >= std::numeric_limits::min()) return std::make_shared(); return std::make_shared(); diff --git a/dbms/src/DataTypes/IDataType.h b/dbms/src/DataTypes/IDataType.h index 727d80540ce..f88fdfc22a5 100644 --- a/dbms/src/DataTypes/IDataType.h +++ b/dbms/src/DataTypes/IDataType.h @@ -398,6 +398,8 @@ public: virtual bool lowCardinality() const { return false; } + /// Strings, Numbers, Date, DateTime, Nullable + virtual bool canBeInsideLowCardinality() const { return false; } /// Updates avg_value_size_hint for newly read column. Uses to optimize deserialization. Zero expected for first column. static void updateAvgValueSizeHint(const IColumn & column, double & avg_value_size_hint); diff --git a/dbms/src/DataTypes/NumberTraits.h b/dbms/src/DataTypes/NumberTraits.h index 3ebe9420052..c3642f20c3b 100644 --- a/dbms/src/DataTypes/NumberTraits.h +++ b/dbms/src/DataTypes/NumberTraits.h @@ -43,18 +43,18 @@ template <> struct Construct { using Type = UInt8; }; template <> struct Construct { using Type = UInt16; }; template <> struct Construct { using Type = UInt32; }; template <> struct Construct { using Type = UInt64; }; -template <> struct Construct { using Type = Float32; }; -template <> struct Construct { using Type = Float32; }; -template <> struct Construct { using Type = Float32; }; -template <> struct Construct { using Type = Float64; }; -template <> struct Construct { using Type = Int8; }; -template <> struct Construct { using Type = Int16; }; -template <> struct Construct { using Type = Int32; }; -template <> struct Construct { using Type = Int64; }; -template <> struct Construct { using Type = Float32; }; -template <> struct Construct { using Type = Float32; }; -template <> struct Construct { using Type = Float32; }; -template <> struct Construct { using Type = Float64; }; +template <> struct Construct { using Type = Float32; }; +template <> struct Construct { using Type = Float32; }; +template <> struct Construct { using Type = Float32; }; +template <> struct Construct { using Type = Float64; }; +template <> struct Construct { using Type = Int8; }; +template <> struct Construct { using Type = Int16; }; +template <> struct Construct { using Type = Int32; }; +template <> struct Construct { using Type = Int64; }; +template <> struct Construct { using Type = Float32; }; +template <> struct Construct { using Type = Float32; }; +template <> struct Construct { using Type = Float32; }; +template <> struct Construct { using Type = Float64; }; /** The result of addition or multiplication is calculated according to the following rules: diff --git a/dbms/src/DataTypes/tests/CMakeLists.txt b/dbms/src/DataTypes/tests/CMakeLists.txt index 6186c7dfef4..c2afc6eb2fe 100644 --- a/dbms/src/DataTypes/tests/CMakeLists.txt +++ b/dbms/src/DataTypes/tests/CMakeLists.txt @@ -1,10 +1,10 @@ set(SRCS ) add_executable (data_types_number_fixed data_types_number_fixed.cpp ${SRCS}) -target_link_libraries (data_types_number_fixed dbms) +target_link_libraries (data_types_number_fixed PRIVATE dbms) add_executable (data_type_string data_type_string.cpp ${SRCS}) -target_link_libraries (data_type_string dbms) +target_link_libraries (data_type_string PRIVATE dbms) add_executable (data_type_get_common_type data_type_get_common_type.cpp ${SRCS}) -target_link_libraries (data_type_get_common_type dbms gtest_main) +target_link_libraries (data_type_get_common_type PRIVATE dbms gtest_main) diff --git a/dbms/src/DataTypes/tests/data_type_string.cpp b/dbms/src/DataTypes/tests/data_type_string.cpp index 65a15d059b4..bb394b1784c 100644 --- a/dbms/src/DataTypes/tests/data_type_string.cpp +++ b/dbms/src/DataTypes/tests/data_type_string.cpp @@ -25,7 +25,7 @@ try { auto column = ColumnString::create(); - ColumnString::Chars_t & data = column->getChars(); + ColumnString::Chars & data = column->getChars(); ColumnString::Offsets & offsets = column->getOffsets(); data.resize(n * size); diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 7bb4ae298e6..e0fe4294e7d 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -392,7 +392,7 @@ void DatabaseOrdinary::renameTable( catch (const Poco::Exception & e) { /// Better diagnostics. - throw Exception{e}; + throw Exception{Exception::CreateFromPoco, e}; } ASTPtr ast = getQueryFromMetadata(detail::getTableMetadataPath(metadata_path, table_name)); diff --git a/dbms/src/Databases/DatabasesCommon.cpp b/dbms/src/Databases/DatabasesCommon.cpp index 2617390fd6d..2d8dbbc2392 100644 --- a/dbms/src/Databases/DatabasesCommon.cpp +++ b/dbms/src/Databases/DatabasesCommon.cpp @@ -163,7 +163,7 @@ DatabaseWithOwnTablesBase::~DatabaseWithOwnTablesBase() { shutdown(); } - catch(...) + catch (...) { tryLogCurrentException(__PRETTY_FUNCTION__); } diff --git a/dbms/src/Dictionaries/DictionarySourceFactory.cpp b/dbms/src/Dictionaries/DictionarySourceFactory.cpp index 1da2268c1e5..d2deb769839 100644 --- a/dbms/src/Dictionaries/DictionarySourceFactory.cpp +++ b/dbms/src/Dictionaries/DictionarySourceFactory.cpp @@ -155,8 +155,7 @@ DictionarySourcePtr DictionarySourceFactory::create( else if ("odbc" == source_type) { #if USE_POCO_SQLODBC || USE_POCO_DATAODBC - const auto & global_config = context.getConfigRef(); - BridgeHelperPtr bridge = std::make_shared>(global_config, context.getSettings().http_receive_timeout, config.getString(config_prefix + ".odbc.connection_string")); + BridgeHelperPtr bridge = std::make_shared>(context, context.getSettings().http_receive_timeout, config.getString(config_prefix + ".odbc.connection_string")); return std::make_unique(dict_struct, config, config_prefix + ".odbc", sample_block, context, bridge); #else throw Exception{"Dictionary source of type `odbc` is disabled because poco library was built without ODBC support.", diff --git a/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp b/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp index 978d7b9e496..2a277e10fe2 100644 --- a/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp +++ b/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp @@ -10,6 +10,15 @@ #include +namespace DB +{ + namespace ErrorCodes + { + extern const int INCORRECT_DATA; + } +} + + RegionsHierarchy::RegionsHierarchy(IRegionsHierarchyDataSourcePtr data_source_) : data_source(data_source_) { @@ -36,7 +45,7 @@ void RegionsHierarchy::reload() RegionParents new_continent(initial_size); RegionParents new_top_continent(initial_size); RegionPopulations new_populations(initial_size); - RegionDepths new_depths(initial_size); + RegionDepths new_depths(initial_size); RegionTypes types(initial_size); RegionID max_region_id = 0; @@ -49,7 +58,8 @@ void RegionsHierarchy::reload() if (region_entry.id > max_region_id) { if (region_entry.id > max_size) - throw DB::Exception("Region id is too large: " + DB::toString(region_entry.id) + ", should be not more than " + DB::toString(max_size)); + throw DB::Exception("Region id is too large: " + DB::toString(region_entry.id) + ", should be not more than " + DB::toString(max_size), + DB::ErrorCodes::INCORRECT_DATA); max_region_id = region_entry.id; diff --git a/dbms/src/Dictionaries/Embedded/RegionsNames.cpp b/dbms/src/Dictionaries/Embedded/RegionsNames.cpp index 4966f30455a..2c48f1f18c2 100644 --- a/dbms/src/Dictionaries/Embedded/RegionsNames.cpp +++ b/dbms/src/Dictionaries/Embedded/RegionsNames.cpp @@ -8,6 +8,14 @@ #include +namespace DB +{ + namespace ErrorCodes + { + extern const int INCORRECT_DATA; + } +} + RegionsNames::RegionsNames(IRegionsNamesDataProviderPtr data_provider) { @@ -78,7 +86,8 @@ void RegionsNames::reload() max_region_id = name_entry.id; if (name_entry.id > max_size) - throw DB::Exception("Region id is too large: " + DB::toString(name_entry.id) + ", should be not more than " + DB::toString(max_size)); + throw DB::Exception("Region id is too large: " + DB::toString(name_entry.id) + ", should be not more than " + DB::toString(max_size), + DB::ErrorCodes::INCORRECT_DATA); } while (name_entry.id >= new_names_refs.size()) diff --git a/dbms/src/Dictionaries/ExecutableDictionarySource.cpp b/dbms/src/Dictionaries/ExecutableDictionarySource.cpp index 8dae4cfce2a..c17ce3e2461 100644 --- a/dbms/src/Dictionaries/ExecutableDictionarySource.cpp +++ b/dbms/src/Dictionaries/ExecutableDictionarySource.cpp @@ -213,7 +213,7 @@ bool ExecutableDictionarySource::supportsSelectiveLoad() const bool ExecutableDictionarySource::hasUpdateField() const { - if(update_field.empty()) + if (update_field.empty()) return false; else return true; diff --git a/dbms/src/Dictionaries/MongoDBDictionarySource.cpp b/dbms/src/Dictionaries/MongoDBDictionarySource.cpp index ee3c493d3cd..e8fe62bd8b6 100644 --- a/dbms/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/dbms/src/Dictionaries/MongoDBDictionarySource.cpp @@ -86,7 +86,7 @@ static void authenticate(Poco::MongoDB::Connection & connection, Poco::MD5Engine md5; md5.update(first); std::string digest_first(Poco::DigestEngine::digestToHex(md5.digest())); - std::string second = nonce + user + digest_first; + std::string second = nonce + user + digest_first; md5.reset(); md5.update(second); std::string digest_second(Poco::DigestEngine::digestToHex(md5.digest())); diff --git a/dbms/src/Dictionaries/XDBCDictionarySource.cpp b/dbms/src/Dictionaries/XDBCDictionarySource.cpp index 4e9637502e7..e7285f17025 100644 --- a/dbms/src/Dictionaries/XDBCDictionarySource.cpp +++ b/dbms/src/Dictionaries/XDBCDictionarySource.cpp @@ -168,7 +168,7 @@ DictionarySourcePtr XDBCDictionarySource::clone() const std::string XDBCDictionarySource::toString() const { - return bridge_helper->getName() + ": " + db + '.' + table + (where.empty() ? "" : ", where: " + where); + return bridge_helper->getName() + ": " + db + '.' + table + (where.empty() ? "" : ", where: " + where); } bool XDBCDictionarySource::isModified() const diff --git a/dbms/src/Formats/CapnProtoRowInputStream.cpp b/dbms/src/Formats/CapnProtoRowInputStream.cpp index d7cc02a3218..1af37a85cee 100644 --- a/dbms/src/Formats/CapnProtoRowInputStream.cpp +++ b/dbms/src/Formats/CapnProtoRowInputStream.cpp @@ -18,6 +18,13 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_TYPE_OF_FIELD; + extern const int BAD_ARGUMENTS; + extern const int THERE_IS_NO_COLUMN; +} + static String getSchemaPath(const String & schema_dir, const String & schema_file) { return schema_dir + escapeForFileName(schema_file) + ".capnp"; @@ -42,7 +49,7 @@ Field convertNodeToField(capnp::DynamicValue::Reader value) switch (value.getType()) { case capnp::DynamicValue::UNKNOWN: - throw Exception("Unknown field type"); + throw Exception("Unknown field type", ErrorCodes::BAD_TYPE_OF_FIELD); case capnp::DynamicValue::VOID: return Field(); case capnp::DynamicValue::BOOL: @@ -87,9 +94,9 @@ Field convertNodeToField(capnp::DynamicValue::Reader value) return field; } case capnp::DynamicValue::CAPABILITY: - throw Exception("CAPABILITY type not supported"); + throw Exception("CAPABILITY type not supported", ErrorCodes::BAD_TYPE_OF_FIELD); case capnp::DynamicValue::ANY_POINTER: - throw Exception("ANY_POINTER type not supported"); + throw Exception("ANY_POINTER type not supported", ErrorCodes::BAD_TYPE_OF_FIELD); } return Field(); } @@ -99,7 +106,7 @@ capnp::StructSchema::Field getFieldOrThrow(capnp::StructSchema node, const std:: KJ_IF_MAYBE(child, node.findFieldByName(field)) return *child; else - throw Exception("Field " + field + " doesn't exist in schema " + node.getShortDisplayName().cStr()); + throw Exception("Field " + field + " doesn't exist in schema " + node.getShortDisplayName().cStr(), ErrorCodes::THERE_IS_NO_COLUMN); } void CapnProtoRowInputStream::createActions(const NestedFieldList & sortedFields, capnp::StructSchema reader) @@ -135,7 +142,7 @@ void CapnProtoRowInputStream::createActions(const NestedFieldList & sortedFields break; // Collect list } else - throw Exception("Field " + field.tokens[level] + "is neither Struct nor List"); + throw Exception("Field " + field.tokens[level] + "is neither Struct nor List", ErrorCodes::BAD_TYPE_OF_FIELD); } // Read field from the structure @@ -290,7 +297,8 @@ void registerInputFormatCapnProto(FormatFactory & factory) auto schema_and_root = context.getSettingsRef().format_schema.toString(); boost::split(tokens, schema_and_root, boost::is_any_of(":")); if (tokens.size() != 2) - throw Exception("Format CapnProto requires 'format_schema' setting to have a schema_file:root_object format, e.g. 'schema.capnp:Message'"); + throw Exception("Format CapnProto requires 'format_schema' setting to have a schema_file:root_object format, e.g. 'schema.capnp:Message'", + ErrorCodes::BAD_ARGUMENTS); const String & schema_dir = context.getFormatSchemaPath(); diff --git a/dbms/src/Formats/XMLRowOutputStream.cpp b/dbms/src/Formats/XMLRowOutputStream.cpp index f06fc9cc89d..7bd5a5d4c84 100644 --- a/dbms/src/Formats/XMLRowOutputStream.cpp +++ b/dbms/src/Formats/XMLRowOutputStream.cpp @@ -29,7 +29,7 @@ XMLRowOutputStream::XMLRowOutputStream(WriteBuffer & ostr_, const Block & sample for (const char * pos = begin; pos != end; ++pos) { char c = *pos; - if (!( isAlphaASCII(c) + if (!(isAlphaASCII(c) || (pos != begin && isNumericASCII(c)) || c == '_' || c == '-' diff --git a/dbms/src/Formats/tests/CMakeLists.txt b/dbms/src/Formats/tests/CMakeLists.txt index de94fb4d4f3..e12fa0f02fb 100644 --- a/dbms/src/Formats/tests/CMakeLists.txt +++ b/dbms/src/Formats/tests/CMakeLists.txt @@ -1,7 +1,7 @@ set(SRCS ) add_executable (tab_separated_streams tab_separated_streams.cpp ${SRCS}) -target_link_libraries (tab_separated_streams dbms) +target_link_libraries (tab_separated_streams PRIVATE dbms) add_executable (block_row_transforms block_row_transforms.cpp ${SRCS}) -target_link_libraries (block_row_transforms dbms) +target_link_libraries (block_row_transforms PRIVATE dbms) diff --git a/dbms/src/Functions/CMakeLists.txt b/dbms/src/Functions/CMakeLists.txt index d57e3990901..1a1d6f06f64 100644 --- a/dbms/src/Functions/CMakeLists.txt +++ b/dbms/src/Functions/CMakeLists.txt @@ -1,50 +1,8 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) -include(${ClickHouse_SOURCE_DIR}/cmake/dbms_generate_function.cmake) - -set (FUNCTIONS_GENERATED_DIR ${CMAKE_CURRENT_BINARY_DIR}/generated/) - -generate_function_register(Arithmetic - FunctionPlus - FunctionMinus - FunctionMultiply - FunctionDivideFloating - FunctionDivideIntegral - FunctionDivideIntegralOrZero - FunctionModulo - FunctionNegate - FunctionAbs - FunctionBitAnd - FunctionBitOr - FunctionBitXor - FunctionBitNot - FunctionBitShiftLeft - FunctionBitShiftRight - FunctionBitRotateLeft - FunctionBitRotateRight - FunctionLeast - FunctionGreatest - FunctionBitTest - FunctionBitTestAny - FunctionBitTestAll - FunctionGCD - FunctionLCM - FunctionIntExp2 - FunctionIntExp10 -) - -generate_function_register(Projection - FunctionOneOrZero - FunctionProject - FunctionBuildProjectionComposition - FunctionRestoreProjection -) - add_headers_and_sources(clickhouse_functions .) add_headers_and_sources(clickhouse_functions ./GatherUtils) add_headers_and_sources(clickhouse_functions ./Conditional) -#add_headers_and_sources(clickhouse_functions ${ClickHouse_BINARY_DIR}/dbms/src/Functions) -add_headers_and_sources(clickhouse_functions ${FUNCTIONS_GENERATED_DIR}) list(REMOVE_ITEM clickhouse_functions_sources IFunction.cpp FunctionFactory.cpp FunctionHelpers.cpp) @@ -52,7 +10,7 @@ list(REMOVE_ITEM clickhouse_functions_headers IFunction.h FunctionFactory.h Func add_library(clickhouse_functions ${LINK_MODE} ${clickhouse_functions_sources}) -target_link_libraries(clickhouse_functions PUBLIC dbms PRIVATE ${CONSISTENT_HASHING_LIBRARY} consistent-hashing-sumbur ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES} murmurhash) +target_link_libraries(clickhouse_functions PUBLIC dbms PRIVATE ${CONSISTENT_HASHING_LIBRARY} consistent-hashing-sumbur ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES} murmurhash ${BASE64_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${DIVIDE_INCLUDE_DIR}) @@ -81,3 +39,7 @@ endif () if (USE_EMBEDDED_COMPILER) target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) endif () + +if (USE_BASE64) + target_include_directories (clickhouse_functions SYSTEM PRIVATE ${BASE64_INCLUDE_DIR}) +endif() diff --git a/dbms/src/Functions/EmptyImpl.h b/dbms/src/Functions/EmptyImpl.h index e66edf5f4ba..a9702fe8869 100644 --- a/dbms/src/Functions/EmptyImpl.h +++ b/dbms/src/Functions/EmptyImpl.h @@ -19,7 +19,7 @@ struct EmptyImpl /// If the function will return constant value for FixedString data type. static constexpr auto is_fixed_to_constant = false; - static void vector(const ColumnString::Chars_t & /*data*/, const ColumnString::Offsets & offsets, PaddedPODArray & res) + static void vector(const ColumnString::Chars & /*data*/, const ColumnString::Offsets & offsets, PaddedPODArray & res) { size_t size = offsets.size(); ColumnString::Offset prev_offset = 1; @@ -31,12 +31,12 @@ struct EmptyImpl } /// Only make sense if is_fixed_to_constant. - static void vector_fixed_to_constant(const ColumnString::Chars_t & /*data*/, size_t /*n*/, UInt8 & /*res*/) + static void vector_fixed_to_constant(const ColumnString::Chars & /*data*/, size_t /*n*/, UInt8 & /*res*/) { throw Exception("Logical error: 'vector_fixed_to_constant method' is called", ErrorCodes::LOGICAL_ERROR); } - static void vector_fixed_to_vector(const ColumnString::Chars_t & data, size_t n, PaddedPODArray & res) + static void vector_fixed_to_vector(const ColumnString::Chars & data, size_t n, PaddedPODArray & res) { std::vector empty_chars(n); size_t size = data.size() / n; diff --git a/dbms/src/Functions/FunctionArrayMapped.h b/dbms/src/Functions/FunctionArrayMapped.h index c23e756d4d8..f14ac244a99 100644 --- a/dbms/src/Functions/FunctionArrayMapped.h +++ b/dbms/src/Functions/FunctionArrayMapped.h @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB @@ -70,7 +71,7 @@ public: if (!array_type) throw Exception("Argument " + toString(i + 2) + " of function " + getName() + " must be array. Found " + arguments[i + 1]->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - nested_types[i] = array_type->getNestedType(); + nested_types[i] = removeLowCardinality(array_type->getNestedType()); } const DataTypeFunction * function_type = checkAndGetDataType(arguments[0].get()); @@ -121,7 +122,7 @@ public: /// The types of the remaining arguments are already checked in getLambdaArgumentTypes. - DataTypePtr return_type = data_type_function->getReturnType(); + DataTypePtr return_type = removeLowCardinality(data_type_function->getReturnType()); if (Impl::needBoolean() && !WhichDataType(return_type).isUInt8()) throw Exception("Expression for function " + getName() + " must return UInt8, found " + return_type->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -184,6 +185,8 @@ public: if (!column_const_array) throw Exception("Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); column_array_ptr = column_const_array->convertToFullColumn(); + if (column_array_ptr->lowCardinality()) + column_array_ptr = column_array_ptr->convertToFullColumnIfLowCardinality(); column_array = checkAndGetColumn(column_array_ptr.get()); } @@ -209,7 +212,8 @@ public: } arrays.emplace_back(ColumnWithTypeAndName(column_array->getDataPtr(), - array_type->getNestedType(), array_with_type_and_name.name)); + removeLowCardinality(array_type->getNestedType()), + array_with_type_and_name.name)); } /// Put all the necessary columns multiplied by the sizes of arrays into the block. @@ -217,8 +221,11 @@ public: auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); replicated_column_function->appendArguments(arrays); - block.getByPosition(result).column = Impl::execute(*column_first_array, - replicated_column_function->reduce().column); + auto lambda_result = replicated_column_function->reduce().column; + if (lambda_result->lowCardinality()) + lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); + + block.getByPosition(result).column = Impl::execute(*column_first_array, lambda_result); } } }; diff --git a/dbms/src/Functions/FunctionBase64Conversion.h b/dbms/src/Functions/FunctionBase64Conversion.h new file mode 100644 index 00000000000..5ee6ff8f257 --- /dev/null +++ b/dbms/src/Functions/FunctionBase64Conversion.h @@ -0,0 +1,169 @@ +#include +#if USE_BASE64 +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +using namespace GatherUtils; + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int INCORRECT_DATA; +} + +struct Base64Encode +{ + static constexpr auto name = "base64Encode"; + static size_t getBufferSize(size_t string_length, size_t string_count) + { + return ((string_length - string_count) / 3 + string_count) * 4 + string_count; + } +}; + +struct Base64Decode +{ + static constexpr auto name = "base64Decode"; + + static size_t getBufferSize(size_t string_length, size_t string_count) + { + return ((string_length - string_count) / 4 + string_count) * 3 + string_count; + } +}; + +struct TryBase64Decode +{ + static constexpr auto name = "tryBase64Decode"; + + static size_t getBufferSize(size_t string_length, size_t string_count) + { + return Base64Decode::getBufferSize(string_length, string_count); + } +}; + +template +class FunctionBase64Conversion : public IFunction +{ +public: + static constexpr auto name = Func::name; + + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return Func::name; + } + + size_t getNumberOfArguments() const override + { + return 1; + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (!WhichDataType(arguments[0].type).isString()) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of 1 argument of function " + getName() + ". Must be String.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + const ColumnPtr column_string = block.getByPosition(arguments[0]).column; + const ColumnString * input = checkAndGetColumn(column_string.get()); + + if (!input) + throw Exception( + "Illegal column " + block.getByPosition(arguments[0]).column->getName() + " of first argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + + auto dst_column = ColumnString::create(); + auto & dst_data = dst_column->getChars(); + auto & dst_offsets = dst_column->getOffsets(); + + size_t reserve = Func::getBufferSize(input->getChars().size(), input->size()); + dst_data.resize(reserve); + dst_offsets.resize(input_rows_count); + + const ColumnString::Offsets & src_offsets = input->getOffsets(); + + auto source = reinterpret_cast(input->getChars().data()); + auto dst = reinterpret_cast(dst_data.data()); + auto dst_pos = dst; + + size_t src_offset_prev = 0; + + int codec = getCodec(); + for (size_t row = 0; row < input_rows_count; ++row) + { + size_t srclen = src_offsets[row] - src_offset_prev - 1; + size_t outlen = 0; + + if constexpr (std::is_same_v) + { + base64_encode(source, srclen, dst_pos, &outlen, codec); + } + else if constexpr (std::is_same_v) + { + if (!base64_decode(source, srclen, dst_pos, &outlen, codec)) + { + throw Exception("Failed to " + getName() + " input '" + String(source, srclen) + "'", ErrorCodes::INCORRECT_DATA); + } + } + else + { + // during decoding character array can be partially polluted + // if fail, revert back and clean + auto savepoint = dst_pos; + if (!base64_decode(source, srclen, dst_pos, &outlen, codec)) + { + outlen = 0; + dst_pos = savepoint; + // clean the symbol + dst_pos[0] = 0; + } + } + + source += srclen + 1; + dst_pos += outlen + 1; + + dst_offsets[row] = dst_pos - dst; + src_offset_prev = src_offsets[row]; + } + + dst_data.resize(dst_pos - dst); + + block.getByPosition(result).column = std::move(dst_column); + } + +private: + static int getCodec() + { + /// You can provide different value if you want to test specific codecs. + /// Due to poor implementation of "base64" library (it will write to a global variable), + /// it doesn't scale for multiple threads. Never use non-zero values in production. + return 0; + } +}; +} +#endif diff --git a/dbms/src/Functions/FunctionBinaryArithmetic.h b/dbms/src/Functions/FunctionBinaryArithmetic.h new file mode 100644 index 00000000000..a668c43b122 --- /dev/null +++ b/dbms/src/Functions/FunctionBinaryArithmetic.h @@ -0,0 +1,921 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if USE_EMBEDDED_COMPILER +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#include // Y_IGNORE +#pragma GCC diagnostic pop +#endif + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int LOGICAL_ERROR; + extern const int DECIMAL_OVERFLOW; + extern const int CANNOT_ADD_DIFFERENT_AGGREGATE_STATES; + extern const int ILLEGAL_DIVISION; +} + + +/** Arithmetic operations: +, -, *, /, %, + * intDiv (integer division) + * Bitwise operations: |, &, ^, ~. + * Etc. + */ + +template +struct BinaryOperationImplBase +{ + using ResultType = ResultType_; + + static void NO_INLINE vector_vector(const PaddedPODArray & a, const PaddedPODArray & b, PaddedPODArray & c) + { + size_t size = a.size(); + for (size_t i = 0; i < size; ++i) + c[i] = Op::template apply(a[i], b[i]); + } + + static void NO_INLINE vector_constant(const PaddedPODArray & a, B b, PaddedPODArray & c) + { + size_t size = a.size(); + for (size_t i = 0; i < size; ++i) + c[i] = Op::template apply(a[i], b); + } + + static void NO_INLINE constant_vector(A a, const PaddedPODArray & b, PaddedPODArray & c) + { + size_t size = b.size(); + for (size_t i = 0; i < size; ++i) + c[i] = Op::template apply(a, b[i]); + } + + static ResultType constant_constant(A a, B b) + { + return Op::template apply(a, b); + } +}; + +template +struct BinaryOperationImpl : BinaryOperationImplBase +{ +}; + + +template struct PlusImpl; +template struct MinusImpl; +template struct MultiplyImpl; +template struct DivideFloatingImpl; +template struct DivideIntegralImpl; +template struct DivideIntegralOrZeroImpl; +template struct LeastBaseImpl; +template struct GreatestBaseImpl; +template struct ModuloImpl; + + +template struct NativeType { using Type = T; }; +template <> struct NativeType { using Type = Int32; }; +template <> struct NativeType { using Type = Int64; }; +template <> struct NativeType { using Type = Int128; }; + +/// Binary operations for Decimals need scale args +/// +|- scale one of args (which scale factor is not 1). ScaleR = oneof(Scale1, Scale2); +/// * no agrs scale. ScaleR = Scale1 + Scale2; +/// / first arg scale. ScaleR = Scale1 (scale_a = DecimalType::getScale()). +template typename Operation, typename ResultType_, bool _check_overflow = true> +struct DecimalBinaryOperation +{ + static constexpr bool is_plus_minus = std::is_same_v, PlusImpl> || + std::is_same_v, MinusImpl>; + static constexpr bool is_multiply = std::is_same_v, MultiplyImpl>; + static constexpr bool is_float_division = std::is_same_v, DivideFloatingImpl>; + static constexpr bool is_int_division = std::is_same_v, DivideIntegralImpl> || + std::is_same_v, DivideIntegralOrZeroImpl>; + static constexpr bool is_division = is_float_division || is_int_division; + static constexpr bool is_compare = std::is_same_v, LeastBaseImpl> || + std::is_same_v, GreatestBaseImpl>; + static constexpr bool is_plus_minus_compare = is_plus_minus || is_compare; + static constexpr bool can_overflow = is_plus_minus || is_multiply; + + using ResultType = ResultType_; + using NativeResultType = typename NativeType::Type; + using Op = std::conditional_t, /// substitute divide by intDiv (throw on division by zero) + Operation>; + using ColVecA = std::conditional_t, ColumnDecimal, ColumnVector>; + using ColVecB = std::conditional_t, ColumnDecimal, ColumnVector>; + using ArrayA = typename ColVecA::Container; + using ArrayB = typename ColVecB::Container; + using ArrayC = typename ColumnDecimal::Container; + using SelfNoOverflow = DecimalBinaryOperation; + + static void vector_vector(const ArrayA & a, const ArrayB & b, ArrayC & c, ResultType scale_a, ResultType scale_b, bool check_overflow) + { + if (check_overflow) + vector_vector(a, b, c, scale_a, scale_b); + else + SelfNoOverflow::vector_vector(a, b, c, scale_a, scale_b); + } + + static void vector_constant(const ArrayA & a, B b, ArrayC & c, ResultType scale_a, ResultType scale_b, bool check_overflow) + { + if (check_overflow) + vector_constant(a, b, c, scale_a, scale_b); + else + SelfNoOverflow::vector_constant(a, b, c, scale_a, scale_b); + } + + static void constant_vector(A a, const ArrayB & b, ArrayC & c, ResultType scale_a, ResultType scale_b, bool check_overflow) + { + if (check_overflow) + constant_vector(a, b, c, scale_a, scale_b); + else + SelfNoOverflow::constant_vector(a, b, c, scale_a, scale_b); + } + + static ResultType constant_constant(A a, B b, ResultType scale_a, ResultType scale_b, bool check_overflow) + { + if (check_overflow) + return constant_constant(a, b, scale_a, scale_b); + else + return SelfNoOverflow::constant_constant(a, b, scale_a, scale_b); + } + + static void NO_INLINE vector_vector(const ArrayA & a, const ArrayB & b, ArrayC & c, + ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) + { + size_t size = a.size(); + if constexpr (is_plus_minus_compare) + { + if (scale_a != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a[i], b[i], scale_a); + return; + } + else if (scale_b != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a[i], b[i], scale_b); + return; + } + } + else if constexpr (is_division && IsDecimalNumber) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaledDiv(a[i], b[i], scale_a); + return; + } + + /// default: use it if no return before + for (size_t i = 0; i < size; ++i) + c[i] = apply(a[i], b[i]); + } + + static void NO_INLINE vector_constant(const ArrayA & a, B b, ArrayC & c, + ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) + { + size_t size = a.size(); + if constexpr (is_plus_minus_compare) + { + if (scale_a != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a[i], b, scale_a); + return; + } + else if (scale_b != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a[i], b, scale_b); + return; + } + } + else if constexpr (is_division && IsDecimalNumber) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaledDiv(a[i], b, scale_a); + return; + } + + /// default: use it if no return before + for (size_t i = 0; i < size; ++i) + c[i] = apply(a[i], b); + } + + static void NO_INLINE constant_vector(A a, const ArrayB & b, ArrayC & c, + ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) + { + size_t size = b.size(); + if constexpr (is_plus_minus_compare) + { + if (scale_a != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a, b[i], scale_a); + return; + } + else if (scale_b != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a, b[i], scale_b); + return; + } + } + else if constexpr (is_division && IsDecimalNumber) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaledDiv(a, b[i], scale_a); + return; + } + + /// default: use it if no return before + for (size_t i = 0; i < size; ++i) + c[i] = apply(a, b[i]); + } + + static ResultType constant_constant(A a, B b, ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) + { + if constexpr (is_plus_minus_compare) + { + if (scale_a != 1) + return applyScaled(a, b, scale_a); + else if (scale_b != 1) + return applyScaled(a, b, scale_b); + } + else if constexpr (is_division && IsDecimalNumber) + return applyScaledDiv(a, b, scale_a); + return apply(a, b); + } + +private: + /// there's implicit type convertion here + static NativeResultType apply(NativeResultType a, NativeResultType b) + { + if constexpr (can_overflow && _check_overflow) + { + NativeResultType res; + if (Op::template apply(a, b, res)) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); + return res; + } + else + return Op::template apply(a, b); + } + + template + static NativeResultType applyScaled(NativeResultType a, NativeResultType b, NativeResultType scale) + { + if constexpr (is_plus_minus_compare) + { + NativeResultType res; + + if constexpr (_check_overflow) + { + bool overflow = false; + if constexpr (scale_left) + overflow |= common::mulOverflow(a, scale, a); + else + overflow |= common::mulOverflow(b, scale, b); + + if constexpr (can_overflow) + overflow |= Op::template apply(a, b, res); + else + res = Op::template apply(a, b); + + if (overflow) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); + } + else + { + if constexpr (scale_left) + a *= scale; + else + b *= scale; + res = Op::template apply(a, b); + } + + return res; + } + } + + static NativeResultType applyScaledDiv(NativeResultType a, NativeResultType b, NativeResultType scale) + { + if constexpr (is_division) + { + if constexpr (_check_overflow) + { + bool overflow = false; + if constexpr (!IsDecimalNumber) + overflow |= common::mulOverflow(scale, scale, scale); + overflow |= common::mulOverflow(a, scale, a); + if (overflow) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); + } + else + { + if constexpr (!IsDecimalNumber) + scale *= scale; + a *= scale; + } + + return Op::template apply(a, b); + } + } +}; + + +/// Used to indicate undefined operation +struct InvalidType; + +template struct Case : std::bool_constant { using type = T; }; + +/// Switch, ...> -- select the first Ti for which Ci is true; InvalidType if none. +template using Switch = typename std::disjunction>::type; + +template constexpr bool IsIntegral = false; +template <> constexpr bool IsIntegral = true; +template <> constexpr bool IsIntegral = true; +template <> constexpr bool IsIntegral = true; +template <> constexpr bool IsIntegral = true; +template <> constexpr bool IsIntegral = true; +template <> constexpr bool IsIntegral = true; +template <> constexpr bool IsIntegral = true; +template <> constexpr bool IsIntegral = true; + +template constexpr bool IsFloatingPoint = false; +template <> constexpr bool IsFloatingPoint = true; +template <> constexpr bool IsFloatingPoint = true; + +template constexpr bool IsDateOrDateTime = false; +template <> constexpr bool IsDateOrDateTime = true; +template <> constexpr bool IsDateOrDateTime = true; + +template constexpr bool UseLeftDecimal = false; +template <> constexpr bool UseLeftDecimal, DataTypeDecimal> = true; +template <> constexpr bool UseLeftDecimal, DataTypeDecimal> = true; +template <> constexpr bool UseLeftDecimal, DataTypeDecimal> = true; + +template using DataTypeFromFieldType = std::conditional_t, InvalidType, DataTypeNumber>; + +template