mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-10-13 20:10:50 +00:00
Merge branch 'master' of github.com:yandex/ClickHouse
This commit is contained in:
commit
4f2b06a6c7
3
.gitmodules
vendored
3
.gitmodules
vendored
@ -46,3 +46,6 @@
|
||||
[submodule "contrib/unixodbc"]
|
||||
path = contrib/unixodbc
|
||||
url = https://github.com/ClickHouse-Extras/UnixODBC.git
|
||||
[submodule "contrib/base64"]
|
||||
path = contrib/base64
|
||||
url = https://github.com/aklomp/base64.git
|
||||
|
@ -1,10 +1,15 @@
|
||||
## ClickHouse release 18.14.15, 2018-11-21
|
||||
|
||||
### Bug fixes:
|
||||
* The size of memory chunk was overestimated while deserializing the column of type `Array(String)` that leads to "Memory limit exceeded" errors. The issue appeared in version 18.12.13. [#3589](https://github.com/yandex/ClickHouse/issues/3589)
|
||||
|
||||
## ClickHouse release 18.14.14, 2018-11-20
|
||||
|
||||
### Bug fixes:
|
||||
* Fixed `ON CLUSTER` queries when cluster configured as secure (flag `<secure>`). [#3599]
|
||||
* Fixed `ON CLUSTER` queries when cluster configured as secure (flag `<secure>`). [#3599](https://github.com/yandex/ClickHouse/pull/3599)
|
||||
|
||||
### Build changes:
|
||||
* Fixed problems (llvm-7 from system, macos) [#3582]
|
||||
* Fixed problems (llvm-7 from system, macos) [#3582](https://github.com/yandex/ClickHouse/pull/3582)
|
||||
|
||||
## ClickHouse release 18.14.11, 2018-10-29
|
||||
|
||||
|
@ -1,10 +1,15 @@
|
||||
## ClickHouse release 18.14.15, 2018-11-21
|
||||
|
||||
### Исправления ошибок:
|
||||
* При чтении столбцов типа `Array(String)`, размер требуемого куска памяти оценивался слишком большим, что приводило к исключению "Memory limit exceeded" при выполнении запроса. Ошибка появилась в версии 18.12.13. [#3589](https://github.com/yandex/ClickHouse/issues/3589)
|
||||
|
||||
## ClickHouse release 18.14.14, 2018-11-20
|
||||
|
||||
### Исправления ошибок:
|
||||
* Исправлена работа запросов `ON CLUSTER` в случае, когда в конфигурации кластера включено шифрование (флаг `<secure>`). [#3599]
|
||||
* Исправлена работа запросов `ON CLUSTER` в случае, когда в конфигурации кластера включено шифрование (флаг `<secure>`). [#3599](https://github.com/yandex/ClickHouse/pull/3599)
|
||||
|
||||
### Улучшения процесса сборки ClickHouse:
|
||||
* Испрпавлены проблемы сборки (llvm-7 из системы, macos) [#3582]
|
||||
* Испрпавлены проблемы сборки (llvm-7 из системы, macos) [#3582](https://github.com/yandex/ClickHouse/pull/3582)
|
||||
|
||||
## ClickHouse release 18.14.13, 2018-11-08
|
||||
|
||||
|
@ -256,6 +256,7 @@ include (cmake/find_capnp.cmake)
|
||||
include (cmake/find_llvm.cmake)
|
||||
include (cmake/find_cpuid.cmake)
|
||||
include (cmake/find_consistent-hashing.cmake)
|
||||
include (cmake/find_base64.cmake)
|
||||
if (ENABLE_TESTS)
|
||||
include (cmake/find_gtest.cmake)
|
||||
endif ()
|
||||
|
@ -7,4 +7,6 @@ ClickHouse is an open-source column-oriented database management system that all
|
||||
* [Official website](https://clickhouse.yandex/) has quick high-level overview of ClickHouse on main page.
|
||||
* [Tutorial](https://clickhouse.yandex/tutorial.html) shows how to set up and query small ClickHouse cluster.
|
||||
* [Documentation](https://clickhouse.yandex/docs/en/) provides more in-depth information.
|
||||
* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
|
||||
* [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any.
|
||||
* You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person.
|
||||
|
@ -1,10 +0,0 @@
|
||||
function(generate_function_register FUNCTION_AREA)
|
||||
foreach(FUNCTION IN LISTS ARGN)
|
||||
configure_file (registerFunction.h.in ${FUNCTIONS_GENERATED_DIR}register${FUNCTION}.h)
|
||||
configure_file (registerFunction.cpp.in ${FUNCTIONS_GENERATED_DIR}register${FUNCTION}.cpp)
|
||||
set(REGISTER_HEADERS "${REGISTER_HEADERS}#include \"register${FUNCTION}.h\"\n")
|
||||
set(REGISTER_FUNCTIONS "${REGISTER_FUNCTIONS} register${FUNCTION}(factory);\n")
|
||||
endforeach()
|
||||
|
||||
configure_file (registerFunctions_area.cpp.in ${FUNCTIONS_GENERATED_DIR}registerFunctions${FUNCTION_AREA}.cpp)
|
||||
endfunction()
|
12
cmake/find_base64.cmake
Normal file
12
cmake/find_base64.cmake
Normal file
@ -0,0 +1,12 @@
|
||||
option (ENABLE_BASE64 "Enable base64" ON)
|
||||
|
||||
if (ENABLE_BASE64)
|
||||
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64")
|
||||
message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
|
||||
else()
|
||||
set (BASE64_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64/include)
|
||||
set (BASE64_LIBRARY base64)
|
||||
set (USE_BASE64 1)
|
||||
endif()
|
||||
endif ()
|
||||
|
@ -71,10 +71,10 @@ if (ENABLE_ODBC)
|
||||
)
|
||||
|
||||
# MinGW find usually fails
|
||||
if(MINGW)
|
||||
if (MINGW)
|
||||
set(ODBC_INCLUDE_DIRECTORIES ".")
|
||||
set(ODBC_LIBRARIES odbc32)
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(ODBC
|
||||
@ -82,6 +82,10 @@ if (ENABLE_ODBC)
|
||||
ODBC_INCLUDE_DIRECTORIES
|
||||
ODBC_LIBRARIES)
|
||||
|
||||
if (USE_STATIC_LIBRARIES)
|
||||
list(APPEND ODBC_LIBRARIES ${LTDL_LIBRARY})
|
||||
endif ()
|
||||
|
||||
mark_as_advanced(ODBC_FOUND ODBC_LIBRARIES ODBC_INCLUDE_DIRECTORIES)
|
||||
endif ()
|
||||
endif ()
|
||||
|
@ -93,8 +93,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
|
||||
endif ()
|
||||
|
||||
if (OPENSSL_FOUND AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL))
|
||||
set (Poco_NetSSL_LIBRARY PocoNetSSL)
|
||||
set (Poco_Crypto_LIBRARY PocoCrypto)
|
||||
set (Poco_NetSSL_LIBRARY PocoNetSSL ${OPENSSL_LIBRARIES})
|
||||
set (Poco_Crypto_LIBRARY PocoCrypto ${OPENSSL_LIBRARIES})
|
||||
endif ()
|
||||
|
||||
if (USE_STATIC_LIBRARIES AND USE_INTERNAL_ZLIB_LIBRARY)
|
||||
|
@ -1,5 +1,4 @@
|
||||
set(DIVIDE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libdivide)
|
||||
set(CITYHASH_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/cityhash102/include)
|
||||
set(COMMON_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/libs/libcommon/include ${ClickHouse_BINARY_DIR}/libs/libcommon/include)
|
||||
set(DBMS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/dbms/src ${ClickHouse_BINARY_DIR}/dbms/src)
|
||||
set(DOUBLE_CONVERSION_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/double-conversion)
|
||||
|
@ -10,6 +10,9 @@ list(APPEND dirs ${dirs1})
|
||||
get_property (dirs1 TARGET common PROPERTY INCLUDE_DIRECTORIES)
|
||||
list(APPEND dirs ${dirs1})
|
||||
|
||||
get_property (dirs1 TARGET cityhash PROPERTY INCLUDE_DIRECTORIES)
|
||||
list(APPEND dirs ${dirs1})
|
||||
|
||||
if (USE_INTERNAL_BOOST_LIBRARY)
|
||||
get_property (dirs1 TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY} PROPERTY INCLUDE_DIRECTORIES)
|
||||
list(APPEND dirs ${dirs1})
|
||||
|
@ -45,6 +45,38 @@ if (HAVE_SSE42)
|
||||
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
|
||||
endif ()
|
||||
|
||||
set (TEST_FLAG "-mssse3")
|
||||
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
|
||||
check_cxx_source_compiles("
|
||||
#include <tmmintrin.h>
|
||||
int main() {
|
||||
__m64 a = _mm_abs_pi8(__m64());
|
||||
(void)a;
|
||||
return 0;
|
||||
}
|
||||
" HAVE_SSSE3)
|
||||
|
||||
set (TEST_FLAG "-mavx")
|
||||
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
|
||||
check_cxx_source_compiles("
|
||||
#include <immintrin.h>
|
||||
int main() {
|
||||
auto a = _mm256_insert_epi8(__m256i(), 0, 0);
|
||||
(void)a;
|
||||
return 0;
|
||||
}
|
||||
" HAVE_AVX)
|
||||
|
||||
set (TEST_FLAG "-mavx2")
|
||||
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
|
||||
check_cxx_source_compiles("
|
||||
#include <immintrin.h>
|
||||
int main() {
|
||||
auto a = _mm256_add_epi16(__m256i(), __m256i());
|
||||
(void)a;
|
||||
return 0;
|
||||
}
|
||||
" HAVE_AVX2)
|
||||
|
||||
# gcc -dM -E -mpopcnt - < /dev/null | sort > gcc-dump-popcnt
|
||||
#define __POPCNT__ 1
|
||||
@ -65,5 +97,3 @@ if (HAVE_POPCNT AND NOT ARCH_AARCH64)
|
||||
endif ()
|
||||
|
||||
cmake_pop_check_state ()
|
||||
|
||||
# TODO: add here sse3 test if you want use it
|
||||
|
3
contrib/CMakeLists.txt
vendored
3
contrib/CMakeLists.txt
vendored
@ -191,3 +191,6 @@ if (USE_INTERNAL_LLVM_LIBRARY)
|
||||
add_subdirectory (llvm/llvm)
|
||||
endif ()
|
||||
|
||||
if (USE_BASE64)
|
||||
add_subdirectory (base64-cmake)
|
||||
endif()
|
||||
|
1
contrib/base64
vendored
Submodule
1
contrib/base64
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit a27c565d1b6c676beaf297fe503c4518185666f7
|
1
contrib/base64-cmake/.gitignore
vendored
Normal file
1
contrib/base64-cmake/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
config.h
|
52
contrib/base64-cmake/CMakeLists.txt
Normal file
52
contrib/base64-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,52 @@
|
||||
SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64)
|
||||
|
||||
set(base64_compile_instructions "")
|
||||
LIST(LENGTH base64_compile_instructions 0)
|
||||
macro(cast_to_bool var instruction)
|
||||
if (HAVE_${var})
|
||||
set(base64_${var} 1)
|
||||
set(base64_${var}_opt ${instruction})
|
||||
else()
|
||||
set(base64_${var} 0)
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
cast_to_bool(SSSE3 "-mssse3")
|
||||
cast_to_bool(SSE41 "-msse4.1")
|
||||
cast_to_bool(SSE42 "-msse4.2")
|
||||
cast_to_bool(AVX "-mavx")
|
||||
cast_to_bool(AVX2 "-mavx2")
|
||||
|
||||
# write config.h file, to include it in application
|
||||
file(READ config-header.tpl header)
|
||||
file(WRITE config.h ${header})
|
||||
file(APPEND config.h "#define HAVE_SSSE3 ${base64_SSSE3}\n")
|
||||
file(APPEND config.h "#define HAVE_SSE41 ${base64_SSE41}\n")
|
||||
file(APPEND config.h "#define HAVE_SSE42 ${base64_SSE42}\n")
|
||||
file(APPEND config.h "#define HAVE_AVX ${base64_AVX}\n")
|
||||
file(APPEND config.h "#define HAVE_AVX2 ${base64_AVX2}\n")
|
||||
|
||||
set(HAVE_FAST_UNALIGNED_ACCESS 0)
|
||||
if (${base64_SSSE3} OR ${base64_SSE41} OR ${base64_SSE42} OR ${base64_AVX} OR ${base64_AVX2})
|
||||
set(HAVE_FAST_UNALIGNED_ACCESS 1)
|
||||
endif ()
|
||||
|
||||
file(APPEND config.h "#define HAVE_FAST_UNALIGNED_ACCESS " ${HAVE_FAST_UNALIGNED_ACCESS} "\n")
|
||||
|
||||
add_library(base64 ${LINK_MODE}
|
||||
${LIBRARY_DIR}/lib/lib.c
|
||||
${LIBRARY_DIR}/lib/codec_choose.c
|
||||
${LIBRARY_DIR}/lib/arch/avx/codec.c
|
||||
${LIBRARY_DIR}/lib/arch/avx2/codec.c
|
||||
${LIBRARY_DIR}/lib/arch/generic/codec.c
|
||||
${LIBRARY_DIR}/lib/arch/neon32/codec.c
|
||||
${LIBRARY_DIR}/lib/arch/neon64/codec.c
|
||||
${LIBRARY_DIR}/lib/arch/sse41/codec.c
|
||||
${LIBRARY_DIR}/lib/arch/sse42/codec.c
|
||||
${LIBRARY_DIR}/lib/arch/ssse3/codec.c
|
||||
|
||||
${LIBRARY_DIR}/lib/codecs.h
|
||||
config.h)
|
||||
|
||||
target_compile_options(base64 PRIVATE ${base64_SSSE3_opt} ${base64_SSE41_opt} ${base64_SSE42_opt} ${base64_AVX_opt} ${base64_AVX2_opt})
|
||||
target_include_directories(base64 PRIVATE ${LIBRARY_DIR}/include .)
|
2
contrib/base64-cmake/config-header.tpl
Normal file
2
contrib/base64-cmake/config-header.tpl
Normal file
@ -0,0 +1,2 @@
|
||||
#define HAVE_NEON32 0
|
||||
#define HAVE_NEON64 0
|
@ -1,9 +1,8 @@
|
||||
add_library(cityhash
|
||||
src/city.cc
|
||||
|
||||
include/citycrc.h
|
||||
include/city.h
|
||||
src/config.h)
|
||||
|
||||
target_include_directories (cityhash BEFORE PUBLIC include)
|
||||
target_include_directories (cityhash PRIVATE src)
|
||||
target_include_directories(cityhash BEFORE PUBLIC include)
|
||||
target_include_directories(cityhash PRIVATE src)
|
||||
|
@ -119,7 +119,7 @@ endif ()
|
||||
|
||||
if (USE_EMBEDDED_COMPILER)
|
||||
llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
|
||||
target_link_libraries (dbms ${REQUIRED_LLVM_LIBRARIES})
|
||||
target_link_libraries (dbms PRIVATE ${REQUIRED_LLVM_LIBRARIES})
|
||||
target_include_directories (dbms SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
|
||||
endif ()
|
||||
|
||||
@ -150,33 +150,48 @@ if (NOT ARCH_ARM AND CPUID_LIBRARY)
|
||||
endif()
|
||||
|
||||
target_link_libraries (clickhouse_common_io
|
||||
PUBLIC
|
||||
common
|
||||
PRIVATE
|
||||
string_utils
|
||||
widechar_width
|
||||
${LINK_LIBRARIES_ONLY_ON_X86_64}
|
||||
${LZ4_LIBRARY}
|
||||
${ZSTD_LIBRARY}
|
||||
${DOUBLE_CONVERSION_LIBRARIES}
|
||||
pocoext
|
||||
PUBLIC
|
||||
${Poco_Net_LIBRARY}
|
||||
${Poco_Util_LIBRARY}
|
||||
${Poco_Foundation_LIBRARY}
|
||||
${RE2_LIBRARY}
|
||||
${RE2_ST_LIBRARY}
|
||||
${CITYHASH_LIBRARIES}
|
||||
PRIVATE
|
||||
${ZLIB_LIBRARIES}
|
||||
${EXECINFO_LIBRARY}
|
||||
${ELF_LIBRARY}
|
||||
PUBLIC
|
||||
${Boost_SYSTEM_LIBRARY}
|
||||
PRIVATE
|
||||
apple_rt
|
||||
${CMAKE_DL_LIBS}
|
||||
)
|
||||
|
||||
target_link_libraries (dbms
|
||||
PRIVATE
|
||||
clickhouse_parsers
|
||||
clickhouse_common_config
|
||||
PUBLIC
|
||||
clickhouse_common_io
|
||||
pocoext
|
||||
PUBLIC
|
||||
${MYSQLXX_LIBRARY}
|
||||
${RE2_LIBRARY}
|
||||
${RE2_ST_LIBRARY}
|
||||
PRIVATE
|
||||
${BTRIE_LIBRARIES}
|
||||
${Boost_PROGRAM_OPTIONS_LIBRARY}
|
||||
PUBLIC
|
||||
${Boost_SYSTEM_LIBRARY}
|
||||
)
|
||||
|
||||
if (NOT USE_INTERNAL_RE2_LIBRARY)
|
||||
@ -193,8 +208,8 @@ if (Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
|
||||
endif()
|
||||
|
||||
if (USE_POCO_SQLODBC)
|
||||
target_link_libraries (clickhouse_common_io ${Poco_SQL_LIBRARY})
|
||||
target_link_libraries (dbms ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
|
||||
target_link_libraries (clickhouse_common_io PRIVATE ${Poco_SQL_LIBRARY})
|
||||
target_link_libraries (dbms PRIVATE ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
|
||||
if (NOT USE_INTERNAL_POCO_LIBRARY)
|
||||
target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIR})
|
||||
target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR} PUBLIC ${Poco_SQL_INCLUDE_DIR})
|
||||
@ -208,48 +223,44 @@ if (Poco_Data_FOUND)
|
||||
endif()
|
||||
|
||||
if (USE_POCO_DATAODBC)
|
||||
target_link_libraries (clickhouse_common_io ${Poco_Data_LIBRARY})
|
||||
target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
|
||||
target_link_libraries (clickhouse_common_io PRIVATE ${Poco_Data_LIBRARY})
|
||||
target_link_libraries (dbms PRIVATE ${Poco_DataODBC_LIBRARY})
|
||||
if (NOT USE_INTERNAL_POCO_LIBRARY)
|
||||
target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (USE_POCO_MONGODB)
|
||||
target_link_libraries (dbms ${Poco_MongoDB_LIBRARY})
|
||||
target_link_libraries (dbms PRIVATE ${Poco_MongoDB_LIBRARY})
|
||||
endif()
|
||||
|
||||
if (USE_POCO_NETSSL)
|
||||
target_link_libraries (clickhouse_common_io ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
|
||||
target_link_libraries (dbms ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
|
||||
target_link_libraries (clickhouse_common_io PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
|
||||
target_link_libraries (dbms PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
|
||||
endif()
|
||||
|
||||
target_link_libraries (dbms ${Poco_Foundation_LIBRARY})
|
||||
target_link_libraries (dbms PRIVATE ${Poco_Foundation_LIBRARY})
|
||||
|
||||
if (USE_ICU)
|
||||
target_link_libraries (dbms ${ICU_LIBS})
|
||||
target_link_libraries (dbms PRIVATE ${ICU_LIBS})
|
||||
target_include_directories (dbms SYSTEM PRIVATE ${ICU_INCLUDE_DIR})
|
||||
endif ()
|
||||
|
||||
if (USE_CAPNP)
|
||||
target_link_libraries (dbms ${CAPNP_LIBRARY})
|
||||
target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARY})
|
||||
if (NOT USE_INTERNAL_CAPNP_LIBRARY)
|
||||
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR})
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
if (USE_RDKAFKA)
|
||||
target_link_libraries (dbms ${RDKAFKA_LIBRARY})
|
||||
target_link_libraries (dbms PRIVATE ${RDKAFKA_LIBRARY})
|
||||
if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
|
||||
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
target_link_libraries(dbms ${OPENSSL_CRYPTO_LIBRARY})
|
||||
|
||||
target_link_libraries (dbms
|
||||
Threads::Threads
|
||||
)
|
||||
target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY} Threads::Threads)
|
||||
|
||||
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
|
||||
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
|
||||
@ -285,6 +296,6 @@ if (ENABLE_TESTS)
|
||||
# attach all dbms gtest sources
|
||||
grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms dbms_gtest_sources)
|
||||
add_executable(unit_tests_dbms ${dbms_gtest_sources})
|
||||
target_link_libraries(unit_tests_dbms gtest_main dbms)
|
||||
target_link_libraries(unit_tests_dbms PRIVATE gtest_main dbms clickhouse_common_zookeeper)
|
||||
add_check(unit_tests_dbms)
|
||||
endif ()
|
||||
|
@ -22,3 +22,5 @@ endif ()
|
||||
set (VERSION_NAME "${PROJECT_NAME}" CACHE STRING "")
|
||||
set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}" CACHE STRING "")
|
||||
set (VERSION_SO "${VERSION_STRING}" CACHE STRING "")
|
||||
|
||||
math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000")
|
||||
|
@ -48,45 +48,45 @@ else ()
|
||||
link_directories (${LLVM_LIBRARY_DIRS})
|
||||
endif ()
|
||||
add_executable (clickhouse main.cpp)
|
||||
target_link_libraries (clickhouse clickhouse_common_io)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils)
|
||||
target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
|
||||
target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
|
||||
|
||||
if (ENABLE_CLICKHOUSE_SERVER)
|
||||
target_link_libraries (clickhouse clickhouse-server-lib)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse-server-lib)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_CLIENT)
|
||||
target_link_libraries (clickhouse clickhouse-client-lib)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse-client-lib)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_LOCAL)
|
||||
target_link_libraries (clickhouse clickhouse-local-lib)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse-local-lib)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_BENCHMARK)
|
||||
target_link_libraries (clickhouse clickhouse-benchmark-lib)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse-benchmark-lib)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_PERFORMANCE)
|
||||
target_link_libraries (clickhouse clickhouse-performance-test-lib)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse-performance-test-lib)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_COPIER)
|
||||
target_link_libraries (clickhouse clickhouse-copier-lib)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse-copier-lib)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
|
||||
target_link_libraries (clickhouse clickhouse-extract-from-config-lib)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse-extract-from-config-lib)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_COMPRESSOR)
|
||||
target_link_libraries (clickhouse clickhouse-compressor-lib)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse-compressor-lib)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_FORMAT)
|
||||
target_link_libraries (clickhouse clickhouse-format-lib)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse-format-lib)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_OBFUSCATOR)
|
||||
target_link_libraries (clickhouse clickhouse-obfuscator-lib)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse-obfuscator-lib)
|
||||
endif ()
|
||||
if (USE_EMBEDDED_COMPILER)
|
||||
target_link_libraries (clickhouse clickhouse-compiler-lib)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse-compiler-lib)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
|
||||
target_link_libraries (clickhouse clickhouse-odbc-bridge-lib)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse-odbc-bridge-lib)
|
||||
endif()
|
||||
|
||||
set (CLICKHOUSE_BUNDLE)
|
||||
|
@ -42,10 +42,8 @@ namespace DB
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int POCO_EXCEPTION;
|
||||
extern const int STD_EXCEPTION;
|
||||
extern const int UNKNOWN_EXCEPTION;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int EMPTY_DATA_PASSED;
|
||||
}
|
||||
|
||||
class Benchmark
|
||||
@ -170,7 +168,7 @@ private:
|
||||
}
|
||||
|
||||
if (queries.empty())
|
||||
throw Exception("Empty list of queries.");
|
||||
throw Exception("Empty list of queries.", ErrorCodes::EMPTY_DATA_PASSED);
|
||||
|
||||
std::cerr << "Loaded " << queries.size() << " queries.\n";
|
||||
}
|
||||
|
@ -1,8 +1,8 @@
|
||||
add_library (clickhouse-benchmark-lib ${LINK_MODE} Benchmark.cpp)
|
||||
target_link_libraries (clickhouse-benchmark-lib clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
target_link_libraries (clickhouse-benchmark-lib PRIVATE clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-benchmark clickhouse-benchmark.cpp)
|
||||
target_link_libraries (clickhouse-benchmark clickhouse-benchmark-lib clickhouse_aggregate_functions)
|
||||
target_link_libraries (clickhouse-benchmark PRIVATE clickhouse-benchmark-lib clickhouse_aggregate_functions)
|
||||
endif ()
|
||||
|
@ -6,9 +6,9 @@ if (CLICKHOUSE_SPLIT_BINARY)
|
||||
if (USE_EMBEDDED_COMPILER)
|
||||
link_directories (${LLVM_LIBRARY_DIRS})
|
||||
add_executable (clickhouse-clang clickhouse-clang.cpp)
|
||||
target_link_libraries (clickhouse-clang clickhouse-compiler-lib)
|
||||
target_link_libraries (clickhouse-clang PRIVATE clickhouse-compiler-lib)
|
||||
add_executable (clickhouse-lld clickhouse-lld.cpp)
|
||||
target_link_libraries (clickhouse-lld clickhouse-compiler-lib)
|
||||
target_link_libraries (clickhouse-lld PRIVATE clickhouse-compiler-lib)
|
||||
install (TARGETS clickhouse-clang clickhouse-lld RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif ()
|
||||
endif ()
|
||||
|
@ -1,13 +1,12 @@
|
||||
add_library (clickhouse-client-lib ${LINK_MODE} Client.cpp)
|
||||
target_link_libraries (clickhouse-client-lib clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
target_link_libraries (clickhouse-client-lib PRIVATE clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
if (READLINE_INCLUDE_DIR)
|
||||
target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
|
||||
endif ()
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-client clickhouse-client.cpp)
|
||||
target_link_libraries (clickhouse-client clickhouse-client-lib)
|
||||
target_link_libraries (clickhouse-client PRIVATE clickhouse-client-lib)
|
||||
endif ()
|
||||
|
||||
install (FILES clickhouse-client.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-client COMPONENT clickhouse-client RENAME config.xml)
|
||||
|
||||
|
@ -18,7 +18,7 @@
|
||||
#include <Poco/File.h>
|
||||
#include <Poco/Util/Application.h>
|
||||
#include <common/readline_use.h>
|
||||
#include <common/find_first_symbols.h>
|
||||
#include <common/find_symbols.h>
|
||||
#include <Common/ClickHouseRevision.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Common/Exception.h>
|
||||
@ -86,9 +86,6 @@ namespace DB
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int POCO_EXCEPTION;
|
||||
extern const int STD_EXCEPTION;
|
||||
extern const int UNKNOWN_EXCEPTION;
|
||||
extern const int NETWORK_ERROR;
|
||||
extern const int NO_DATA_TO_INSERT;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
@ -529,7 +526,7 @@ private:
|
||||
|
||||
if (max_client_network_bandwidth)
|
||||
{
|
||||
ThrottlerPtr throttler = std::make_shared<Throttler>(max_client_network_bandwidth, 0, "");
|
||||
ThrottlerPtr throttler = std::make_shared<Throttler>(max_client_network_bandwidth, 0, "");
|
||||
connection->setThrottler(throttler);
|
||||
}
|
||||
|
||||
|
@ -184,7 +184,7 @@ public:
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false) << "\n";
|
||||
std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n";
|
||||
}
|
||||
|
||||
/// Note that keyword suggestions are available even if we cannot load data from server.
|
||||
|
@ -1,8 +1,8 @@
|
||||
add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp)
|
||||
target_link_libraries (clickhouse-compressor-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
# Also in utils
|
||||
add_executable (clickhouse-compressor clickhouse-compressor.cpp)
|
||||
target_link_libraries (clickhouse-compressor clickhouse-compressor-lib)
|
||||
target_link_libraries (clickhouse-compressor PRIVATE clickhouse-compressor-lib)
|
||||
endif ()
|
||||
|
@ -1,5 +1,5 @@
|
||||
add_library (clickhouse-copier-lib ${LINK_MODE} ClusterCopier.cpp)
|
||||
target_link_libraries (clickhouse-copier-lib clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions)
|
||||
target_link_libraries (clickhouse-copier-lib PRIVATE clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions daemon)
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-copier clickhouse-copier.cpp)
|
||||
|
@ -323,7 +323,7 @@ struct TaskTable
|
||||
struct TaskCluster
|
||||
{
|
||||
TaskCluster(const String & task_zookeeper_path_, const String & default_local_database_)
|
||||
: task_zookeeper_path(task_zookeeper_path_), default_local_database(default_local_database_) {}
|
||||
: task_zookeeper_path(task_zookeeper_path_), default_local_database(default_local_database_) {}
|
||||
|
||||
void loadTasks(const Poco::Util::AbstractConfiguration & config, const String & base_key = "");
|
||||
|
||||
@ -410,8 +410,7 @@ BlockInputStreamPtr squashStreamIntoOneBlock(const BlockInputStreamPtr & stream)
|
||||
return std::make_shared<SquashingBlockInputStream>(
|
||||
stream,
|
||||
std::numeric_limits<size_t>::max(),
|
||||
std::numeric_limits<size_t>::max()
|
||||
);
|
||||
std::numeric_limits<size_t>::max());
|
||||
}
|
||||
|
||||
Block getBlockWithAllStreamData(const BlockInputStreamPtr & stream)
|
||||
|
@ -1,7 +1,7 @@
|
||||
add_library (clickhouse-extract-from-config-lib ${LINK_MODE} ExtractFromConfig.cpp)
|
||||
target_link_libraries (clickhouse-extract-from-config-lib clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
target_link_libraries (clickhouse-extract-from-config-lib PRIVATE clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-extract-from-config clickhouse-extract-from-config.cpp)
|
||||
target_link_libraries (clickhouse-extract-from-config clickhouse-extract-from-config-lib)
|
||||
target_link_libraries (clickhouse-extract-from-config PRIVATE clickhouse-extract-from-config-lib)
|
||||
endif ()
|
||||
|
@ -1,6 +1,6 @@
|
||||
add_library (clickhouse-format-lib ${LINK_MODE} Format.cpp)
|
||||
target_link_libraries (clickhouse-format-lib dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
target_link_libraries (clickhouse-format-lib PRIVATE dbms clickhouse_common_io clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-format clickhouse-format.cpp)
|
||||
target_link_libraries (clickhouse-format clickhouse-format-lib)
|
||||
target_link_libraries (clickhouse-format PRIVATE clickhouse-format-lib)
|
||||
endif ()
|
||||
|
@ -1,7 +1,7 @@
|
||||
add_library (clickhouse-local-lib ${LINK_MODE} LocalServer.cpp)
|
||||
target_link_libraries (clickhouse-local-lib clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
target_link_libraries (clickhouse-local-lib PRIVATE clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-local clickhouse-local.cpp)
|
||||
target_link_libraries (clickhouse-local clickhouse-local-lib)
|
||||
target_link_libraries (clickhouse-local PRIVATE clickhouse-local-lib)
|
||||
endif ()
|
||||
|
@ -1,7 +1,7 @@
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <utility> /// pair
|
||||
#include <utility> /// pair
|
||||
|
||||
#if __has_include("config_tools.h")
|
||||
#include "config_tools.h"
|
||||
|
@ -1,8 +1,8 @@
|
||||
add_library (clickhouse-obfuscator-lib ${LINK_MODE} Obfuscator.cpp)
|
||||
target_link_libraries (clickhouse-obfuscator-lib dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
target_link_libraries (clickhouse-obfuscator-lib PRIVATE dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-obfuscator clickhouse-obfuscator.cpp)
|
||||
set_target_properties(clickhouse-obfuscator PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
|
||||
target_link_libraries (clickhouse-obfuscator clickhouse-obfuscator-lib)
|
||||
target_link_libraries (clickhouse-obfuscator PRIVATE clickhouse-obfuscator-lib)
|
||||
endif ()
|
||||
|
@ -9,23 +9,23 @@ add_library (clickhouse-odbc-bridge-lib ${LINK_MODE}
|
||||
validateODBCConnectionString.cpp
|
||||
)
|
||||
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib clickhouse_common_io daemon dbms)
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE clickhouse_common_io daemon dbms)
|
||||
target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
|
||||
|
||||
if (USE_POCO_SQLODBC)
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQLODBC_LIBRARY})
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQLODBC_LIBRARY})
|
||||
target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR})
|
||||
endif ()
|
||||
if (Poco_SQL_FOUND)
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQL_LIBRARY})
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQL_LIBRARY})
|
||||
endif ()
|
||||
|
||||
if (USE_POCO_DATAODBC)
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_DataODBC_LIBRARY})
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_DataODBC_LIBRARY})
|
||||
target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
|
||||
endif()
|
||||
if (Poco_Data_FOUND)
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_Data_LIBRARY})
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_Data_LIBRARY})
|
||||
endif ()
|
||||
|
||||
|
||||
@ -35,5 +35,5 @@ endif ()
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-odbc-bridge odbc-bridge.cpp)
|
||||
target_link_libraries (clickhouse-odbc-bridge clickhouse-odbc-bridge-lib)
|
||||
target_link_libraries (clickhouse-odbc-bridge PRIVATE clickhouse-odbc-bridge-lib)
|
||||
endif ()
|
||||
|
@ -124,9 +124,9 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques
|
||||
auto identifier_quote = getIdentifierQuote(hdbc);
|
||||
if (identifier_quote.length() == 0)
|
||||
settings.identifier_quoting_style = IdentifierQuotingStyle::None;
|
||||
else if(identifier_quote[0] == '`')
|
||||
else if (identifier_quote[0] == '`')
|
||||
settings.identifier_quoting_style = IdentifierQuotingStyle::Backticks;
|
||||
else if(identifier_quote[0] == '"')
|
||||
else if (identifier_quote[0] == '"')
|
||||
settings.identifier_quoting_style = IdentifierQuotingStyle::DoubleQuotes;
|
||||
else
|
||||
throw Exception("Can not map quote identifier '" + identifier_quote + "' to IdentifierQuotingStyle value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
@ -25,7 +25,7 @@ Poco::Net::HTTPRequestHandler * HandlerFactory::createRequestHandler(const Poco:
|
||||
#else
|
||||
return nullptr;
|
||||
#endif
|
||||
else if(uri.getPath() == "/identifier_quote")
|
||||
else if (uri.getPath() == "/identifier_quote")
|
||||
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
|
||||
return new IdentifierQuoteHandler(keep_alive_timeout, context);
|
||||
#else
|
||||
|
@ -1,2 +1,2 @@
|
||||
add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp)
|
||||
target_link_libraries (validate-odbc-connection-string clickhouse-odbc-bridge-lib)
|
||||
target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse-odbc-bridge-lib clickhouse_common_io)
|
||||
|
@ -2,7 +2,7 @@
|
||||
#include <cstring>
|
||||
#include <algorithm>
|
||||
#include <Poco/String.h>
|
||||
#include <common/find_first_symbols.h>
|
||||
#include <common/find_symbols.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include "validateODBCConnectionString.h"
|
||||
|
@ -1,8 +1,8 @@
|
||||
add_library (clickhouse-performance-test-lib ${LINK_MODE} PerformanceTest.cpp)
|
||||
target_link_libraries (clickhouse-performance-test-lib clickhouse_common_io dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
target_link_libraries (clickhouse-performance-test-lib PRIVATE dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-performance-test clickhouse-performance-test.cpp)
|
||||
target_link_libraries (clickhouse-performance-test clickhouse-performance-test-lib dbms)
|
||||
target_link_libraries (clickhouse-performance-test PRIVATE clickhouse-performance-test-lib)
|
||||
endif ()
|
||||
|
@ -49,10 +49,10 @@ namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int POCO_EXCEPTION;
|
||||
extern const int STD_EXCEPTION;
|
||||
extern const int UNKNOWN_EXCEPTION;
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int FILE_DOESNT_EXIST;
|
||||
}
|
||||
|
||||
static String pad(size_t padding)
|
||||
@ -156,7 +156,7 @@ struct StopConditionsSet
|
||||
else if (key == "average_speed_not_changing_for_ms")
|
||||
average_speed_not_changing_for_ms.value = stop_conditions_view->getUInt64(key);
|
||||
else
|
||||
throw DB::Exception("Met unkown stop condition: " + key);
|
||||
throw DB::Exception("Met unkown stop condition: " + key, DB::ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
++initialized_count;
|
||||
}
|
||||
@ -521,7 +521,7 @@ public:
|
||||
{
|
||||
if (input_files.size() < 1)
|
||||
{
|
||||
throw DB::Exception("No tests were specified", 0);
|
||||
throw DB::Exception("No tests were specified", DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
std::string name;
|
||||
@ -694,7 +694,7 @@ private:
|
||||
size_t ram_size_needed = config->getUInt64("preconditions.ram_size");
|
||||
size_t actual_ram = getMemoryAmount();
|
||||
if (!actual_ram)
|
||||
throw DB::Exception("ram_size precondition not available on this platform", ErrorCodes::NOT_IMPLEMENTED);
|
||||
throw DB::Exception("ram_size precondition not available on this platform", DB::ErrorCodes::NOT_IMPLEMENTED);
|
||||
|
||||
if (ram_size_needed > actual_ram)
|
||||
{
|
||||
@ -868,12 +868,12 @@ private:
|
||||
|
||||
if (!test_config->has("query") && !test_config->has("query_file"))
|
||||
{
|
||||
throw DB::Exception("Missing query fields in test's config: " + test_name);
|
||||
throw DB::Exception("Missing query fields in test's config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
if (test_config->has("query") && test_config->has("query_file"))
|
||||
{
|
||||
throw DB::Exception("Found both query and query_file fields. Choose only one");
|
||||
throw DB::Exception("Found both query and query_file fields. Choose only one", DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
if (test_config->has("query"))
|
||||
@ -885,7 +885,7 @@ private:
|
||||
{
|
||||
const String filename = test_config->getString("query_file");
|
||||
if (filename.empty())
|
||||
throw DB::Exception("Empty file name");
|
||||
throw DB::Exception("Empty file name", DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
bool tsv = fs::path(filename).extension().string() == ".tsv";
|
||||
|
||||
@ -909,7 +909,7 @@ private:
|
||||
|
||||
if (queries.empty())
|
||||
{
|
||||
throw DB::Exception("Did not find any query to execute: " + test_name);
|
||||
throw DB::Exception("Did not find any query to execute: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
if (test_config->has("substitutions"))
|
||||
@ -929,7 +929,7 @@ private:
|
||||
|
||||
if (!test_config->has("type"))
|
||||
{
|
||||
throw DB::Exception("Missing type property in config: " + test_name);
|
||||
throw DB::Exception("Missing type property in config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
String config_exec_type = test_config->getString("type");
|
||||
@ -938,7 +938,7 @@ private:
|
||||
else if (config_exec_type == "once")
|
||||
exec_type = ExecutionType::Once;
|
||||
else
|
||||
throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name);
|
||||
throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
times_to_run = test_config->getUInt("times_to_run", 1);
|
||||
|
||||
@ -951,7 +951,7 @@ private:
|
||||
}
|
||||
|
||||
if (stop_conditions_template.empty())
|
||||
throw DB::Exception("No termination conditions were found in config");
|
||||
throw DB::Exception("No termination conditions were found in config", DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
for (size_t i = 0; i < times_to_run * queries.size(); ++i)
|
||||
stop_conditions_by_run.push_back(stop_conditions_template);
|
||||
@ -978,7 +978,7 @@ private:
|
||||
else
|
||||
{
|
||||
if (lite_output)
|
||||
throw DB::Exception("Specify main_metric for lite output");
|
||||
throw DB::Exception("Specify main_metric for lite output", DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
if (metrics.size() > 0)
|
||||
@ -1023,22 +1023,14 @@ private:
|
||||
if (exec_type == ExecutionType::Loop)
|
||||
{
|
||||
for (const String & metric : metrics)
|
||||
{
|
||||
if (std::find(non_loop_metrics.begin(), non_loop_metrics.end(), metric) != non_loop_metrics.end())
|
||||
{
|
||||
throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")");
|
||||
}
|
||||
}
|
||||
throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (const String & metric : metrics)
|
||||
{
|
||||
if (std::find(loop_metrics.begin(), loop_metrics.end(), metric) != loop_metrics.end())
|
||||
{
|
||||
throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")");
|
||||
}
|
||||
}
|
||||
throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1439,7 +1431,7 @@ try
|
||||
if (input_files.empty())
|
||||
{
|
||||
std::cerr << std::endl;
|
||||
throw DB::Exception("Did not find any xml files");
|
||||
throw DB::Exception("Did not find any xml files", DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
else
|
||||
std::cerr << " found " << input_files.size() << " files." << std::endl;
|
||||
@ -1454,7 +1446,7 @@ try
|
||||
fs::path file(filename);
|
||||
|
||||
if (!fs::exists(file))
|
||||
throw DB::Exception("File '" + filename + "' does not exist");
|
||||
throw DB::Exception("File '" + filename + "' does not exist", DB::ErrorCodes::FILE_DOESNT_EXIST);
|
||||
|
||||
if (fs::is_directory(file))
|
||||
{
|
||||
@ -1463,7 +1455,7 @@ try
|
||||
else
|
||||
{
|
||||
if (file.extension().string() != ".xml")
|
||||
throw DB::Exception("File '" + filename + "' does not have .xml extension");
|
||||
throw DB::Exception("File '" + filename + "' does not have .xml extension", DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
collected_files.push_back(filename);
|
||||
}
|
||||
}
|
||||
|
@ -10,12 +10,16 @@ add_library (clickhouse-server-lib ${LINK_MODE}
|
||||
TCPHandler.cpp
|
||||
)
|
||||
|
||||
target_link_libraries (clickhouse-server-lib clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions)
|
||||
target_link_libraries (clickhouse-server-lib PRIVATE clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY})
|
||||
if (USE_POCO_NETSSL)
|
||||
target_link_libraries (clickhouse-server-lib PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
|
||||
endif ()
|
||||
|
||||
target_include_directories (clickhouse-server-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-server clickhouse-server.cpp)
|
||||
target_link_libraries (clickhouse-server clickhouse-server-lib)
|
||||
target_link_libraries (clickhouse-server PRIVATE clickhouse-server-lib)
|
||||
install (TARGETS clickhouse-server ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif ()
|
||||
|
||||
|
@ -19,9 +19,6 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ABORTED;
|
||||
extern const int POCO_EXCEPTION;
|
||||
extern const int STD_EXCEPTION;
|
||||
extern const int UNKNOWN_EXCEPTION;
|
||||
extern const int TOO_MANY_SIMULTANEOUS_QUERIES;
|
||||
}
|
||||
|
||||
|
@ -55,6 +55,7 @@
|
||||
namespace CurrentMetrics
|
||||
{
|
||||
extern const Metric Revision;
|
||||
extern const Metric VersionInteger;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
@ -66,6 +67,8 @@ namespace ErrorCodes
|
||||
extern const int SUPPORT_IS_DISABLED;
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
|
||||
extern const int INVALID_CONFIG_PARAMETER;
|
||||
extern const int SYSTEM_ERROR;
|
||||
}
|
||||
|
||||
|
||||
@ -73,7 +76,7 @@ static std::string getCanonicalPath(std::string && path)
|
||||
{
|
||||
Poco::trimInPlace(path);
|
||||
if (path.empty())
|
||||
throw Exception("path configuration parameter is empty");
|
||||
throw Exception("path configuration parameter is empty", ErrorCodes::INVALID_CONFIG_PARAMETER);
|
||||
if (path.back() != '/')
|
||||
path += '/';
|
||||
return std::move(path);
|
||||
@ -108,6 +111,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
registerStorages();
|
||||
|
||||
CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get());
|
||||
CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
|
||||
|
||||
/** Context contains all that query execution is dependent:
|
||||
* settings, available functions, data types, aggregate functions, databases...
|
||||
@ -141,7 +145,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
{
|
||||
LOG_TRACE(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds.");
|
||||
if (0 != mlockall(MCL_CURRENT))
|
||||
LOG_WARNING(log, "Failed mlockall: " + errnoToString());
|
||||
LOG_WARNING(log, "Failed mlockall: " + errnoToString(ErrorCodes::SYSTEM_ERROR));
|
||||
else
|
||||
LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed");
|
||||
}
|
||||
@ -555,10 +559,10 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
socket.setReceiveTimeout(settings.receive_timeout);
|
||||
socket.setSendTimeout(settings.send_timeout);
|
||||
servers.emplace_back(new Poco::Net::TCPServer(
|
||||
new TCPHandlerFactory(*this, /* secure= */ true ),
|
||||
server_pool,
|
||||
socket,
|
||||
new Poco::Net::TCPServerParams));
|
||||
new TCPHandlerFactory(*this, /* secure= */ true),
|
||||
server_pool,
|
||||
socket,
|
||||
new Poco::Net::TCPServerParams));
|
||||
LOG_INFO(log, "Listening tcp_secure: " + address.toString());
|
||||
#else
|
||||
throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
|
||||
@ -633,7 +637,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
message << "Available RAM = " << formatReadableSizeWithBinarySuffix(memory_amount) << ";"
|
||||
<< " physical cores = " << getNumberOfPhysicalCPUCores() << ";"
|
||||
// on ARM processors it can show only enabled at current moment cores
|
||||
<< " threads = " << std::thread::hardware_concurrency() << ".";
|
||||
<< " threads = " << std::thread::hardware_concurrency() << ".";
|
||||
LOG_INFO(log, message.str());
|
||||
}
|
||||
|
||||
|
@ -181,7 +181,7 @@ void TCPHandler::runImpl()
|
||||
/// Reset the input stream, as we received an empty block while receiving external table data.
|
||||
/// So, the stream has been marked as cancelled and we can't read from it anymore.
|
||||
state.block_in.reset();
|
||||
state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker.
|
||||
state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker.
|
||||
});
|
||||
|
||||
/// Processing Query
|
||||
@ -718,7 +718,7 @@ bool TCPHandler::receiveData()
|
||||
{
|
||||
NamesAndTypesList columns = block.getNamesAndTypesList();
|
||||
storage = StorageMemory::create(external_table_name,
|
||||
ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
|
||||
ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, ColumnComments{}});
|
||||
storage->startup();
|
||||
query_context.addExternalTable(external_table_name, storage);
|
||||
}
|
||||
|
@ -89,7 +89,7 @@ public:
|
||||
, connection_context(server.context())
|
||||
, query_context(server.context())
|
||||
{
|
||||
server_display_name = server.config().getString("display_name", getFQDNOrHostName());
|
||||
server_display_name = server.config().getString("display_name", getFQDNOrHostName());
|
||||
}
|
||||
|
||||
void run();
|
||||
|
@ -164,6 +164,20 @@
|
||||
</replica>
|
||||
</shard>
|
||||
</test_shard_localhost>
|
||||
<test_cluster_two_shards_localhost>
|
||||
<shard>
|
||||
<replica>
|
||||
<host>localhost</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
<shard>
|
||||
<replica>
|
||||
<host>localhost</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</test_cluster_two_shards_localhost>
|
||||
<test_shard_localhost_secure>
|
||||
<shard>
|
||||
<replica>
|
||||
|
@ -56,7 +56,7 @@
|
||||
|
||||
Each element of list has one of the following forms:
|
||||
<ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
|
||||
2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
|
||||
2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
|
||||
<host> Hostname. Example: server01.yandex.ru.
|
||||
To check access, DNS query is performed, and all received addresses compared to peer address.
|
||||
<host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$
|
||||
|
@ -1,4 +1,4 @@
|
||||
#!/usr/bin/python3.4
|
||||
#!/usr/bin/python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
|
@ -55,10 +55,10 @@ static AggregateFunctionPtr createAggregateFunctionGroupArray(const std::string
|
||||
else if (parameters.size() == 1)
|
||||
{
|
||||
auto type = parameters[0].getType();
|
||||
if (type != Field::Types::Int64 && type != Field::Types::UInt64)
|
||||
if (type != Field::Types::Int64 && type != Field::Types::UInt64)
|
||||
throw Exception("Parameter for aggregate function " + name + " should be positive number", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
if ((type == Field::Types::Int64 && parameters[0].get<Int64>() < 0) ||
|
||||
if ((type == Field::Types::Int64 && parameters[0].get<Int64>() < 0) ||
|
||||
(type == Field::Types::UInt64 && parameters[0].get<UInt64>() == 0))
|
||||
throw Exception("Parameter for aggregate function " + name + " should be positive number", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
|
@ -274,7 +274,7 @@ public:
|
||||
|
||||
++count;
|
||||
|
||||
left_mean += left_delta / count;
|
||||
left_mean += left_delta / count;
|
||||
right_mean += right_delta / count;
|
||||
co_moment += (left_val - left_mean) * (right_val - old_right_mean);
|
||||
|
||||
|
@ -130,9 +130,6 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory & factory)
|
||||
|
||||
factory.registerFunction("uniqExact",
|
||||
createAggregateFunctionUniq<true, AggregateFunctionUniqExactData, AggregateFunctionUniqExactData<String>>);
|
||||
|
||||
factory.registerFunction("uniqCombined",
|
||||
createAggregateFunctionUniq<false, AggregateFunctionUniqCombinedData, AggregateFunctionUniqCombinedData<UInt64>>);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -22,7 +22,6 @@
|
||||
#include <Common/typeid_cast.h>
|
||||
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
#include <AggregateFunctions/UniqCombinedBiasData.h>
|
||||
#include <AggregateFunctions/UniqVariadicHash.h>
|
||||
|
||||
|
||||
@ -124,46 +123,6 @@ struct AggregateFunctionUniqExactData<String>
|
||||
static String getName() { return "uniqExact"; }
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedData
|
||||
{
|
||||
using Key = UInt32;
|
||||
using Set = CombinedCardinalityEstimator<
|
||||
Key,
|
||||
HashSet<Key, TrivialHash, HashTableGrower<>>,
|
||||
16,
|
||||
14,
|
||||
17,
|
||||
TrivialHash,
|
||||
UInt32,
|
||||
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
|
||||
HyperLogLogMode::FullFeatured>;
|
||||
|
||||
Set set;
|
||||
|
||||
static String getName() { return "uniqCombined"; }
|
||||
};
|
||||
|
||||
template <>
|
||||
struct AggregateFunctionUniqCombinedData<String>
|
||||
{
|
||||
using Key = UInt64;
|
||||
using Set = CombinedCardinalityEstimator<
|
||||
Key,
|
||||
HashSet<Key, TrivialHash, HashTableGrower<>>,
|
||||
16,
|
||||
14,
|
||||
17,
|
||||
TrivialHash,
|
||||
UInt64,
|
||||
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
|
||||
HyperLogLogMode::FullFeatured>;
|
||||
|
||||
Set set;
|
||||
|
||||
static String getName() { return "uniqCombined"; }
|
||||
};
|
||||
|
||||
|
||||
namespace detail
|
||||
{
|
||||
@ -199,39 +158,6 @@ template <> struct AggregateFunctionUniqTraits<Float64>
|
||||
}
|
||||
};
|
||||
|
||||
/** Hash function for uniqCombined.
|
||||
*/
|
||||
template <typename T> struct AggregateFunctionUniqCombinedTraits
|
||||
{
|
||||
static UInt32 hash(T x) { return static_cast<UInt32>(intHash64(x)); }
|
||||
};
|
||||
|
||||
template <> struct AggregateFunctionUniqCombinedTraits<UInt128>
|
||||
{
|
||||
static UInt32 hash(UInt128 x)
|
||||
{
|
||||
return sipHash64(x);
|
||||
}
|
||||
};
|
||||
|
||||
template <> struct AggregateFunctionUniqCombinedTraits<Float32>
|
||||
{
|
||||
static UInt32 hash(Float32 x)
|
||||
{
|
||||
UInt64 res = ext::bit_cast<UInt64>(x);
|
||||
return static_cast<UInt32>(intHash64(res));
|
||||
}
|
||||
};
|
||||
|
||||
template <> struct AggregateFunctionUniqCombinedTraits<Float64>
|
||||
{
|
||||
static UInt32 hash(Float64 x)
|
||||
{
|
||||
UInt64 res = ext::bit_cast<UInt64>(x);
|
||||
return static_cast<UInt32>(intHash64(res));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
/** The structure for the delegation work to add one element to the `uniq` aggregate functions.
|
||||
* Used for partial specialization to add strings.
|
||||
@ -255,19 +181,6 @@ struct OneAdder
|
||||
data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
|
||||
}
|
||||
}
|
||||
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqCombinedData<T>>)
|
||||
{
|
||||
if constexpr (!std::is_same_v<T, String>)
|
||||
{
|
||||
const auto & value = static_cast<const ColumnVector<T> &>(column).getData()[row_num];
|
||||
data.set.insert(AggregateFunctionUniqCombinedTraits<T>::hash(value));
|
||||
}
|
||||
else
|
||||
{
|
||||
StringRef value = column.getDataAt(row_num);
|
||||
data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
|
||||
}
|
||||
}
|
||||
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqExactData<T>>)
|
||||
{
|
||||
if constexpr (!std::is_same_v<T, String>)
|
||||
@ -387,5 +300,4 @@ public:
|
||||
const char * getHeaderFilePath() const override { return __FILE__; }
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
127
dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp
Normal file
127
dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp
Normal file
@ -0,0 +1,127 @@
|
||||
#include <AggregateFunctions/AggregateFunctionUniqCombined.h>
|
||||
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <AggregateFunctions/Helpers.h>
|
||||
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
template <UInt8 K>
|
||||
struct WithK
|
||||
{
|
||||
template <typename T>
|
||||
using AggregateFunction = AggregateFunctionUniqCombined<T, K>;
|
||||
|
||||
template <bool is_exact, bool argument_is_tuple>
|
||||
using AggregateFunctionVariadic = AggregateFunctionUniqCombinedVariadic<is_exact, argument_is_tuple, K>;
|
||||
};
|
||||
|
||||
template <UInt8 K>
|
||||
AggregateFunctionPtr createAggregateFunctionWithK(const DataTypes & argument_types)
|
||||
{
|
||||
/// We use exact hash function if the arguments are not contiguous in memory, because only exact hash function has support for this case.
|
||||
bool use_exact_hash_function = !isAllArgumentsContiguousInMemory(argument_types);
|
||||
|
||||
if (argument_types.size() == 1)
|
||||
{
|
||||
const IDataType & argument_type = *argument_types[0];
|
||||
|
||||
AggregateFunctionPtr res(createWithNumericType<WithK<K>::template AggregateFunction>(*argument_types[0]));
|
||||
|
||||
WhichDataType which(argument_type);
|
||||
if (res)
|
||||
return res;
|
||||
else if (which.isDate())
|
||||
return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeDate::FieldType>>();
|
||||
else if (which.isDateTime())
|
||||
return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeDateTime::FieldType>>();
|
||||
else if (which.isStringOrFixedString())
|
||||
return std::make_shared<typename WithK<K>::template AggregateFunction<String>>();
|
||||
else if (which.isUUID())
|
||||
return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeUUID::FieldType>>();
|
||||
else if (which.isTuple())
|
||||
{
|
||||
if (use_exact_hash_function)
|
||||
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<true, true>>(argument_types);
|
||||
else
|
||||
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<false, true>>(argument_types);
|
||||
}
|
||||
}
|
||||
|
||||
/// "Variadic" method also works as a fallback generic case for a single argument.
|
||||
if (use_exact_hash_function)
|
||||
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<true, false>>(argument_types);
|
||||
else
|
||||
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<false, false>>(argument_types);
|
||||
}
|
||||
|
||||
AggregateFunctionPtr createAggregateFunctionUniqCombined(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params)
|
||||
{
|
||||
/// log2 of the number of cells in HyperLogLog.
|
||||
/// Reasonable default value, selected to be comparable in quality with "uniq" aggregate function.
|
||||
UInt8 precision = 17;
|
||||
|
||||
if (!params.empty())
|
||||
{
|
||||
if (params.size() != 1)
|
||||
throw Exception(
|
||||
"Aggregate function " + name + " requires one parameter or less.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
UInt64 precision_param = applyVisitor(FieldVisitorConvertToNumber<UInt64>(), params[0]);
|
||||
|
||||
// This range is hardcoded below
|
||||
if (precision_param > 20 || precision_param < 12)
|
||||
throw Exception(
|
||||
"Parameter for aggregate function " + name + "is out or range: [12, 20].", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
|
||||
precision = precision_param;
|
||||
}
|
||||
|
||||
if (argument_types.empty())
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
switch (precision)
|
||||
{
|
||||
case 12:
|
||||
return createAggregateFunctionWithK<12>(argument_types);
|
||||
case 13:
|
||||
return createAggregateFunctionWithK<13>(argument_types);
|
||||
case 14:
|
||||
return createAggregateFunctionWithK<14>(argument_types);
|
||||
case 15:
|
||||
return createAggregateFunctionWithK<15>(argument_types);
|
||||
case 16:
|
||||
return createAggregateFunctionWithK<16>(argument_types);
|
||||
case 17:
|
||||
return createAggregateFunctionWithK<17>(argument_types);
|
||||
case 18:
|
||||
return createAggregateFunctionWithK<18>(argument_types);
|
||||
case 19:
|
||||
return createAggregateFunctionWithK<19>(argument_types);
|
||||
case 20:
|
||||
return createAggregateFunctionWithK<20>(argument_types);
|
||||
}
|
||||
|
||||
__builtin_unreachable();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void registerAggregateFunctionUniqCombined(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("uniqCombined", createAggregateFunctionUniqCombined);
|
||||
}
|
||||
|
||||
} // namespace DB
|
229
dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.h
Normal file
229
dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.h
Normal file
@ -0,0 +1,229 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/CombinedCardinalityEstimator.h>
|
||||
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/DataTypeUUID.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
#include <AggregateFunctions/UniqCombinedBiasData.h>
|
||||
#include <AggregateFunctions/UniqVariadicHash.h>
|
||||
|
||||
#include <ext/bit_cast.h>
|
||||
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Common/SipHash.h>
|
||||
|
||||
#include <Common/typeid_cast.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace detail
|
||||
{
|
||||
/** Hash function for uniqCombined.
|
||||
*/
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedTraits
|
||||
{
|
||||
static UInt32 hash(T x)
|
||||
{
|
||||
return static_cast<UInt32>(intHash64(x));
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct AggregateFunctionUniqCombinedTraits<UInt128>
|
||||
{
|
||||
static UInt32 hash(UInt128 x)
|
||||
{
|
||||
return sipHash64(x);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct AggregateFunctionUniqCombinedTraits<Float32>
|
||||
{
|
||||
static UInt32 hash(Float32 x)
|
||||
{
|
||||
UInt64 res = ext::bit_cast<UInt64>(x);
|
||||
return static_cast<UInt32>(intHash64(res));
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct AggregateFunctionUniqCombinedTraits<Float64>
|
||||
{
|
||||
static UInt32 hash(Float64 x)
|
||||
{
|
||||
UInt64 res = ext::bit_cast<UInt64>(x);
|
||||
return static_cast<UInt32>(intHash64(res));
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
|
||||
|
||||
template <typename Key, UInt8 K>
|
||||
struct AggregateFunctionUniqCombinedDataWithKey
|
||||
{
|
||||
// TODO(ilezhankin): pre-generate values for |UniqCombinedBiasData|,
|
||||
// at the moment gen-bias-data.py script doesn't work.
|
||||
|
||||
// We want to migrate from |HashSet| to |HyperLogLogCounter| when the sizes in memory become almost equal.
|
||||
// The size per element in |HashSet| is sizeof(Key)*2 bytes, and the overall size of |HyperLogLogCounter| is 2^K * 6 bits.
|
||||
// For Key=UInt32 we can calculate: 2^X * 4 * 2 ≤ 2^(K-3) * 6 ⇒ X ≤ K-4.
|
||||
using Set = CombinedCardinalityEstimator<Key, HashSet<Key, TrivialHash, HashTableGrower<>>, 16, K - 4, K, TrivialHash, Key>;
|
||||
|
||||
Set set;
|
||||
};
|
||||
|
||||
template <typename Key>
|
||||
struct AggregateFunctionUniqCombinedDataWithKey<Key, 17>
|
||||
{
|
||||
using Set = CombinedCardinalityEstimator<Key,
|
||||
HashSet<Key, TrivialHash, HashTableGrower<>>,
|
||||
16,
|
||||
13,
|
||||
17,
|
||||
TrivialHash,
|
||||
Key,
|
||||
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
|
||||
HyperLogLogMode::FullFeatured>;
|
||||
|
||||
Set set;
|
||||
};
|
||||
|
||||
|
||||
template <typename T, UInt8 K>
|
||||
struct AggregateFunctionUniqCombinedData : public AggregateFunctionUniqCombinedDataWithKey<UInt32, K>
|
||||
{
|
||||
};
|
||||
|
||||
|
||||
template <UInt8 K>
|
||||
struct AggregateFunctionUniqCombinedData<String, K> : public AggregateFunctionUniqCombinedDataWithKey<UInt64, K>
|
||||
{
|
||||
};
|
||||
|
||||
|
||||
template <typename T, UInt8 K>
|
||||
class AggregateFunctionUniqCombined final
|
||||
: public IAggregateFunctionDataHelper<AggregateFunctionUniqCombinedData<T, K>, AggregateFunctionUniqCombined<T, K>>
|
||||
{
|
||||
public:
|
||||
String getName() const override
|
||||
{
|
||||
return "uniqCombined";
|
||||
}
|
||||
|
||||
DataTypePtr getReturnType() const override
|
||||
{
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
}
|
||||
|
||||
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
|
||||
{
|
||||
if constexpr (!std::is_same_v<T, String>)
|
||||
{
|
||||
const auto & value = static_cast<const ColumnVector<T> &>(*columns[0]).getData()[row_num];
|
||||
this->data(place).set.insert(detail::AggregateFunctionUniqCombinedTraits<T>::hash(value));
|
||||
}
|
||||
else
|
||||
{
|
||||
StringRef value = columns[0]->getDataAt(row_num);
|
||||
this->data(place).set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
|
||||
}
|
||||
}
|
||||
|
||||
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
|
||||
{
|
||||
this->data(place).set.merge(this->data(rhs).set);
|
||||
}
|
||||
|
||||
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
|
||||
{
|
||||
this->data(place).set.write(buf);
|
||||
}
|
||||
|
||||
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
|
||||
{
|
||||
this->data(place).set.read(buf);
|
||||
}
|
||||
|
||||
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
|
||||
{
|
||||
static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
|
||||
}
|
||||
|
||||
const char * getHeaderFilePath() const override
|
||||
{
|
||||
return __FILE__;
|
||||
}
|
||||
};
|
||||
|
||||
/** For multiple arguments. To compute, hashes them.
|
||||
* You can pass multiple arguments as is; You can also pass one argument - a tuple.
|
||||
* But (for the possibility of efficient implementation), you can not pass several arguments, among which there are tuples.
|
||||
*/
|
||||
template <bool is_exact, bool argument_is_tuple, UInt8 K>
|
||||
class AggregateFunctionUniqCombinedVariadic final : public IAggregateFunctionDataHelper<AggregateFunctionUniqCombinedData<UInt64, K>,
|
||||
AggregateFunctionUniqCombinedVariadic<is_exact, argument_is_tuple, K>>
|
||||
{
|
||||
private:
|
||||
size_t num_args = 0;
|
||||
|
||||
public:
|
||||
explicit AggregateFunctionUniqCombinedVariadic(const DataTypes & arguments)
|
||||
{
|
||||
if (argument_is_tuple)
|
||||
num_args = typeid_cast<const DataTypeTuple &>(*arguments[0]).getElements().size();
|
||||
else
|
||||
num_args = arguments.size();
|
||||
}
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
return "uniqCombined";
|
||||
}
|
||||
|
||||
DataTypePtr getReturnType() const override
|
||||
{
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
}
|
||||
|
||||
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
|
||||
{
|
||||
this->data(place).set.insert(typename AggregateFunctionUniqCombinedData<UInt64, K>::Set::value_type(
|
||||
UniqVariadicHash<is_exact, argument_is_tuple>::apply(num_args, columns, row_num)));
|
||||
}
|
||||
|
||||
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
|
||||
{
|
||||
this->data(place).set.merge(this->data(rhs).set);
|
||||
}
|
||||
|
||||
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
|
||||
{
|
||||
this->data(place).set.write(buf);
|
||||
}
|
||||
|
||||
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
|
||||
{
|
||||
this->data(place).set.read(buf);
|
||||
}
|
||||
|
||||
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
|
||||
{
|
||||
static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
|
||||
}
|
||||
|
||||
const char * getHeaderFilePath() const override
|
||||
{
|
||||
return __FILE__;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace DB
|
@ -16,10 +16,12 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
struct ComparePairFirst final
|
||||
@ -191,7 +193,7 @@ public:
|
||||
const auto time_arg = arguments.front().get();
|
||||
if (!WhichDataType(time_arg).isDateTime() && !WhichDataType(time_arg).isUInt32())
|
||||
throw Exception{"Illegal type " + time_arg->getName() + " of first argument of aggregate function " + getName()
|
||||
+ ", must be DateTime or UInt32"};
|
||||
+ ", must be DateTime or UInt32", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
|
||||
for (const auto i : ext::range(1, arguments.size()))
|
||||
{
|
||||
|
@ -20,5 +20,5 @@ list(REMOVE_ITEM clickhouse_aggregate_functions_headers
|
||||
)
|
||||
|
||||
add_library(clickhouse_aggregate_functions ${LINK_MODE} ${clickhouse_aggregate_functions_sources})
|
||||
target_link_libraries(clickhouse_aggregate_functions dbms)
|
||||
target_link_libraries(clickhouse_aggregate_functions PRIVATE dbms)
|
||||
target_include_directories (clickhouse_aggregate_functions BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
|
||||
|
@ -21,6 +21,7 @@ void registerAggregateFunctionsStatisticsSimple(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionSum(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionSumMap(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsUniq(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionUniqCombined(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionTopK(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsBitwise(AggregateFunctionFactory &);
|
||||
@ -55,6 +56,7 @@ void registerAggregateFunctions()
|
||||
registerAggregateFunctionSum(factory);
|
||||
registerAggregateFunctionSumMap(factory);
|
||||
registerAggregateFunctionsUniq(factory);
|
||||
registerAggregateFunctionUniqCombined(factory);
|
||||
registerAggregateFunctionUniqUpTo(factory);
|
||||
registerAggregateFunctionTopK(factory);
|
||||
registerAggregateFunctionsBitwise(factory);
|
||||
|
@ -4,3 +4,5 @@
|
||||
#add_library(clickhouse_client ${LINK_MODE} ${clickhouse_client_headers} ${clickhouse_client_sources})
|
||||
#target_link_libraries (clickhouse_client clickhouse_common_io ${Poco_Net_LIBRARY})
|
||||
#target_include_directories (clickhouse_client PRIVATE ${DBMS_INCLUDE_DIR})
|
||||
|
||||
add_subdirectory(tests)
|
||||
|
@ -56,7 +56,7 @@ void Connection::connect()
|
||||
disconnect();
|
||||
|
||||
LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << (default_database.empty() ? "(not specified)" : default_database) << ". User: " << user
|
||||
<< (static_cast<bool>(secure) ? ". Secure" : "") << (static_cast<bool>(compression) ? "" : ". Uncompressed") );
|
||||
<< (static_cast<bool>(secure) ? ". Secure" : "") << (static_cast<bool>(compression) ? "" : ". Uncompressed"));
|
||||
|
||||
if (static_cast<bool>(secure))
|
||||
{
|
||||
@ -108,14 +108,14 @@ void Connection::connect()
|
||||
disconnect();
|
||||
|
||||
/// Add server address to exception. Also Exception will remember stack trace. It's a pity that more precise exception type is lost.
|
||||
throw NetException(e.displayText(), "(" + getDescription() + ")", ErrorCodes::NETWORK_ERROR);
|
||||
throw NetException(e.displayText() + " (" + getDescription() + ")", ErrorCodes::NETWORK_ERROR);
|
||||
}
|
||||
catch (Poco::TimeoutException & e)
|
||||
{
|
||||
disconnect();
|
||||
|
||||
/// Add server address to exception. Also Exception will remember stack trace. It's a pity that more precise exception type is lost.
|
||||
throw NetException(e.displayText(), "(" + getDescription() + ")", ErrorCodes::SOCKET_TIMEOUT);
|
||||
throw NetException(e.displayText() + " (" + getDescription() + ")", ErrorCodes::SOCKET_TIMEOUT);
|
||||
}
|
||||
}
|
||||
|
||||
@ -696,7 +696,7 @@ void Connection::setDescription()
|
||||
{
|
||||
auto resolved_address = getResolvedAddress();
|
||||
description = host + ":" + toString(resolved_address.port());
|
||||
auto ip_address = resolved_address.host().toString();
|
||||
auto ip_address = resolved_address.host().toString();
|
||||
|
||||
if (host != ip_address)
|
||||
description += ", " + ip_address;
|
||||
|
@ -203,7 +203,7 @@ ConnectionPoolWithFailover::tryGetEntry(
|
||||
LOG_TRACE(
|
||||
log, "Server " << result.entry->getDescription() << " has unacceptable replica delay "
|
||||
<< "for table " << table_to_check->database << "." << table_to_check->table
|
||||
<< ": " << delay);
|
||||
<< ": " << delay);
|
||||
ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica);
|
||||
}
|
||||
}
|
||||
|
2
dbms/src/Client/tests/CMakeLists.txt
Normal file
2
dbms/src/Client/tests/CMakeLists.txt
Normal file
@ -0,0 +1,2 @@
|
||||
add_executable(test-connect test_connect.cpp)
|
||||
target_link_libraries (test-connect dbms)
|
59
dbms/src/Client/tests/test_connect.cpp
Normal file
59
dbms/src/Client/tests/test_connect.cpp
Normal file
@ -0,0 +1,59 @@
|
||||
#include <sys/types.h>
|
||||
#include <sys/socket.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <iostream>
|
||||
#include <Poco/Net/StreamSocket.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
|
||||
|
||||
/** In a loop it connects to the server and immediately breaks the connection.
|
||||
* Using the SO_LINGER option, we ensure that the connection is terminated by sending a RST packet (not FIN).
|
||||
* This behavior causes a bug in the TCPServer implementation in the Poco library.
|
||||
*/
|
||||
int main(int argc, char ** argv)
|
||||
try
|
||||
{
|
||||
for (size_t i = 0, num_iters = argc >= 2 ? DB::parse<size_t>(argv[1]) : 1; i < num_iters; ++i)
|
||||
{
|
||||
std::cerr << ".";
|
||||
|
||||
Poco::Net::SocketAddress address("localhost", 9000);
|
||||
|
||||
int fd = socket(PF_INET, SOCK_STREAM, IPPROTO_IP);
|
||||
|
||||
if (fd < 0)
|
||||
DB::throwFromErrno("Cannot create socket", 0);
|
||||
|
||||
linger linger_value;
|
||||
linger_value.l_onoff = 1;
|
||||
linger_value.l_linger = 0;
|
||||
|
||||
if (0 != setsockopt(fd, SOL_SOCKET, SO_LINGER, &linger_value, sizeof(linger_value)))
|
||||
DB::throwFromErrno("Cannot set linger", 0);
|
||||
|
||||
try
|
||||
{
|
||||
int res = connect(fd, address.addr(), address.length());
|
||||
|
||||
if (res != 0 && errno != EINPROGRESS && errno != EWOULDBLOCK)
|
||||
{
|
||||
close(fd);
|
||||
DB::throwFromErrno("Cannot connect", 0);
|
||||
}
|
||||
|
||||
close(fd);
|
||||
}
|
||||
catch (const Poco::Exception & e)
|
||||
{
|
||||
std::cerr << e.displayText() << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
std::cerr << "\n";
|
||||
}
|
||||
catch (const Poco::Exception & e)
|
||||
{
|
||||
std::cerr << e.displayText() << "\n";
|
||||
}
|
@ -437,11 +437,11 @@ ColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hin
|
||||
auto res = ColumnArray::create(data->cloneEmpty());
|
||||
|
||||
const ColumnString & src_string = typeid_cast<const ColumnString &>(*data);
|
||||
const ColumnString::Chars_t & src_chars = src_string.getChars();
|
||||
const ColumnString::Chars & src_chars = src_string.getChars();
|
||||
const Offsets & src_string_offsets = src_string.getOffsets();
|
||||
const Offsets & src_offsets = getOffsets();
|
||||
|
||||
ColumnString::Chars_t & res_chars = typeid_cast<ColumnString &>(res->getData()).getChars();
|
||||
ColumnString::Chars & res_chars = typeid_cast<ColumnString &>(res->getData()).getChars();
|
||||
Offsets & res_string_offsets = typeid_cast<ColumnString &>(res->getData()).getOffsets();
|
||||
Offsets & res_offsets = res->getOffsets();
|
||||
|
||||
@ -697,20 +697,20 @@ ColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
|
||||
if (replicate_offsets.empty())
|
||||
return cloneEmpty();
|
||||
|
||||
if (typeid_cast<const ColumnUInt8 *>(data.get())) return replicateNumber<UInt8>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnUInt16 *>(data.get())) return replicateNumber<UInt16>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnUInt32 *>(data.get())) return replicateNumber<UInt32>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnUInt64 *>(data.get())) return replicateNumber<UInt64>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnInt8 *>(data.get())) return replicateNumber<Int8>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnInt16 *>(data.get())) return replicateNumber<Int16>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnInt32 *>(data.get())) return replicateNumber<Int32>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnInt64 *>(data.get())) return replicateNumber<Int64>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnFloat32 *>(data.get())) return replicateNumber<Float32>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnFloat64 *>(data.get())) return replicateNumber<Float64>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnString *>(data.get())) return replicateString(replicate_offsets);
|
||||
if (typeid_cast<const ColumnConst *>(data.get())) return replicateConst(replicate_offsets);
|
||||
if (typeid_cast<const ColumnNullable *>(data.get())) return replicateNullable(replicate_offsets);
|
||||
if (typeid_cast<const ColumnTuple *>(data.get())) return replicateTuple(replicate_offsets);
|
||||
if (typeid_cast<const ColumnUInt8 *>(data.get())) return replicateNumber<UInt8>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnUInt16 *>(data.get())) return replicateNumber<UInt16>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnUInt32 *>(data.get())) return replicateNumber<UInt32>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnUInt64 *>(data.get())) return replicateNumber<UInt64>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnInt8 *>(data.get())) return replicateNumber<Int8>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnInt16 *>(data.get())) return replicateNumber<Int16>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnInt32 *>(data.get())) return replicateNumber<Int32>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnInt64 *>(data.get())) return replicateNumber<Int64>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnFloat32 *>(data.get())) return replicateNumber<Float32>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnFloat64 *>(data.get())) return replicateNumber<Float64>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnString *>(data.get())) return replicateString(replicate_offsets);
|
||||
if (typeid_cast<const ColumnConst *>(data.get())) return replicateConst(replicate_offsets);
|
||||
if (typeid_cast<const ColumnNullable *>(data.get())) return replicateNullable(replicate_offsets);
|
||||
if (typeid_cast<const ColumnTuple *>(data.get())) return replicateTuple(replicate_offsets);
|
||||
return replicateGeneric(replicate_offsets);
|
||||
}
|
||||
|
||||
@ -781,11 +781,11 @@ ColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const
|
||||
ColumnArray & res_ = static_cast<ColumnArray &>(*res);
|
||||
|
||||
const ColumnString & src_string = typeid_cast<const ColumnString &>(*data);
|
||||
const ColumnString::Chars_t & src_chars = src_string.getChars();
|
||||
const ColumnString::Chars & src_chars = src_string.getChars();
|
||||
const Offsets & src_string_offsets = src_string.getOffsets();
|
||||
const Offsets & src_offsets = getOffsets();
|
||||
|
||||
ColumnString::Chars_t & res_chars = typeid_cast<ColumnString &>(res_.getData()).getChars();
|
||||
ColumnString::Chars & res_chars = typeid_cast<ColumnString &>(res_.getData()).getChars();
|
||||
Offsets & res_string_offsets = typeid_cast<ColumnString &>(res_.getData()).getOffsets();
|
||||
Offsets & res_offsets = res_.getOffsets();
|
||||
|
||||
|
@ -21,7 +21,7 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
int ColumnDecimal<T>::compareAt(size_t n, size_t m, const IColumn & rhs_, int ) const
|
||||
int ColumnDecimal<T>::compareAt(size_t n, size_t m, const IColumn & rhs_, int) const
|
||||
{
|
||||
auto other = static_cast<const Self &>(rhs_);
|
||||
const T & a = data[n];
|
||||
|
@ -248,7 +248,7 @@ ColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) con
|
||||
|
||||
auto res = ColumnFixedString::create(n);
|
||||
|
||||
Chars_t & res_chars = res->chars;
|
||||
Chars & res_chars = res->chars;
|
||||
|
||||
res_chars.resize(n * limit);
|
||||
|
||||
@ -274,7 +274,7 @@ ColumnPtr ColumnFixedString::indexImpl(const PaddedPODArray<Type> & indexes, siz
|
||||
|
||||
auto res = ColumnFixedString::create(n);
|
||||
|
||||
Chars_t & res_chars = res->chars;
|
||||
Chars & res_chars = res->chars;
|
||||
|
||||
res_chars.resize(n * limit);
|
||||
|
||||
@ -296,7 +296,7 @@ ColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const
|
||||
if (0 == col_size)
|
||||
return res;
|
||||
|
||||
Chars_t & res_chars = res->chars;
|
||||
Chars & res_chars = res->chars;
|
||||
res_chars.resize(n * offsets.back());
|
||||
|
||||
Offset curr_offset = 0;
|
||||
|
@ -17,14 +17,14 @@ class ColumnFixedString final : public COWPtrHelper<IColumn, ColumnFixedString>
|
||||
public:
|
||||
friend class COWPtrHelper<IColumn, ColumnFixedString>;
|
||||
|
||||
using Chars_t = PaddedPODArray<UInt8>;
|
||||
using Chars = PaddedPODArray<UInt8>;
|
||||
|
||||
private:
|
||||
/// Bytes of rows, laid in succession. The strings are stored without a trailing zero byte.
|
||||
/** NOTE It is required that the offset and type of chars in the object be the same as that of `data in ColumnUInt8`.
|
||||
* Used in `packFixed` function (AggregationCommon.h)
|
||||
*/
|
||||
Chars_t chars;
|
||||
Chars chars;
|
||||
/// The size of the rows.
|
||||
const size_t n;
|
||||
|
||||
@ -138,8 +138,8 @@ public:
|
||||
|
||||
/// Specialized part of interface, not from IColumn.
|
||||
|
||||
Chars_t & getChars() { return chars; }
|
||||
const Chars_t & getChars() const { return chars; }
|
||||
Chars & getChars() { return chars; }
|
||||
const Chars & getChars() const { return chars; }
|
||||
|
||||
size_t getN() const { return n; }
|
||||
};
|
||||
|
@ -183,7 +183,7 @@ void ColumnFunction::appendArgument(const ColumnWithTypeAndName & column)
|
||||
auto index = captured_columns.size();
|
||||
if (!column.type->equals(*argumnet_types[index]))
|
||||
throw Exception("Cannot capture column " + std::to_string(argumnet_types.size()) +
|
||||
"because it has incompatible type: got " + column.type->getName() +
|
||||
" because it has incompatible type: got " + column.type->getName() +
|
||||
", but " + argumnet_types[index]->getName() + " is expected.", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
captured_columns.push_back(column);
|
||||
|
@ -604,7 +604,7 @@ void ColumnLowCardinality::Index::check(size_t /*max_dictionary_size*/)
|
||||
void ColumnLowCardinality::Index::checkSizeOfType()
|
||||
{
|
||||
if (size_of_type != getSizeOfIndexType(*positions, size_of_type))
|
||||
throw Exception("Invalid size of type. Expected " + toString(8 * size_of_type) +
|
||||
throw Exception("Invalid size of type. Expected " + toString(8 * size_of_type) +
|
||||
", but positions are " + positions->getName(), ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
|
@ -101,7 +101,7 @@ ColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) co
|
||||
|
||||
auto res = ColumnString::create();
|
||||
|
||||
Chars_t & res_chars = res->chars;
|
||||
Chars & res_chars = res->chars;
|
||||
Offsets & res_offsets = res->offsets;
|
||||
|
||||
filterArraysImpl<UInt8>(chars, offsets, res_chars, res_offsets, filt, result_size_hint);
|
||||
@ -126,7 +126,7 @@ ColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const
|
||||
|
||||
auto res = ColumnString::create();
|
||||
|
||||
Chars_t & res_chars = res->chars;
|
||||
Chars & res_chars = res->chars;
|
||||
Offsets & res_offsets = res->offsets;
|
||||
|
||||
if (limit == size)
|
||||
@ -202,7 +202,7 @@ ColumnPtr ColumnString::indexImpl(const PaddedPODArray<Type> & indexes, size_t l
|
||||
|
||||
auto res = ColumnString::create();
|
||||
|
||||
Chars_t & res_chars = res->chars;
|
||||
Chars & res_chars = res->chars;
|
||||
Offsets & res_offsets = res->offsets;
|
||||
|
||||
size_t new_chars_size = 0;
|
||||
@ -287,7 +287,7 @@ ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const
|
||||
if (0 == col_size)
|
||||
return res;
|
||||
|
||||
Chars_t & res_chars = res->chars;
|
||||
Chars & res_chars = res->chars;
|
||||
Offsets & res_offsets = res->offsets;
|
||||
res_chars.reserve(chars.size() / col_size * replicate_offsets.back());
|
||||
res_offsets.reserve(replicate_offsets.back());
|
||||
|
@ -19,7 +19,7 @@ namespace DB
|
||||
class ColumnString final : public COWPtrHelper<IColumn, ColumnString>
|
||||
{
|
||||
public:
|
||||
using Chars_t = PaddedPODArray<UInt8>;
|
||||
using Chars = PaddedPODArray<UInt8>;
|
||||
|
||||
private:
|
||||
friend class COWPtrHelper<IColumn, ColumnString>;
|
||||
@ -29,7 +29,7 @@ private:
|
||||
|
||||
/// Bytes of strings, placed contiguously.
|
||||
/// For convenience, every string ends with terminating zero byte. Note that strings could contain zero bytes in the middle.
|
||||
Chars_t chars;
|
||||
Chars chars;
|
||||
|
||||
size_t ALWAYS_INLINE offsetAt(size_t i) const { return i == 0 ? 0 : offsets[i - 1]; }
|
||||
|
||||
@ -245,8 +245,8 @@ public:
|
||||
bool canBeInsideNullable() const override { return true; }
|
||||
|
||||
|
||||
Chars_t & getChars() { return chars; }
|
||||
const Chars_t & getChars() const { return chars; }
|
||||
Chars & getChars() { return chars; }
|
||||
const Chars & getChars() const { return chars; }
|
||||
|
||||
Offsets & getOffsets() { return offsets; }
|
||||
const Offsets & getOffsets() const { return offsets; }
|
||||
|
@ -81,12 +81,14 @@ public:
|
||||
{
|
||||
return column_holder->allocatedBytes()
|
||||
+ index.allocatedBytes()
|
||||
+ (cached_null_mask ? cached_null_mask->allocatedBytes() : 0);
|
||||
+ (nested_null_mask ? nested_null_mask->allocatedBytes() : 0);
|
||||
}
|
||||
void forEachSubcolumn(IColumn::ColumnCallback callback) override
|
||||
{
|
||||
callback(column_holder);
|
||||
index.setColumn(getRawColumnPtr());
|
||||
if (is_nullable)
|
||||
nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask);
|
||||
}
|
||||
|
||||
const UInt64 * tryGetSavedHash() const override { return index.tryGetSavedHash(); }
|
||||
@ -100,8 +102,8 @@ private:
|
||||
ReverseIndex<UInt64, ColumnType> index;
|
||||
|
||||
/// For DataTypeNullable, stores null map.
|
||||
mutable ColumnPtr cached_null_mask;
|
||||
mutable ColumnPtr cached_column_nullable;
|
||||
ColumnPtr nested_null_mask;
|
||||
ColumnPtr nested_column_nullable;
|
||||
|
||||
class IncrementalHash
|
||||
{
|
||||
@ -118,6 +120,9 @@ private:
|
||||
|
||||
mutable IncrementalHash hash;
|
||||
|
||||
void createNullMask();
|
||||
void updateNullMask();
|
||||
|
||||
static size_t numSpecialValues(bool is_nullable) { return is_nullable ? 2 : 1; }
|
||||
size_t numSpecialValues() const { return numSpecialValues(is_nullable); }
|
||||
|
||||
@ -148,6 +153,7 @@ ColumnUnique<ColumnType>::ColumnUnique(const ColumnUnique & other)
|
||||
, index(numSpecialValues(is_nullable), 0)
|
||||
{
|
||||
index.setColumn(getRawColumnPtr());
|
||||
createNullMask();
|
||||
}
|
||||
|
||||
template <typename ColumnType>
|
||||
@ -158,6 +164,7 @@ ColumnUnique<ColumnType>::ColumnUnique(const IDataType & type)
|
||||
const auto & holder_type = is_nullable ? *static_cast<const DataTypeNullable &>(type).getNestedType() : type;
|
||||
column_holder = holder_type.createColumn()->cloneResized(numSpecialValues());
|
||||
index.setColumn(getRawColumnPtr());
|
||||
createNullMask();
|
||||
}
|
||||
|
||||
template <typename ColumnType>
|
||||
@ -172,32 +179,51 @@ ColumnUnique<ColumnType>::ColumnUnique(MutableColumnPtr && holder, bool is_nulla
|
||||
throw Exception("Holder column for ColumnUnique can't be nullable.", ErrorCodes::ILLEGAL_COLUMN);
|
||||
|
||||
index.setColumn(getRawColumnPtr());
|
||||
createNullMask();
|
||||
}
|
||||
|
||||
template <typename ColumnType>
|
||||
void ColumnUnique<ColumnType>::createNullMask()
|
||||
{
|
||||
if (is_nullable)
|
||||
{
|
||||
size_t size = getRawColumnPtr()->size();
|
||||
if (!nested_null_mask)
|
||||
{
|
||||
ColumnUInt8::MutablePtr null_mask = ColumnUInt8::create(size, UInt8(0));
|
||||
null_mask->getData()[getNullValueIndex()] = 1;
|
||||
nested_null_mask = std::move(null_mask);
|
||||
nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask);
|
||||
}
|
||||
else
|
||||
throw Exception("Null mask for ColumnUnique is already created.", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename ColumnType>
|
||||
void ColumnUnique<ColumnType>::updateNullMask()
|
||||
{
|
||||
if (is_nullable)
|
||||
{
|
||||
if (!nested_null_mask)
|
||||
throw Exception("Null mask for ColumnUnique is was not created.", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
size_t size = getRawColumnPtr()->size();
|
||||
|
||||
if (nested_null_mask->size() != size)
|
||||
{
|
||||
IColumn & null_mask = nested_null_mask->assumeMutableRef();
|
||||
static_cast<ColumnUInt8 &>(null_mask).getData().resize_fill(size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename ColumnType>
|
||||
const ColumnPtr & ColumnUnique<ColumnType>::getNestedColumn() const
|
||||
{
|
||||
if (is_nullable)
|
||||
{
|
||||
size_t size = getRawColumnPtr()->size();
|
||||
if (!cached_null_mask)
|
||||
{
|
||||
ColumnUInt8::MutablePtr null_mask = ColumnUInt8::create(size, UInt8(0));
|
||||
null_mask->getData()[getNullValueIndex()] = 1;
|
||||
cached_null_mask = std::move(null_mask);
|
||||
cached_column_nullable = ColumnNullable::create(column_holder, cached_null_mask);
|
||||
}
|
||||
return nested_column_nullable;
|
||||
|
||||
if (cached_null_mask->size() != size)
|
||||
{
|
||||
MutableColumnPtr null_mask = (*std::move(cached_null_mask)).mutate();
|
||||
static_cast<ColumnUInt8 &>(*null_mask).getData().resize_fill(size);
|
||||
cached_null_mask = std::move(null_mask);
|
||||
cached_column_nullable = ColumnNullable::create(column_holder, cached_null_mask);
|
||||
}
|
||||
|
||||
return cached_column_nullable;
|
||||
}
|
||||
return column_holder;
|
||||
}
|
||||
|
||||
@ -227,6 +253,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsert(const Field & x)
|
||||
if (pos != prev_size)
|
||||
column->popBack(1);
|
||||
|
||||
updateNullMask();
|
||||
|
||||
return pos;
|
||||
}
|
||||
|
||||
@ -260,6 +288,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsertData(const char * pos, size_t lengt
|
||||
index.insertFromLastRow();
|
||||
}
|
||||
|
||||
updateNullMask();
|
||||
|
||||
return insertion_point;
|
||||
}
|
||||
|
||||
@ -288,6 +318,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsertDataWithTerminatingZero(const char
|
||||
if (position != prev_size)
|
||||
column->popBack(1);
|
||||
|
||||
updateNullMask();
|
||||
|
||||
return static_cast<size_t>(position);
|
||||
}
|
||||
|
||||
@ -305,7 +337,7 @@ StringRef ColumnUnique<ColumnType>::serializeValueIntoArena(size_t n, Arena & ar
|
||||
|
||||
size_t nested_size = 0;
|
||||
|
||||
if (n == getNullValueIndex())
|
||||
if (n != getNullValueIndex())
|
||||
nested_size = column_holder->serializeValueIntoArena(n, arena, begin).size;
|
||||
|
||||
return StringRef(pos, sizeof(null_flag) + nested_size);
|
||||
@ -343,6 +375,8 @@ size_t ColumnUnique<ColumnType>::uniqueDeserializeAndInsertFromArena(const char
|
||||
if (index_pos != prev_size)
|
||||
column->popBack(1);
|
||||
|
||||
updateNullMask();
|
||||
|
||||
return static_cast<size_t>(index_pos);
|
||||
}
|
||||
|
||||
@ -533,6 +567,8 @@ MutableColumnPtr ColumnUnique<ColumnType>::uniqueInsertRangeFrom(const IColumn &
|
||||
if (!positions_column)
|
||||
throw Exception("Can't find index type for ColumnUnique", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
updateNullMask();
|
||||
|
||||
return positions_column;
|
||||
}
|
||||
|
||||
@ -577,6 +613,8 @@ IColumnUnique::IndexesWithOverflow ColumnUnique<ColumnType>::uniqueInsertRangeWi
|
||||
if (!positions_column)
|
||||
throw Exception("Can't find index type for ColumnUnique", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
updateNullMask();
|
||||
|
||||
IColumnUnique::IndexesWithOverflow indexes_with_overflow;
|
||||
indexes_with_overflow.indexes = std::move(positions_column);
|
||||
indexes_with_overflow.overflowed_keys = std::move(overflowed_keys);
|
||||
|
@ -72,7 +72,7 @@ FilterDescription::FilterDescription(const IColumn & column_)
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER);
|
||||
|
||||
const NullMap & null_map = nullable_column->getNullMapData();
|
||||
IColumn::Filter & res = concrete_column->getData();
|
||||
IColumn::Filter & res = concrete_column->getData();
|
||||
|
||||
size_t size = res.size();
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
|
@ -377,7 +377,7 @@ protected:
|
||||
};
|
||||
|
||||
using ColumnPtr = IColumn::Ptr;
|
||||
using MutableColumnPtr = IColumn::MutablePtr;
|
||||
using MutableColumnPtr = IColumn::MutablePtr;
|
||||
using Columns = std::vector<ColumnPtr>;
|
||||
using MutableColumns = std::vector<MutableColumnPtr>;
|
||||
|
||||
|
@ -83,7 +83,7 @@ namespace
|
||||
|
||||
|
||||
template <typename IndexType, typename Hash, typename HashTable, typename ColumnType, bool string_hash, bool has_base_index>
|
||||
struct ReverseIndexHashTableCell
|
||||
struct ReverseIndexHashTableCell
|
||||
: public HashTableCell<IndexType, Hash, ReverseIndexHashTableState<ColumnType, string_hash, has_base_index>>
|
||||
{
|
||||
using Base = HashTableCell<IndexType, Hash, ReverseIndexHashTableState<ColumnType, string_hash, has_base_index>>;
|
||||
@ -272,7 +272,7 @@ public:
|
||||
auto hash = calcHashes();
|
||||
ptr = &hash->getData()[0];
|
||||
UInt64 * expected = nullptr;
|
||||
if(saved_hash_ptr.compare_exchange_strong(expected, ptr))
|
||||
if (saved_hash_ptr.compare_exchange_strong(expected, ptr))
|
||||
saved_hash = std::move(hash);
|
||||
else
|
||||
ptr = expected;
|
||||
|
@ -4,4 +4,5 @@
|
||||
namespace ClickHouseRevision
|
||||
{
|
||||
unsigned get() { return VERSION_REVISION; }
|
||||
unsigned getVersionInteger() { return VERSION_INTEGER; }
|
||||
}
|
||||
|
@ -3,4 +3,5 @@
|
||||
namespace ClickHouseRevision
|
||||
{
|
||||
unsigned get();
|
||||
unsigned getVersionInteger();
|
||||
}
|
||||
|
@ -1,9 +1,8 @@
|
||||
|
||||
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
|
||||
|
||||
add_headers_and_sources(clickhouse_common_config .)
|
||||
|
||||
add_library(clickhouse_common_config ${LINK_MODE} ${clickhouse_common_config_headers} ${clickhouse_common_config_sources})
|
||||
|
||||
target_link_libraries (clickhouse_common_config clickhouse_common_zookeeper string_utils ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY})
|
||||
target_include_directories (clickhouse_common_config PRIVATE ${DBMS_INCLUDE_DIR})
|
||||
target_link_libraries(clickhouse_common_config PUBLIC common PRIVATE clickhouse_common_zookeeper string_utils PUBLIC ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY})
|
||||
target_include_directories(clickhouse_common_config PUBLIC ${DBMS_INCLUDE_DIR})
|
||||
|
@ -260,10 +260,10 @@ void ConfigProcessor::doIncludesRecursive(
|
||||
|
||||
/// Substitute <layer> for the number extracted from the hostname only if there is an
|
||||
/// empty <layer> tag without attributes in the original file.
|
||||
if ( node->nodeName() == "layer" &&
|
||||
!node->hasAttributes() &&
|
||||
!node->hasChildNodes() &&
|
||||
node->nodeValue().empty())
|
||||
if (node->nodeName() == "layer"
|
||||
&& !node->hasAttributes()
|
||||
&& !node->hasChildNodes()
|
||||
&& node->nodeValue().empty())
|
||||
{
|
||||
NodePtr new_node = config->createTextNode(layerFromHost());
|
||||
node->appendChild(new_node);
|
||||
|
@ -18,7 +18,15 @@
|
||||
|
||||
#include <common/Types.h>
|
||||
|
||||
#define SMALL_READ_WRITE_BUFFER_SIZE 16
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_OPEN_FILE;
|
||||
extern const int CANNOT_READ_ALL_DATA;
|
||||
extern const int ATTEMPT_TO_READ_AFTER_EOF;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** Stores a number in the file.
|
||||
@ -26,6 +34,9 @@
|
||||
*/
|
||||
class CounterInFile
|
||||
{
|
||||
private:
|
||||
static inline constexpr size_t SMALL_READ_WRITE_BUFFER_SIZE = 16;
|
||||
|
||||
public:
|
||||
/// path - the name of the file, including the path
|
||||
CounterInFile(const std::string & path_) : path(path_) {}
|
||||
@ -56,13 +67,13 @@ public:
|
||||
|
||||
int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0666);
|
||||
if (-1 == fd)
|
||||
DB::throwFromErrno("Cannot open file " + path);
|
||||
DB::throwFromErrno("Cannot open file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
|
||||
|
||||
try
|
||||
{
|
||||
int flock_ret = flock(fd, LOCK_EX);
|
||||
if (-1 == flock_ret)
|
||||
DB::throwFromErrno("Cannot lock file " + path);
|
||||
DB::throwFromErrno("Cannot lock file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
|
||||
|
||||
if (!file_doesnt_exists)
|
||||
{
|
||||
@ -130,7 +141,7 @@ public:
|
||||
|
||||
int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0666);
|
||||
if (-1 == fd)
|
||||
DB::throwFromErrno("Cannot open file " + path);
|
||||
DB::throwFromErrno("Cannot open file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
|
||||
|
||||
try
|
||||
{
|
||||
@ -178,6 +189,3 @@ private:
|
||||
std::string path;
|
||||
std::mutex mutex;
|
||||
};
|
||||
|
||||
|
||||
#undef SMALL_READ_WRITE_BUFFER_SIZE
|
||||
|
@ -39,7 +39,8 @@
|
||||
M(StorageBufferRows, "Number of rows in buffers of Buffer tables") \
|
||||
M(StorageBufferBytes, "Number of bytes in buffers of Buffer tables") \
|
||||
M(DictCacheRequests, "Number of requests in fly to data sources of dictionaries of cache type.") \
|
||||
M(Revision, "Revision of the server. It is a number incremented for every release or release candidate.") \
|
||||
M(Revision, "Revision of the server. It is a number incremented for every release or release candidate except patch releases.") \
|
||||
M(VersionInteger, "Version of the server in a single integer number in base-1000. For example, version 11.22.33 is translated to 11022033.") \
|
||||
M(RWLockWaitingReaders, "Number of threads waiting for read on a table RWLock.") \
|
||||
M(RWLockWaitingWriters, "Number of threads waiting for write on a table RWLock.") \
|
||||
M(RWLockActiveReaders, "Number of threads holding read lock in a table RWLock.") \
|
||||
|
@ -107,7 +107,7 @@ Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host_an
|
||||
|
||||
Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host, UInt16 port)
|
||||
{
|
||||
return !impl->disable_cache ? Poco::Net::SocketAddress(impl->cache_host(host), port) : Poco::Net::SocketAddress(host, port);
|
||||
return !impl->disable_cache ? Poco::Net::SocketAddress(impl->cache_host(host), port) : Poco::Net::SocketAddress(host, port);
|
||||
}
|
||||
|
||||
void DNSResolver::dropCache()
|
||||
|
@ -397,6 +397,10 @@ namespace ErrorCodes
|
||||
extern const int CANNOT_UPDATE_COLUMN = 420;
|
||||
extern const int CANNOT_ADD_DIFFERENT_AGGREGATE_STATES = 421;
|
||||
extern const int UNSUPPORTED_URI_SCHEME = 422;
|
||||
extern const int CANNOT_GETTIMEOFDAY = 423;
|
||||
extern const int CANNOT_LINK = 424;
|
||||
extern const int SYSTEM_ERROR = 425;
|
||||
extern const int NULL_POINTER_DEREFERENCE = 426;
|
||||
|
||||
extern const int KEEPER_EXCEPTION = 999;
|
||||
extern const int POCO_EXCEPTION = 1000;
|
||||
|
@ -52,7 +52,7 @@ std::string errnoToString(int code, int e)
|
||||
|
||||
void throwFromErrno(const std::string & s, int code, int e)
|
||||
{
|
||||
throw ErrnoException(s + ", " + errnoToString(code, e));
|
||||
throw ErrnoException(s + ", " + errnoToString(code, e), code, e);
|
||||
}
|
||||
|
||||
void tryLogCurrentException(const char * log_name, const std::string & start_of_message)
|
||||
|
@ -14,19 +14,25 @@ namespace Poco { class Logger; }
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int POCO_EXCEPTION;
|
||||
}
|
||||
|
||||
class Exception : public Poco::Exception
|
||||
{
|
||||
public:
|
||||
Exception() {} /// For deferred initialization.
|
||||
Exception(const std::string & msg, int code = 0) : Poco::Exception(msg, code) {}
|
||||
Exception(const std::string & msg, const std::string & arg, int code = 0) : Poco::Exception(msg, arg, code) {}
|
||||
Exception(const std::string & msg, const Exception & exc, int code = 0) : Poco::Exception(msg, exc, code), trace(exc.trace) {}
|
||||
explicit Exception(const Poco::Exception & exc) : Poco::Exception(exc.displayText()) {}
|
||||
Exception(const std::string & msg, int code) : Poco::Exception(msg, code) {}
|
||||
Exception(const std::string & msg, const Exception & nested_exception, int code)
|
||||
: Poco::Exception(msg, nested_exception, code), trace(nested_exception.trace) {}
|
||||
|
||||
const char * name() const throw() override { return "DB::Exception"; }
|
||||
const char * className() const throw() override { return "DB::Exception"; }
|
||||
DB::Exception * clone() const override { return new DB::Exception(*this); }
|
||||
enum CreateFromPocoTag { CreateFromPoco };
|
||||
Exception(CreateFromPocoTag, const Poco::Exception & exc) : Poco::Exception(exc.displayText(), ErrorCodes::POCO_EXCEPTION) {}
|
||||
|
||||
Exception * clone() const override { return new Exception(*this); }
|
||||
void rethrow() const override { throw *this; }
|
||||
const char * name() const throw() override { return "DB::Exception"; }
|
||||
|
||||
/// Add something to the existing message.
|
||||
void addMessage(const std::string & arg) { extendedMessage(arg); }
|
||||
@ -35,6 +41,8 @@ public:
|
||||
|
||||
private:
|
||||
StackTrace trace;
|
||||
|
||||
const char * className() const throw() override { return "DB::Exception"; }
|
||||
};
|
||||
|
||||
|
||||
@ -42,25 +50,27 @@ private:
|
||||
class ErrnoException : public Exception
|
||||
{
|
||||
public:
|
||||
ErrnoException(const std::string & msg, int code = 0, int saved_errno_ = 0)
|
||||
ErrnoException(const std::string & msg, int code, int saved_errno_)
|
||||
: Exception(msg, code), saved_errno(saved_errno_) {}
|
||||
ErrnoException(const std::string & msg, const std::string & arg, int code = 0, int saved_errno_ = 0)
|
||||
: Exception(msg, arg, code), saved_errno(saved_errno_) {}
|
||||
ErrnoException(const std::string & msg, const Exception & exc, int code = 0, int saved_errno_ = 0)
|
||||
: Exception(msg, exc, code), saved_errno(saved_errno_) {}
|
||||
|
||||
ErrnoException * clone() const override { return new ErrnoException(*this); }
|
||||
void rethrow() const override { throw *this; }
|
||||
|
||||
int getErrno() const { return saved_errno; }
|
||||
|
||||
private:
|
||||
int saved_errno;
|
||||
|
||||
const char * name() const throw() override { return "DB::ErrnoException"; }
|
||||
const char * className() const throw() override { return "DB::ErrnoException"; }
|
||||
};
|
||||
|
||||
|
||||
using Exceptions = std::vector<std::exception_ptr>;
|
||||
|
||||
|
||||
std::string errnoToString(int code = 0, int the_errno = errno);
|
||||
[[noreturn]] void throwFromErrno(const std::string & s, int code = 0, int the_errno = errno);
|
||||
std::string errnoToString(int code, int the_errno = errno);
|
||||
[[noreturn]] void throwFromErrno(const std::string & s, int code, int the_errno = errno);
|
||||
|
||||
|
||||
/** Try to write an exception to the log (and forget about it).
|
||||
|
@ -18,7 +18,7 @@ namespace ErrorCodes
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
UInt128 stringToUUID(const String & );
|
||||
UInt128 stringToUUID(const String &);
|
||||
|
||||
|
||||
/** StaticVisitor (and its descendants) - class with overloaded operator() for all types of fields.
|
||||
@ -38,16 +38,16 @@ typename std::decay_t<Visitor>::ResultType applyVisitor(Visitor && visitor, F &&
|
||||
{
|
||||
switch (field.getType())
|
||||
{
|
||||
case Field::Types::Null: return visitor(field.template get<Null>());
|
||||
case Field::Types::UInt64: return visitor(field.template get<UInt64>());
|
||||
case Field::Types::Null: return visitor(field.template get<Null>());
|
||||
case Field::Types::UInt64: return visitor(field.template get<UInt64>());
|
||||
case Field::Types::UInt128: return visitor(field.template get<UInt128>());
|
||||
case Field::Types::Int64: return visitor(field.template get<Int64>());
|
||||
case Field::Types::Int64: return visitor(field.template get<Int64>());
|
||||
case Field::Types::Float64: return visitor(field.template get<Float64>());
|
||||
case Field::Types::String: return visitor(field.template get<String>());
|
||||
case Field::Types::Array: return visitor(field.template get<Array>());
|
||||
case Field::Types::Tuple: return visitor(field.template get<Tuple>());
|
||||
case Field::Types::Decimal32: return visitor(field.template get<DecimalField<Decimal32>>());
|
||||
case Field::Types::Decimal64: return visitor(field.template get<DecimalField<Decimal64>>());
|
||||
case Field::Types::String: return visitor(field.template get<String>());
|
||||
case Field::Types::Array: return visitor(field.template get<Array>());
|
||||
case Field::Types::Tuple: return visitor(field.template get<Tuple>());
|
||||
case Field::Types::Decimal32: return visitor(field.template get<DecimalField<Decimal32>>());
|
||||
case Field::Types::Decimal64: return visitor(field.template get<DecimalField<Decimal64>>());
|
||||
case Field::Types::Decimal128: return visitor(field.template get<DecimalField<Decimal128>>());
|
||||
|
||||
default:
|
||||
|
@ -229,8 +229,8 @@ public:
|
||||
void setHasZero() { has_zero = true; }
|
||||
void clearHasZero() { has_zero = false; }
|
||||
|
||||
Cell * zeroValue() { return reinterpret_cast<Cell*>(&zero_value_storage); }
|
||||
const Cell * zeroValue() const { return reinterpret_cast<const Cell*>(&zero_value_storage); }
|
||||
Cell * zeroValue() { return reinterpret_cast<Cell*>(&zero_value_storage); }
|
||||
const Cell * zeroValue() const { return reinterpret_cast<const Cell*>(&zero_value_storage); }
|
||||
};
|
||||
|
||||
template <typename Cell>
|
||||
@ -240,8 +240,8 @@ struct ZeroValueStorage<false, Cell>
|
||||
void setHasZero() { throw DB::Exception("HashTable: logical error", DB::ErrorCodes::LOGICAL_ERROR); }
|
||||
void clearHasZero() {}
|
||||
|
||||
Cell * zeroValue() { return nullptr; }
|
||||
const Cell * zeroValue() const { return nullptr; }
|
||||
Cell * zeroValue() { return nullptr; }
|
||||
const Cell * zeroValue() const { return nullptr; }
|
||||
};
|
||||
|
||||
|
||||
|
@ -3,6 +3,15 @@
|
||||
#include <Common/HashTable/HashMap.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int INCORRECT_DATA;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** Replacement of the hash table for a small number (<10) of keys.
|
||||
* Implemented as an array with linear search.
|
||||
* The array is located inside the object.
|
||||
@ -13,7 +22,6 @@
|
||||
* you should check if the table is not full,
|
||||
* and do a `fallback` in this case (for example, use a real hash table).
|
||||
*/
|
||||
|
||||
template
|
||||
<
|
||||
typename Key,
|
||||
@ -86,7 +94,7 @@ public:
|
||||
DB::readVarUInt(size, in);
|
||||
|
||||
if (size > capacity)
|
||||
throw DB::Exception("Illegal size");
|
||||
throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
|
||||
|
||||
is_initialized = true;
|
||||
}
|
||||
@ -306,7 +314,7 @@ public:
|
||||
DB::readVarUInt(new_size, rb);
|
||||
|
||||
if (new_size > capacity)
|
||||
throw DB::Exception("Illegal size");
|
||||
throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
|
||||
|
||||
for (size_t i = 0; i < new_size; ++i)
|
||||
buf[i].read(rb);
|
||||
@ -324,7 +332,7 @@ public:
|
||||
DB::readText(new_size, rb);
|
||||
|
||||
if (new_size > capacity)
|
||||
throw DB::Exception("Illegal size");
|
||||
throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
|
||||
|
||||
for (size_t i = 0; i < new_size; ++i)
|
||||
{
|
||||
|
@ -310,9 +310,9 @@ public:
|
||||
{
|
||||
/// Normalizing factor for harmonic mean.
|
||||
static constexpr double alpha_m =
|
||||
bucket_count == 2 ? 0.351 :
|
||||
bucket_count == 4 ? 0.532 :
|
||||
bucket_count == 8 ? 0.626 :
|
||||
bucket_count == 2 ? 0.351 :
|
||||
bucket_count == 4 ? 0.532 :
|
||||
bucket_count == 8 ? 0.626 :
|
||||
bucket_count == 16 ? 0.673 :
|
||||
bucket_count == 32 ? 0.697 :
|
||||
bucket_count == 64 ? 0.709 : 0.7213 / (1 + 1.079 / bucket_count);
|
||||
|
@ -6,20 +6,17 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class NetException : public DB::Exception
|
||||
class NetException : public Exception
|
||||
{
|
||||
public:
|
||||
NetException(const std::string & msg, int code = 0) : DB::Exception(msg, code) {}
|
||||
NetException(const std::string & msg, const std::string & arg, int code = 0) : DB::Exception(msg, arg, code) {}
|
||||
NetException(const std::string & msg, const DB::Exception & exc, int code = 0) : DB::Exception(msg, exc, code) {}
|
||||
NetException(const std::string & msg, int code) : Exception(msg, code) {}
|
||||
|
||||
explicit NetException(const DB::Exception & exc) : DB::Exception(exc) {}
|
||||
explicit NetException(const Poco::Exception & exc) : DB::Exception(exc.displayText()) {}
|
||||
NetException * clone() const override { return new NetException(*this); }
|
||||
void rethrow() const override { throw *this; }
|
||||
|
||||
private:
|
||||
const char * name() const throw() override { return "DB::NetException"; }
|
||||
const char * className() const throw() override { return "DB::NetException"; }
|
||||
DB::NetException * clone() const override { return new DB::NetException(*this); }
|
||||
void rethrow() const override { throw *this; }
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -305,7 +305,7 @@ void PoolWithFailoverBase<TNestedPool>::reportError(const Entry & entry)
|
||||
return;
|
||||
}
|
||||
}
|
||||
throw DB::Exception("Can't find pool to report error.");
|
||||
throw DB::Exception("Can't find pool to report error", DB::ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
template <typename TNestedPool>
|
||||
|
@ -37,7 +37,7 @@ namespace ProfileEvents
|
||||
|
||||
/// Global level static initializer
|
||||
Counters(Counter * allocated_counters)
|
||||
: counters(allocated_counters), parent(nullptr), level(VariableContext::Global) {}
|
||||
: counters(allocated_counters), parent(nullptr), level(VariableContext::Global) {}
|
||||
|
||||
Counter & operator[] (Event event)
|
||||
{
|
||||
|
@ -134,7 +134,7 @@ RWLockFIFO::LockHandler RWLockFIFO::getLock(RWLockFIFO::Type type, RWLockFIFO::C
|
||||
}
|
||||
|
||||
/// Wait a notification
|
||||
it_group->cv.wait(lock, [&] () { return it_group == queue.begin(); } );
|
||||
it_group->cv.wait(lock, [&] () { return it_group == queue.begin(); });
|
||||
|
||||
it_client->start_time = time(nullptr);
|
||||
finalize_metrics();
|
||||
|
@ -4,9 +4,11 @@
|
||||
#include <dlfcn.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/ShellCommand.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <IO/WriteBufferFromVector.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <port/unistd.h>
|
||||
#include <csignal>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -75,13 +77,27 @@ namespace
|
||||
namespace DB
|
||||
{
|
||||
|
||||
ShellCommand::ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd, bool terminate_in_destructor_)
|
||||
: pid(pid)
|
||||
, terminate_in_destructor(terminate_in_destructor_)
|
||||
, log(&Poco::Logger::get("ShellCommand"))
|
||||
, in(in_fd)
|
||||
, out(out_fd)
|
||||
, err(err_fd) {}
|
||||
|
||||
ShellCommand::~ShellCommand()
|
||||
{
|
||||
if (!wait_called)
|
||||
if (terminate_in_destructor)
|
||||
{
|
||||
int retcode = kill(pid, SIGTERM);
|
||||
if (retcode != 0)
|
||||
LOG_WARNING(log, "Cannot kill pid " << pid << " errno '" << errnoToString(retcode) << "'");
|
||||
}
|
||||
else if (!wait_called)
|
||||
tryWait();
|
||||
}
|
||||
|
||||
std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only)
|
||||
std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor)
|
||||
{
|
||||
/** Here it is written that with a normal call `vfork`, there is a chance of deadlock in multithreaded programs,
|
||||
* because of the resolving of characters in the shared library
|
||||
@ -128,7 +144,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, c
|
||||
_exit(int(ReturnCodes::CANNOT_EXEC));
|
||||
}
|
||||
|
||||
std::unique_ptr<ShellCommand> res(new ShellCommand(pid, pipe_stdin.write_fd, pipe_stdout.read_fd, pipe_stderr.read_fd));
|
||||
std::unique_ptr<ShellCommand> res(new ShellCommand(pid, pipe_stdin.write_fd, pipe_stdout.read_fd, pipe_stderr.read_fd, terminate_in_destructor));
|
||||
|
||||
/// Now the ownership of the file descriptors is passed to the result.
|
||||
pipe_stdin.write_fd = -1;
|
||||
@ -139,7 +155,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, c
|
||||
}
|
||||
|
||||
|
||||
std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command, bool pipe_stdin_only)
|
||||
std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command, bool pipe_stdin_only, bool terminate_in_destructor)
|
||||
{
|
||||
/// Arguments in non-constant chunks of memory (as required for `execv`).
|
||||
/// Moreover, their copying must be done before calling `vfork`, so after `vfork` do a minimum of things.
|
||||
@ -149,11 +165,11 @@ std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command,
|
||||
|
||||
char * const argv[] = { argv0.data(), argv1.data(), argv2.data(), nullptr };
|
||||
|
||||
return executeImpl("/bin/sh", argv, pipe_stdin_only);
|
||||
return executeImpl("/bin/sh", argv, pipe_stdin_only, terminate_in_destructor);
|
||||
}
|
||||
|
||||
|
||||
std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & path, const std::vector<std::string> & arguments)
|
||||
std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & path, const std::vector<std::string> & arguments, bool terminate_in_destructor)
|
||||
{
|
||||
size_t argv_sum_size = path.size() + 1;
|
||||
for (const auto & arg : arguments)
|
||||
@ -174,7 +190,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & pa
|
||||
|
||||
argv[arguments.size() + 1] = nullptr;
|
||||
|
||||
return executeImpl(path.data(), argv.data(), false);
|
||||
return executeImpl(path.data(), argv.data(), false, terminate_in_destructor);
|
||||
}
|
||||
|
||||
|
||||
|
@ -28,11 +28,13 @@ class ShellCommand
|
||||
private:
|
||||
pid_t pid;
|
||||
bool wait_called = false;
|
||||
bool terminate_in_destructor;
|
||||
|
||||
ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd)
|
||||
: pid(pid), in(in_fd), out(out_fd), err(err_fd) {}
|
||||
Poco::Logger * log;
|
||||
|
||||
static std::unique_ptr<ShellCommand> executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only);
|
||||
ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd, bool terminate_in_destructor_);
|
||||
|
||||
static std::unique_ptr<ShellCommand> executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor);
|
||||
|
||||
public:
|
||||
WriteBufferFromFile in; /// If the command reads from stdin, do not forget to call in.close() after writing all the data there.
|
||||
@ -41,11 +43,13 @@ public:
|
||||
|
||||
~ShellCommand();
|
||||
|
||||
/// Run the command using /bin/sh -c
|
||||
static std::unique_ptr<ShellCommand> execute(const std::string & command, bool pipe_stdin_only = false);
|
||||
/// Run the command using /bin/sh -c.
|
||||
/// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
|
||||
static std::unique_ptr<ShellCommand> execute(const std::string & command, bool pipe_stdin_only = false, bool terminate_in_destructor = false);
|
||||
|
||||
/// Run the executable with the specified arguments. `arguments` - without argv[0].
|
||||
static std::unique_ptr<ShellCommand> executeDirect(const std::string & path, const std::vector<std::string> & arguments);
|
||||
/// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
|
||||
static std::unique_ptr<ShellCommand> executeDirect(const std::string & path, const std::vector<std::string> & arguments, bool terminate_in_destructor = false);
|
||||
|
||||
/// Wait for the process to end, throw an exception if the code is not 0 or if the process was not completed by itself.
|
||||
void wait();
|
||||
|
@ -20,6 +20,14 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_OPEN_FILE;
|
||||
extern const int CANNOT_CLOSE_FILE;
|
||||
extern const int CANNOT_TRUNCATE_FILE;
|
||||
extern const int CANNOT_SEEK_THROUGH_FILE;
|
||||
}
|
||||
|
||||
|
||||
StatusFile::StatusFile(const std::string & path_)
|
||||
: path(path_)
|
||||
@ -43,7 +51,7 @@ StatusFile::StatusFile(const std::string & path_)
|
||||
fd = ::open(path.c_str(), O_WRONLY | O_CREAT, 0666);
|
||||
|
||||
if (-1 == fd)
|
||||
throwFromErrno("Cannot open file " + path);
|
||||
throwFromErrno("Cannot open file " + path, ErrorCodes::CANNOT_OPEN_FILE);
|
||||
|
||||
try
|
||||
{
|
||||
@ -51,16 +59,16 @@ StatusFile::StatusFile(const std::string & path_)
|
||||
if (-1 == flock_ret)
|
||||
{
|
||||
if (errno == EWOULDBLOCK)
|
||||
throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running.");
|
||||
throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running.", ErrorCodes::CANNOT_OPEN_FILE);
|
||||
else
|
||||
throwFromErrno("Cannot lock file " + path);
|
||||
throwFromErrno("Cannot lock file " + path, ErrorCodes::CANNOT_OPEN_FILE);
|
||||
}
|
||||
|
||||
if (0 != ftruncate(fd, 0))
|
||||
throwFromErrno("Cannot ftruncate " + path);
|
||||
throwFromErrno("Cannot ftruncate " + path, ErrorCodes::CANNOT_TRUNCATE_FILE);
|
||||
|
||||
if (0 != lseek(fd, 0, SEEK_SET))
|
||||
throwFromErrno("Cannot lseek " + path);
|
||||
throwFromErrno("Cannot lseek " + path, ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
|
||||
|
||||
/// Write information about current server instance to the file.
|
||||
{
|
||||
@ -82,10 +90,10 @@ StatusFile::StatusFile(const std::string & path_)
|
||||
StatusFile::~StatusFile()
|
||||
{
|
||||
if (0 != close(fd))
|
||||
LOG_ERROR(&Logger::get("StatusFile"), "Cannot close file " << path << ", " << errnoToString());
|
||||
LOG_ERROR(&Logger::get("StatusFile"), "Cannot close file " << path << ", " << errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
|
||||
|
||||
if (0 != unlink(path.c_str()))
|
||||
LOG_ERROR(&Logger::get("StatusFile"), "Cannot unlink file " << path << ", " << errnoToString());
|
||||
LOG_ERROR(&Logger::get("StatusFile"), "Cannot unlink file " << path << ", " << errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -28,15 +28,15 @@ public:
|
||||
*/
|
||||
Stopwatch(clockid_t clock_type_ = CLOCK_MONOTONIC) : clock_type(clock_type_) { start(); }
|
||||
|
||||
void start() { start_ns = nanoseconds(); is_running = true; }
|
||||
void stop() { stop_ns = nanoseconds(); is_running = false; }
|
||||
void reset() { start_ns = 0; stop_ns = 0; is_running = false; }
|
||||
void restart() { start(); }
|
||||
UInt64 elapsed() const { return elapsedNanoseconds(); }
|
||||
UInt64 elapsedNanoseconds() const { return is_running ? nanoseconds() - start_ns : stop_ns - start_ns; }
|
||||
UInt64 elapsedMicroseconds() const { return elapsedNanoseconds() / 1000U; }
|
||||
UInt64 elapsedMilliseconds() const { return elapsedNanoseconds() / 1000000UL; }
|
||||
double elapsedSeconds() const { return static_cast<double>(elapsedNanoseconds()) / 1000000000ULL; }
|
||||
void start() { start_ns = nanoseconds(); is_running = true; }
|
||||
void stop() { stop_ns = nanoseconds(); is_running = false; }
|
||||
void reset() { start_ns = 0; stop_ns = 0; is_running = false; }
|
||||
void restart() { start(); }
|
||||
UInt64 elapsed() const { return elapsedNanoseconds(); }
|
||||
UInt64 elapsedNanoseconds() const { return is_running ? nanoseconds() - start_ns : stop_ns - start_ns; }
|
||||
UInt64 elapsedMicroseconds() const { return elapsedNanoseconds() / 1000U; }
|
||||
UInt64 elapsedMilliseconds() const { return elapsedNanoseconds() / 1000000UL; }
|
||||
double elapsedSeconds() const { return static_cast<double>(elapsedNanoseconds()) / 1000000000ULL; }
|
||||
|
||||
private:
|
||||
UInt64 start_ns = 0;
|
||||
@ -53,10 +53,10 @@ class AtomicStopwatch
|
||||
public:
|
||||
AtomicStopwatch(clockid_t clock_type_ = CLOCK_MONOTONIC) : clock_type(clock_type_) { restart(); }
|
||||
|
||||
void restart() { start_ns = nanoseconds(); }
|
||||
UInt64 elapsed() const { return nanoseconds() - start_ns; }
|
||||
UInt64 elapsedMilliseconds() const { return elapsed() / 1000000UL; }
|
||||
double elapsedSeconds() const { return static_cast<double>(elapsed()) / 1000000000ULL; }
|
||||
void restart() { start_ns = nanoseconds(); }
|
||||
UInt64 elapsed() const { return nanoseconds() - start_ns; }
|
||||
UInt64 elapsedMilliseconds() const { return elapsed() / 1000000UL; }
|
||||
double elapsedSeconds() const { return static_cast<double>(elapsed()) / 1000000000ULL; }
|
||||
|
||||
/** If specified amount of time has passed, then restarts timer and returns true.
|
||||
* Otherwise returns false.
|
||||
@ -140,10 +140,10 @@ struct StopwatchRUsage
|
||||
{
|
||||
StopwatchRUsage() = default;
|
||||
|
||||
void start() { start_ts = Timestamp::current(); is_running = true; }
|
||||
void stop() { stop_ts = Timestamp::current(); is_running = false; }
|
||||
void reset() { start_ts = Timestamp(); stop_ts = Timestamp(); is_running = false; }
|
||||
void restart() { start(); }
|
||||
void start() { start_ts = Timestamp::current(); is_running = true; }
|
||||
void stop() { stop_ts = Timestamp::current(); is_running = false; }
|
||||
void reset() { start_ts = Timestamp(); stop_ts = Timestamp(); is_running = false; }
|
||||
void restart() { start(); }
|
||||
|
||||
UInt64 elapsed(bool count_user = true, bool count_sys = true) const
|
||||
{
|
||||
|
@ -141,9 +141,9 @@ struct TasksStatsCounters
|
||||
if (curr.stat.version < TASKSTATS_VERSION)
|
||||
return;
|
||||
|
||||
profile_events.increment(ProfileEvents::OSReadChars, safeDiff(prev.stat.read_char, curr.stat.read_char));
|
||||
profile_events.increment(ProfileEvents::OSReadChars, safeDiff(prev.stat.read_char, curr.stat.read_char));
|
||||
profile_events.increment(ProfileEvents::OSWriteChars, safeDiff(prev.stat.write_char, curr.stat.write_char));
|
||||
profile_events.increment(ProfileEvents::OSReadBytes, safeDiff(prev.stat.read_bytes, curr.stat.read_bytes));
|
||||
profile_events.increment(ProfileEvents::OSReadBytes, safeDiff(prev.stat.read_bytes, curr.stat.read_bytes));
|
||||
profile_events.increment(ProfileEvents::OSWriteBytes, safeDiff(prev.stat.write_bytes, curr.stat.write_bytes));
|
||||
}
|
||||
|
||||
|
@ -68,6 +68,7 @@ protected:
|
||||
public:
|
||||
using Configuration = Poco::Util::AbstractConfiguration;
|
||||
|
||||
Context & context;
|
||||
const Configuration & config;
|
||||
|
||||
static constexpr inline auto DEFAULT_HOST = "localhost";
|
||||
@ -78,8 +79,8 @@ public:
|
||||
static constexpr inline auto IDENTIFIER_QUOTE_HANDLER = "/identifier_quote";
|
||||
static constexpr inline auto PING_OK_ANSWER = "Ok.";
|
||||
|
||||
XDBCBridgeHelper(const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_)
|
||||
: http_timeout(http_timeout_), connection_string(connection_string_), config(config_)
|
||||
XDBCBridgeHelper(Context & global_context_, const Poco::Timespan & http_timeout_, const std::string & connection_string_)
|
||||
: http_timeout(http_timeout_), connection_string(connection_string_), context(global_context_), config(context.getConfigRef())
|
||||
{
|
||||
size_t bridge_port = config.getUInt(BridgeHelperMixin::configPrefix() + ".port", DEFAULT_PORT);
|
||||
std::string bridge_host = config.getString(BridgeHelperMixin::configPrefix() + ".host", DEFAULT_HOST);
|
||||
@ -210,7 +211,8 @@ private:
|
||||
/* Contains logic for instantiation of the bridge instance */
|
||||
void startBridge() const
|
||||
{
|
||||
BridgeHelperMixin::startBridge(config, log, http_timeout);
|
||||
auto cmd = BridgeHelperMixin::startBridge(config, log, http_timeout);
|
||||
context.addXDBCBridgeCommand(std::move(cmd));
|
||||
}
|
||||
};
|
||||
|
||||
@ -230,7 +232,7 @@ struct JDBCBridgeMixin
|
||||
return "JDBC";
|
||||
}
|
||||
|
||||
static void startBridge(const Poco::Util::AbstractConfiguration &, const Poco::Logger *, const Poco::Timespan &)
|
||||
static std::unique_ptr<ShellCommand> startBridge(const Poco::Util::AbstractConfiguration &, const Poco::Logger *, const Poco::Timespan &)
|
||||
{
|
||||
throw Exception("jdbc-bridge is not running. Please, start it manually", ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING);
|
||||
}
|
||||
@ -253,11 +255,13 @@ struct ODBCBridgeMixin
|
||||
return "ODBC";
|
||||
}
|
||||
|
||||
static void startBridge(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout)
|
||||
static std::unique_ptr<ShellCommand> startBridge(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout)
|
||||
{
|
||||
/// Path to executable folder
|
||||
Poco::Path path{config.getString("application.dir", "/usr/bin")};
|
||||
|
||||
|
||||
std::vector<std::string> cmd_args;
|
||||
path.setFileName(
|
||||
#if CLICKHOUSE_SPLIT_BINARY
|
||||
"clickhouse-odbc-bridge"
|
||||
@ -268,34 +272,35 @@ struct ODBCBridgeMixin
|
||||
|
||||
std::stringstream command;
|
||||
|
||||
command << path.toString() <<
|
||||
#if CLICKHOUSE_SPLIT_BINARY
|
||||
" "
|
||||
#else
|
||||
" odbc-bridge "
|
||||
#if !CLICKHOUSE_SPLIT_BINARY
|
||||
cmd_args.push_back("odbc-bridge");
|
||||
#endif
|
||||
;
|
||||
|
||||
command << "--http-port " << config.getUInt(configPrefix() + ".port", DEFAULT_PORT) << ' ';
|
||||
command << "--listen-host " << config.getString(configPrefix() + ".listen_host", XDBCBridgeHelper<ODBCBridgeMixin>::DEFAULT_HOST)
|
||||
<< ' ';
|
||||
command << "--http-timeout " << http_timeout.totalMicroseconds() << ' ';
|
||||
cmd_args.push_back("--http-port");
|
||||
cmd_args.push_back(std::to_string(config.getUInt(configPrefix() + ".port", DEFAULT_PORT)));
|
||||
cmd_args.push_back("--listen-host");
|
||||
cmd_args.push_back(config.getString(configPrefix() + ".listen_host", XDBCBridgeHelper<ODBCBridgeMixin>::DEFAULT_HOST));
|
||||
cmd_args.push_back("--http-timeout");
|
||||
cmd_args.push_back(std::to_string(http_timeout.totalMicroseconds()));
|
||||
if (config.has("logger." + configPrefix() + "_log"))
|
||||
command << "--log-path " << config.getString("logger." + configPrefix() + "_log") << ' ';
|
||||
{
|
||||
cmd_args.push_back("--log-path");
|
||||
cmd_args.push_back(config.getString("logger." + configPrefix() + "_log"));
|
||||
}
|
||||
if (config.has("logger." + configPrefix() + "_errlog"))
|
||||
command << "--err-log-path " << config.getString("logger." + configPrefix() + "_errlog") << ' ';
|
||||
{
|
||||
cmd_args.push_back("--err-log-path");
|
||||
cmd_args.push_back(config.getString("logger." + configPrefix() + "_errlog"));
|
||||
}
|
||||
if (config.has("logger." + configPrefix() + "_level"))
|
||||
command << "--log-level " << config.getString("logger." + configPrefix() + "_level") << ' ';
|
||||
command << "&"; /// we don't want to wait this process
|
||||
{
|
||||
cmd_args.push_back("--log-level");
|
||||
cmd_args.push_back(config.getString("logger." + configPrefix() + "_level"));
|
||||
}
|
||||
|
||||
auto command_str = command.str();
|
||||
LOG_TRACE(log, "Starting " + serviceAlias());
|
||||
|
||||
std::cerr << command_str << std::endl;
|
||||
|
||||
LOG_TRACE(log, "Starting " + serviceAlias() + " with command: " << command_str);
|
||||
|
||||
auto cmd = ShellCommand::execute(command_str);
|
||||
cmd->wait();
|
||||
return ShellCommand::executeDirect(path.toString(), cmd_args, true);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user