2023-10-29 01:07:24 +00:00
add_compile_options ( $< $<OR:$<COMPILE_LANGUAGE:C > , $< COMPILE_LANGUAGE:CXX > >: ${ COVERAGE_FLAGS } > )
2020-05-18 01:19:50 +00:00
if ( USE_CLANG_TIDY )
set ( CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}" )
endif ( )
2022-06-30 21:35:44 +00:00
include ( ${ ClickHouse_SOURCE_DIR } /cmake/split_debug_symbols.cmake )
2022-03-10 21:23:28 +00:00
2020-09-19 16:42:36 +00:00
# The `clickhouse` binary is a multi purpose tool that contains multiple execution modes (client, server, etc.),
2024-02-11 19:00:37 +00:00
# So client/server/... is just a symlink to `clickhouse` binary.
#
# But, there are several components that requires extra libraries, like keeper
# requires NuRaft, that regular binary does not requires, so you can disable
# compilation of this components.
#
# If you do not know what modes you need, turn then all.
2022-10-01 14:29:41 +00:00
option ( ENABLE_CLICKHOUSE_ALL "Enable all ClickHouse modes by default" ON )
2020-09-15 21:12:37 +00:00
2021-09-22 00:22:57 +00:00
# https://clickhouse.com/docs/en/operations/utilities/clickhouse-obfuscator/
# https://clickhouse.com/docs/en/operations/utilities/odbc-bridge/
2021-06-20 08:24:43 +00:00
# TODO Also needs NANODBC.
2022-01-09 08:34:05 +00:00
if ( ENABLE_ODBC AND NOT USE_MUSL )
2022-08-03 11:19:13 +00:00
option ( ENABLE_CLICKHOUSE_ODBC_BRIDGE "HTTP-server working like a proxy to ODBC driver" ${ ENABLE_CLICKHOUSE_ALL } )
2021-04-20 11:07:05 +00:00
else ( )
option ( ENABLE_CLICKHOUSE_ODBC_BRIDGE "HTTP-server working like a proxy to ODBC driver" OFF )
endif ( )
2020-08-09 20:10:51 +00:00
2022-01-09 08:34:05 +00:00
if ( NOT USE_MUSL )
feat: implement catboost in library-bridge
This commit moves the catboost model evaluation out of the server
process into the library-bridge binary. This serves two goals: On the
one hand, crashes / memory corruptions of the catboost library no longer
affect the server. On the other hand, we can forbid loading dynamic
libraries in the server (catboost was the last consumer of this
functionality), thus improving security.
SQL syntax:
SELECT
catboostEvaluate('/path/to/model.bin', FEAT_1, ..., FEAT_N) > 0 AS prediction,
ACTION AS target
FROM amazon_train
LIMIT 10
Required configuration:
<catboost_lib_path>/path/to/libcatboostmodel.so</catboost_lib_path>
*** Implementation Details ***
The internal protocol between the server and the library-bridge is
simple:
- HTTP GET on path "/extdict_ping":
A ping, used during the handshake to check if the library-bridge runs.
- HTTP POST on path "extdict_request"
(1) Send a "catboost_GetTreeCount" request from the server to the
bridge, containing a library path (e.g /home/user/libcatboost.so) and
a model path (e.g. /home/user/model.bin). Rirst, this unloads the
catboost library handler associated to the model path (if it was
loaded), then loads the catboost library handler associated to the
model path, then executes GetTreeCount() on the library handler and
finally sends the result back to the server. Step (1) is called once
by the server from FunctionCatBoostEvaluate::getReturnTypeImpl(). The
library path handler is unloaded in the beginning because it contains
state which may no longer be valid if the user runs
catboost("/path/to/model.bin", ...) more than once and if "model.bin"
was updated in between.
(2) Send "catboost_Evaluate" from the server to the bridge, containing
the model path and the features to run the interference on. Step (2)
is called multiple times (once per chunk) by the server from function
FunctionCatBoostEvaluate::executeImpl(). The library handler for the
given model path is expected to be already loaded by Step (1).
Fixes #27870
2022-08-05 07:53:06 +00:00
option ( ENABLE_CLICKHOUSE_LIBRARY_BRIDGE "HTTP-server working like a proxy to external dynamically loaded libraries" ${ ENABLE_CLICKHOUSE_ALL } )
2022-01-09 08:34:05 +00:00
endif ( )
2021-03-05 09:38:00 +00:00
2021-09-22 00:22:57 +00:00
# https://presentations.clickhouse.com/matemarketing_2020/
2021-06-18 11:21:05 +00:00
option ( ENABLE_CLICKHOUSE_GIT_IMPORT "A tool to analyze Git repositories" ${ ENABLE_CLICKHOUSE_ALL } )
2020-12-18 21:19:11 +00:00
2021-05-12 10:39:07 +00:00
option ( ENABLE_CLICKHOUSE_KEEPER "ClickHouse alternative to ZooKeeper" ${ ENABLE_CLICKHOUSE_ALL } )
2021-06-17 16:32:50 +00:00
option ( ENABLE_CLICKHOUSE_KEEPER_CONVERTER "Util allows to convert ZooKeeper logs and snapshots into clickhouse-keeper snapshot" ${ ENABLE_CLICKHOUSE_ALL } )
2023-03-10 02:23:57 +00:00
option ( ENABLE_CLICKHOUSE_KEEPER_CLIENT "ClickHouse Keeper Client" ${ ENABLE_CLICKHOUSE_ALL } )
2022-01-17 18:45:54 +00:00
if ( NOT ENABLE_NURAFT )
2021-05-22 23:26:40 +00:00
# RECONFIGURE_MESSAGE_LEVEL should not be used here,
2022-01-17 18:45:54 +00:00
# since ENABLE_NURAFT is set to OFF for FreeBSD and Darwin.
2021-06-17 16:32:50 +00:00
message ( STATUS "clickhouse-keeper and clickhouse-keeper-converter will not be built (lack of NuRaft)" )
2021-05-22 23:26:40 +00:00
set ( ENABLE_CLICKHOUSE_KEEPER OFF )
2021-06-17 16:32:50 +00:00
set ( ENABLE_CLICKHOUSE_KEEPER_CONVERTER OFF )
2021-05-22 23:26:40 +00:00
endif ( )
2021-05-12 10:39:07 +00:00
2024-02-11 19:00:37 +00:00
message ( STATUS "ClickHouse extra components:" )
2020-10-12 12:38:52 +00:00
2022-06-27 05:36:27 +00:00
if ( ENABLE_CLICKHOUSE_SELF_EXTRACTING )
message ( STATUS "Self-extracting executable: ON" )
else ( )
message ( STATUS "Self-extracting executable: OFF" )
endif ( )
2020-10-12 12:38:52 +00:00
if ( ENABLE_CLICKHOUSE_ODBC_BRIDGE )
message ( STATUS "ODBC bridge mode: ON" )
else ( )
message ( STATUS "ODBC bridge mode: OFF" )
endif ( )
2021-03-05 09:38:00 +00:00
if ( ENABLE_CLICKHOUSE_LIBRARY_BRIDGE )
message ( STATUS "Library bridge mode: ON" )
else ( )
message ( STATUS "Library bridge mode: OFF" )
endif ( )
2021-05-12 10:39:07 +00:00
if ( ENABLE_CLICKHOUSE_KEEPER )
message ( STATUS "ClickHouse keeper mode: ON" )
else ( )
message ( STATUS "ClickHouse keeper mode: OFF" )
endif ( )
2021-06-17 16:32:50 +00:00
if ( ENABLE_CLICKHOUSE_KEEPER_CONVERTER )
message ( STATUS "ClickHouse keeper-converter mode: ON" )
else ( )
message ( STATUS "ClickHouse keeper-converter mode: OFF" )
endif ( )
2023-03-10 02:23:57 +00:00
if ( ENABLE_CLICKHOUSE_KEEPER_CLIENT )
message ( STATUS "ClickHouse keeper-client mode: ON" )
else ( )
message ( STATUS "ClickHouse keeper-client mode: OFF" )
2019-03-04 17:47:31 +00:00
endif ( )
2022-09-28 08:28:47 +00:00
configure_file ( config_tools.h.in ${ CONFIG_INCLUDE_PATH } /config_tools.h )
2018-02-15 20:57:08 +00:00
2019-03-04 17:47:31 +00:00
macro ( clickhouse_target_link_split_lib target name )
2023-01-03 18:51:10 +00:00
target_link_libraries ( ${ target } PRIVATE clickhouse- ${ name } -lib )
2019-03-04 17:47:31 +00:00
endmacro ( )
macro ( clickhouse_program_add_library name )
string ( TOUPPER ${ name } name_uc )
string ( REPLACE "-" "_" name_uc ${ name_uc } )
# Some dark magic
set ( CLICKHOUSE_ ${ name_uc } _SOURCES ${ CLICKHOUSE_${name_uc } _SOURCES} PARENT_SCOPE )
set ( CLICKHOUSE_ ${ name_uc } _LINK ${ CLICKHOUSE_${name_uc } _LINK} PARENT_SCOPE )
set ( CLICKHOUSE_ ${ name_uc } _INCLUDE ${ CLICKHOUSE_${name_uc } _INCLUDE} PARENT_SCOPE )
2023-01-03 18:51:10 +00:00
add_library ( clickhouse- ${ name } -lib ${ CLICKHOUSE_${name_uc } _SOURCES} )
2019-03-04 17:47:31 +00:00
2023-01-03 18:51:10 +00:00
set ( _link ${ CLICKHOUSE_${name_uc } _LINK} ) # can't use ${} in if()
if ( _link )
target_link_libraries ( clickhouse- ${ name } -lib ${ CLICKHOUSE_${name_uc } _LINK} )
2019-03-04 17:47:31 +00:00
endif ( )
2023-01-03 18:51:10 +00:00
set ( _include ${ CLICKHOUSE_${name_uc } _INCLUDE} ) # can't use ${} in if()
if ( _include )
target_include_directories ( clickhouse- ${ name } -lib ${ CLICKHOUSE_${name_uc } _INCLUDE} )
2019-03-04 17:47:31 +00:00
endif ( )
endmacro ( )
macro ( clickhouse_program_add name )
clickhouse_program_add_library ( ${ name } )
endmacro ( )
2018-06-05 20:09:51 +00:00
add_subdirectory ( server )
add_subdirectory ( client )
add_subdirectory ( local )
add_subdirectory ( benchmark )
add_subdirectory ( extract-from-config )
add_subdirectory ( compressor )
add_subdirectory ( copier )
add_subdirectory ( format )
2018-06-16 00:27:59 +00:00
add_subdirectory ( obfuscator )
2020-08-08 18:38:34 +00:00
add_subdirectory ( install )
2020-09-07 03:22:47 +00:00
add_subdirectory ( git-import )
2021-03-18 07:14:13 +00:00
add_subdirectory ( bash-completion )
2021-08-28 20:21:05 +00:00
add_subdirectory ( static-files-disk-uploader )
2022-05-23 00:03:13 +00:00
add_subdirectory ( su )
2022-04-08 07:52:16 +00:00
add_subdirectory ( disks )
2021-05-22 23:26:40 +00:00
if ( ENABLE_CLICKHOUSE_KEEPER )
add_subdirectory ( keeper )
endif ( )
2018-12-14 19:28:37 +00:00
2021-06-17 16:32:50 +00:00
if ( ENABLE_CLICKHOUSE_KEEPER_CONVERTER )
add_subdirectory ( keeper-converter )
endif ( )
2023-03-10 02:23:57 +00:00
if ( ENABLE_CLICKHOUSE_KEEPER_CLIENT )
add_subdirectory ( keeper-client )
endif ( )
2018-12-14 19:28:37 +00:00
if ( ENABLE_CLICKHOUSE_ODBC_BRIDGE )
add_subdirectory ( odbc-bridge )
endif ( )
2017-11-12 17:08:52 +00:00
2021-03-05 09:38:00 +00:00
if ( ENABLE_CLICKHOUSE_LIBRARY_BRIDGE )
add_subdirectory ( library-bridge )
endif ( )
2022-06-27 12:25:21 +00:00
if ( ENABLE_CLICKHOUSE_SELF_EXTRACTING )
2022-06-27 05:36:27 +00:00
add_subdirectory ( self-extracting )
2022-06-27 12:25:21 +00:00
endif ( )
2021-06-17 16:32:50 +00:00
2022-07-23 23:15:54 +00:00
clickhouse_add_executable ( clickhouse main.cpp )
2018-06-19 18:09:09 +00:00
2022-07-23 23:15:54 +00:00
# A library that prevent usage of several functions from libc.
if ( ARCH_AMD64 AND OS_LINUX AND NOT OS_ANDROID )
2022-07-31 08:05:12 +00:00
set ( HARMFUL_LIB harmful )
2022-07-23 23:15:54 +00:00
endif ( )
2019-03-11 18:03:32 +00:00
2022-07-23 23:15:54 +00:00
target_link_libraries ( clickhouse PRIVATE clickhouse_common_io string_utils ${ HARMFUL_LIB } )
target_include_directories ( clickhouse PRIVATE ${ CMAKE_CURRENT_BINARY_DIR } )
2021-01-04 14:08:58 +00:00
2022-07-23 23:15:54 +00:00
if ( ENABLE_CLICKHOUSE_KEEPER )
clickhouse_target_link_split_lib ( clickhouse keeper )
endif ( )
if ( ENABLE_CLICKHOUSE_KEEPER_CONVERTER )
clickhouse_target_link_split_lib ( clickhouse keeper-converter )
endif ( )
2023-03-10 02:23:57 +00:00
if ( ENABLE_CLICKHOUSE_KEEPER_CLIENT )
clickhouse_target_link_split_lib ( clickhouse keeper-client )
endif ( )
2024-02-11 19:00:37 +00:00
clickhouse_target_link_split_lib ( clickhouse install )
2021-01-04 14:08:58 +00:00
2022-07-23 23:15:54 +00:00
set ( CLICKHOUSE_BUNDLE )
2024-02-11 19:00:37 +00:00
macro ( clickhouse_program_install name lib_name )
clickhouse_target_link_split_lib ( clickhouse ${ lib_name } )
add_custom_target ( ${ name } ALL COMMAND ${ CMAKE_COMMAND } -E create_symlink clickhouse ${ name } DEPENDS clickhouse )
install ( FILES "${CMAKE_CURRENT_BINARY_DIR}/${name}" DESTINATION ${ CMAKE_INSTALL_BINDIR } COMPONENT clickhouse )
list ( APPEND CLICKHOUSE_BUNDLE ${ name } )
2024-02-12 11:04:17 +00:00
foreach ( alias ${ ARGN } )
message ( STATUS "Adding alias ${alias} for ${name}" )
add_custom_target ( ${ alias } ALL COMMAND ${ CMAKE_COMMAND } -E create_symlink clickhouse ${ alias } DEPENDS clickhouse )
install ( FILES "${CMAKE_CURRENT_BINARY_DIR}/${alias}" DESTINATION ${ CMAKE_INSTALL_BINDIR } COMPONENT clickhouse )
2024-02-12 13:30:42 +00:00
list ( APPEND CLICKHOUSE_BUNDLE ${ alias } )
2024-02-12 11:04:17 +00:00
endforeach ( )
2024-02-11 19:00:37 +00:00
endmacro ( )
2022-07-28 15:54:22 +00:00
if ( ENABLE_CLICKHOUSE_SELF_EXTRACTING )
list ( APPEND CLICKHOUSE_BUNDLE self-extracting )
endif ( )
2024-01-08 18:24:44 +00:00
2024-02-11 19:00:37 +00:00
clickhouse_program_install ( clickhouse-server server )
2024-02-12 11:04:17 +00:00
clickhouse_program_install ( clickhouse-client client chc )
clickhouse_program_install ( clickhouse-local local chl ch )
2024-02-11 19:00:37 +00:00
clickhouse_program_install ( clickhouse-benchmark benchmark )
clickhouse_program_install ( clickhouse-copier copier )
clickhouse_program_install ( clickhouse-extract-from-config extract-from-config )
clickhouse_program_install ( clickhouse-compressor compressor )
clickhouse_program_install ( clickhouse-format format )
clickhouse_program_install ( clickhouse-obfuscator obfuscator )
clickhouse_program_install ( clickhouse-git-import git-import )
clickhouse_program_install ( clickhouse-static-files-disk-uploader static-files-disk-uploader )
clickhouse_program_install ( clickhouse-disks disks )
clickhouse_program_install ( clickhouse-su su )
2017-12-28 12:58:39 +00:00
2022-07-23 23:15:54 +00:00
if ( ENABLE_CLICKHOUSE_KEEPER )
if ( NOT BUILD_STANDALONE_KEEPER AND CREATE_KEEPER_SYMLINK )
add_custom_target ( clickhouse-keeper ALL COMMAND ${ CMAKE_COMMAND } -E create_symlink clickhouse clickhouse-keeper DEPENDS clickhouse )
install ( FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper" DESTINATION ${ CMAKE_INSTALL_BINDIR } COMPONENT clickhouse )
2021-06-17 16:32:50 +00:00
endif ( )
2017-11-12 12:58:40 +00:00
2022-07-23 23:15:54 +00:00
# otherwise we don't build keeper
if ( BUILD_STANDALONE_KEEPER OR CREATE_KEEPER_SYMLINK )
list ( APPEND CLICKHOUSE_BUNDLE clickhouse-keeper )
2022-03-10 21:23:28 +00:00
endif ( )
2022-07-23 23:15:54 +00:00
endif ( )
if ( ENABLE_CLICKHOUSE_KEEPER_CONVERTER )
add_custom_target ( clickhouse-keeper-converter ALL COMMAND ${ CMAKE_COMMAND } -E create_symlink clickhouse clickhouse-keeper-converter DEPENDS clickhouse )
install ( FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper-converter" DESTINATION ${ CMAKE_INSTALL_BINDIR } COMPONENT clickhouse )
list ( APPEND CLICKHOUSE_BUNDLE clickhouse-keeper-converter )
endif ( )
2023-03-10 02:23:57 +00:00
if ( ENABLE_CLICKHOUSE_KEEPER_CLIENT )
2023-06-14 00:23:39 +00:00
if ( NOT BUILD_STANDALONE_KEEPER )
add_custom_target ( clickhouse-keeper-client ALL COMMAND ${ CMAKE_COMMAND } -E create_symlink clickhouse clickhouse-keeper-client DEPENDS clickhouse )
install ( FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper-client" DESTINATION ${ CMAKE_INSTALL_BINDIR } COMPONENT clickhouse )
# symlink to standalone keeper binary
else ( )
add_custom_target ( clickhouse-keeper-client ALL COMMAND ${ CMAKE_COMMAND } -E create_symlink clickhouse-keeper clickhouse-keeper-client DEPENDS clickhouse-keeper )
install ( FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper-client" DESTINATION "${CMAKE_INSTALL_BINDIR}" COMPONENT clickhouse-keeper )
2021-06-18 11:21:05 +00:00
endif ( )
2022-04-20 10:11:55 +00:00
2023-03-10 02:23:57 +00:00
list ( APPEND CLICKHOUSE_BUNDLE clickhouse-keeper-client )
endif ( )
2018-06-16 00:27:59 +00:00
2022-07-23 23:15:54 +00:00
add_custom_target ( clickhouse-bundle ALL DEPENDS ${ CLICKHOUSE_BUNDLE } )
2018-06-05 20:09:51 +00:00
2022-07-23 23:15:54 +00:00
if ( USE_BINARY_HASH )
add_custom_command ( TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${ OBJCOPY_PATH } --add-section .clickhouse.hash=hash clickhouse COMMENT "Adding section '.clickhouse.hash' to clickhouse binary" VERBATIM )
endif ( )
2021-01-07 02:56:57 +00:00
2023-11-10 05:13:55 +00:00
if ( CHECK_LARGE_OBJECT_SIZES )
add_custom_command ( TARGET clickhouse POST_BUILD
C O M M A N D " $ { C M A K E _ S O U R C E _ D I R } / u t i l s / c h e c k - s t y l e / c h e c k - l a r g e - o b j e c t s . s h " " $ { C M A K E _ B I N A R Y _ D I R } " )
endif ( )
2018-06-06 19:00:11 +00:00
2022-07-23 23:15:54 +00:00
if ( SPLIT_DEBUG_SYMBOLS )
clickhouse_split_debug_symbols ( TARGET clickhouse DESTINATION_DIR ${ CMAKE_CURRENT_BINARY_DIR } / ${ SPLITTED_DEBUG_SYMBOLS_DIR } BINARY_PATH clickhouse )
else ( )
clickhouse_make_empty_debug_info_for_nfpm ( TARGET clickhouse DESTINATION_DIR ${ CMAKE_CURRENT_BINARY_DIR } / ${ SPLITTED_DEBUG_SYMBOLS_DIR } )
install ( TARGETS clickhouse RUNTIME DESTINATION ${ CMAKE_INSTALL_BINDIR } COMPONENT clickhouse )
2022-03-10 21:23:28 +00:00
endif ( )
2023-11-11 06:27:10 +00:00
# A target to get stripped binary.
# Note: this is different to the above (extract debug symbols to a separate place)
2023-11-11 06:28:26 +00:00
add_custom_target ( clickhouse-stripped ALL
2023-11-11 06:27:10 +00:00
C O M M A N D " $ { S T R I P _ P A T H } " - o " $ { C M A K E _ C U R R E N T _ B I N A R Y _ D I R } / c l i c k h o u s e - s t r i p p e d " - - s t r i p - d e b u g - - r e m o v e - s e c t i o n = . c o m m e n t - - r e m o v e - s e c t i o n = . n o t e " $ { C M A K E _ C U R R E N T _ B I N A R Y _ D I R } / c l i c k h o u s e "
D E P E N D S c l i c k h o u s e
C O M M E N T " S t r i p p i n g c l i c k h o u s e b i n a r y " V E R B A T I M )
2022-01-17 20:26:05 +00:00
if ( ENABLE_TESTS )
2021-03-04 03:49:46 +00:00
set ( CLICKHOUSE_UNIT_TESTS_TARGETS unit_tests_dbms )
2020-06-04 08:26:22 +00:00
add_custom_target ( clickhouse-tests ALL DEPENDS ${ CLICKHOUSE_UNIT_TESTS_TARGETS } )
2020-05-21 15:02:52 +00:00
add_dependencies ( clickhouse-bundle clickhouse-tests )
endif ( )
2021-10-07 18:01:36 +00:00
if ( ENABLE_FUZZING )
add_compile_definitions ( FUZZING_MODE=1 )
endif ( )
2023-11-15 13:21:59 +00:00
if ( TARGET ch_contrib::protobuf )
get_property ( google_proto_files TARGET ch_contrib::protobuf PROPERTY google_proto_files )
foreach ( proto_file IN LISTS google_proto_files )
2023-11-27 13:39:01 +00:00
install ( FILES ${ proto_file } DESTINATION ${ CMAKE_INSTALL_DATAROOTDIR } /clickhouse/protos/google/protobuf )
2023-11-15 13:21:59 +00:00
endforeach ( )
endif ( )
