From 4cb53093a0aede40132dd4d3d40e5827a5c9b923 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 27 Sep 2019 20:19:26 +0800 Subject: [PATCH] Get rid of malloc symbols in libcommon (#7065) --- CMakeLists.txt | 27 +++++++++++++++++++++--- dbms/CMakeLists.txt | 16 +++++--------- dbms/programs/CMakeLists.txt | 6 +++--- dbms/src/Client/tests/CMakeLists.txt | 2 +- dbms/src/Processors/tests/CMakeLists.txt | 14 ++++++------ libs/libcommon/CMakeLists.txt | 24 --------------------- libs/libcommon/src/tests/CMakeLists.txt | 18 ++++++++-------- libs/libmysqlxx/src/tests/CMakeLists.txt | 2 +- 8 files changed, 50 insertions(+), 59 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bb7387ddeb0..a5460cd1c76 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,3 @@ -project(ClickHouse) cmake_minimum_required(VERSION 3.3) foreach(policy @@ -13,6 +12,7 @@ foreach(policy endif() endforeach() +project(ClickHouse) include (cmake/target.cmake) # Ignore export() since we don't use it, @@ -348,7 +348,7 @@ include (libs/libcommon/cmake/find_jemalloc.cmake) include (libs/libcommon/cmake/find_cctz.cmake) include (libs/libmysqlxx/cmake/find_mysqlclient.cmake) -# When testing for memory leaks with Valgrind, dont link tcmalloc or jemalloc. +# When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc. if (USE_JEMALLOC) message (STATUS "Link jemalloc: ${JEMALLOC_LIBRARIES}") @@ -367,7 +367,7 @@ elseif (USE_TCMALLOC) endif () elseif (SANITIZE) message (STATUS "Will use ${SANITIZE} sanitizer.") -else () +elseif (OS_LINUX) message (WARNING "Non default allocator is disabled. This is not recommended for production Linux builds.") endif () @@ -376,8 +376,29 @@ include (cmake/print_flags.cmake) install (EXPORT global DESTINATION cmake) add_subdirectory (contrib EXCLUDE_FROM_ALL) + +macro (add_executable target) + # invoke built-in add_executable + _add_executable (${ARGV}) + get_target_property (type ${target} TYPE) + if (${type} STREQUAL EXECUTABLE) + set_property (GLOBAL APPEND PROPERTY CLICKHOUSE_EXECUTABLES ${target}) + endif() +endmacro() + +set_property (GLOBAL PROPERTY CLICKHOUSE_EXECUTABLES "") add_subdirectory (libs) add_subdirectory (utils) +get_property (executables GLOBAL PROPERTY CLICKHOUSE_EXECUTABLES) +foreach (executable ${executables}) + target_link_libraries (${executable} PRIVATE ${MALLOC_LIBRARIES}) +endforeach () + +set_property (GLOBAL PROPERTY CLICKHOUSE_EXECUTABLES "") add_subdirectory (dbms) +get_property (executables GLOBAL PROPERTY CLICKHOUSE_EXECUTABLES) +foreach (executable ${executables}) + target_link_libraries (${executable} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES}) +endforeach () include (cmake/print_include_directories.cmake) diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 22a3111a70e..30e73815bc7 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -100,6 +100,7 @@ set(dbms_sources) add_headers_and_sources(clickhouse_common_io src/Common) add_headers_and_sources(clickhouse_common_io src/Common/HashTable) add_headers_and_sources(clickhouse_common_io src/IO) +list (REMOVE_ITEM clickhouse_common_io_sources src/Common/new_delete.cpp) if(USE_RDKAFKA) add_headers_and_sources(dbms src/Storages/Kafka) @@ -139,6 +140,9 @@ endif () add_library(clickhouse_common_io ${clickhouse_common_io_headers} ${clickhouse_common_io_sources}) +add_library (clickhouse_new_delete STATIC src/Common/new_delete.cpp) +target_link_libraries (clickhouse_new_delete clickhouse_common_io) + if (OS_FREEBSD) target_compile_definitions (clickhouse_common_io PUBLIC CLOCK_MONOTONIC_COARSE=CLOCK_MONOTONIC_FAST) endif () @@ -419,17 +423,7 @@ endif() if (USE_JEMALLOC) dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp - target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # new_delete.cpp - # common/memory.h - if (MAKE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES) - # skip if we have bundled build, since jemalloc is static in this case - elseif (${JEMALLOC_LIBRARIES} MATCHES "${CMAKE_STATIC_LIBRARY_SUFFIX}$") - # if the library is static we do not need to link with it, - # since in this case it will be in libs/libcommon, - # and we do not want to link with jemalloc multiple times. - else() - target_link_libraries(clickhouse_common_io PRIVATE ${JEMALLOC_LIBRARIES}) - endif() + target_include_directories (clickhouse_new_delete SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) endif () dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/Formats/include) diff --git a/dbms/programs/CMakeLists.txt b/dbms/programs/CMakeLists.txt index bac3269468e..138321360f3 100644 --- a/dbms/programs/CMakeLists.txt +++ b/dbms/programs/CMakeLists.txt @@ -24,9 +24,9 @@ configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h) macro(clickhouse_target_link_split_lib target name) if(NOT CLICKHOUSE_ONE_SHARED) - target_link_libraries(${target} PRIVATE clickhouse-${name}-lib ${MALLOC_LIBRARIES}) + target_link_libraries(${target} PRIVATE clickhouse-${name}-lib) else() - target_link_libraries(${target} PRIVATE clickhouse-lib ${MALLOC_LIBRARIES}) + target_link_libraries(${target} PRIVATE clickhouse-lib) endif() endmacro() @@ -111,7 +111,7 @@ if (CLICKHOUSE_SPLIT_BINARY) install(PROGRAMS clickhouse-split-helper DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME clickhouse COMPONENT clickhouse) else () add_executable (clickhouse main.cpp) - target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils ${MALLOC_LIBRARIES}) + target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils) target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR}) target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/dbms/src/Client/tests/CMakeLists.txt b/dbms/src/Client/tests/CMakeLists.txt index f4471136a8a..d952c006bb5 100644 --- a/dbms/src/Client/tests/CMakeLists.txt +++ b/dbms/src/Client/tests/CMakeLists.txt @@ -1,2 +1,2 @@ add_executable(test-connect test_connect.cpp) -target_link_libraries (test-connect dbms) +target_link_libraries (test-connect PRIVATE dbms) diff --git a/dbms/src/Processors/tests/CMakeLists.txt b/dbms/src/Processors/tests/CMakeLists.txt index 5f44ec2a8fd..4ddb6c68416 100644 --- a/dbms/src/Processors/tests/CMakeLists.txt +++ b/dbms/src/Processors/tests/CMakeLists.txt @@ -6,10 +6,10 @@ add_executable (processors_test_merge_sorting_transform processors_test_merge_so add_executable (processors_test_expand_pipeline processors_test_expand_pipeline.cpp) add_executable (processors_test_aggregation processors_test_aggregation.cpp) -target_link_libraries (processors_test dbms) -target_link_libraries (processors_test_chain dbms) -target_link_libraries (processors_test_merge dbms) -target_link_libraries (processors_test_expand_pipeline dbms) -target_link_libraries (processors_test_merging_sorted_transform dbms) -target_link_libraries (processors_test_merge_sorting_transform dbms) -target_link_libraries (processors_test_aggregation dbms clickhouse_aggregate_functions) +target_link_libraries (processors_test PRIVATE dbms) +target_link_libraries (processors_test_chain PRIVATE dbms) +target_link_libraries (processors_test_merge PRIVATE dbms) +target_link_libraries (processors_test_expand_pipeline PRIVATE dbms) +target_link_libraries (processors_test_merging_sorted_transform PRIVATE dbms) +target_link_libraries (processors_test_merge_sorting_transform PRIVATE dbms) +target_link_libraries (processors_test_aggregation PRIVATE dbms clickhouse_aggregate_functions) diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index 62c64a9bdb0..357e457b240 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -65,29 +65,6 @@ add_library (common ${CONFIG_COMMON}) -# When testing for memory leaks with Valgrind, dont link tcmalloc or jemalloc. - -if (USE_JEMALLOC) - message (STATUS "Link jemalloc: ${JEMALLOC_LIBRARIES}") - set (MALLOC_LIBRARIES ${JEMALLOC_LIBRARIES}) -elseif (USE_TCMALLOC) - if (DEBUG_TCMALLOC AND NOT GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG) - message (FATAL_ERROR "Requested DEBUG_TCMALLOC but debug library is not found. You should install Google Perftools. Example: sudo apt-get install libgoogle-perftools-dev") - endif () - - if (DEBUG_TCMALLOC AND GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG) - message (STATUS "Link libtcmalloc_minimal_debug for testing: ${GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG}") - set (MALLOC_LIBRARIES ${GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG}) - else () - message (STATUS "Link libtcmalloc_minimal: ${GPERFTOOLS_TCMALLOC_MINIMAL}") - set (MALLOC_LIBRARIES ${GPERFTOOLS_TCMALLOC_MINIMAL}) - endif () -elseif (SANITIZE) - message (STATUS "Will use ${SANITIZE} sanitizer.") -elseif (OS_LINUX) - message (WARNING "Non default allocator is disabled. This is not recommended for production Linux builds.") -endif () - if (USE_INTERNAL_MEMCPY) set (MEMCPY_LIBRARIES memcpy) endif () @@ -120,7 +97,6 @@ target_link_libraries (common PUBLIC ${Boost_SYSTEM_LIBRARY} PRIVATE - ${MALLOC_LIBRARIES} ${MEMCPY_LIBRARIES}) if (RT_LIBRARY) diff --git a/libs/libcommon/src/tests/CMakeLists.txt b/libs/libcommon/src/tests/CMakeLists.txt index 15d872ac49d..486914e4ca7 100644 --- a/libs/libcommon/src/tests/CMakeLists.txt +++ b/libs/libcommon/src/tests/CMakeLists.txt @@ -10,20 +10,20 @@ add_executable (realloc-perf allocator.cpp) set(PLATFORM_LIBS ${CMAKE_DL_LIBS}) -target_link_libraries (date_lut_init common ${PLATFORM_LIBS}) -target_link_libraries (date_lut2 common ${PLATFORM_LIBS}) -target_link_libraries (date_lut3 common ${PLATFORM_LIBS}) -target_link_libraries (date_lut4 common ${PLATFORM_LIBS}) -target_link_libraries (date_lut_default_timezone common ${PLATFORM_LIBS}) -target_link_libraries (local_date_time_comparison common) -target_link_libraries (realloc-perf common) +target_link_libraries (date_lut_init PRIVATE common ${PLATFORM_LIBS}) +target_link_libraries (date_lut2 PRIVATE common ${PLATFORM_LIBS}) +target_link_libraries (date_lut3 PRIVATE common ${PLATFORM_LIBS}) +target_link_libraries (date_lut4 PRIVATE common ${PLATFORM_LIBS}) +target_link_libraries (date_lut_default_timezone PRIVATE common ${PLATFORM_LIBS}) +target_link_libraries (local_date_time_comparison PRIVATE common) +target_link_libraries (realloc-perf PRIVATE common) add_check(local_date_time_comparison) if(USE_GTEST) add_executable(unit_tests_libcommon gtest_json_test.cpp gtest_strong_typedef.cpp gtest_find_symbols.cpp) - target_link_libraries(unit_tests_libcommon common ${GTEST_MAIN_LIBRARIES} ${GTEST_LIBRARIES}) + target_link_libraries(unit_tests_libcommon PRIVATE common ${GTEST_MAIN_LIBRARIES} ${GTEST_LIBRARIES}) add_check(unit_tests_libcommon) endif() add_executable (dump_variable dump_variable.cpp) -target_link_libraries (dump_variable clickhouse_common_io) +target_link_libraries (dump_variable PRIVATE clickhouse_common_io) diff --git a/libs/libmysqlxx/src/tests/CMakeLists.txt b/libs/libmysqlxx/src/tests/CMakeLists.txt index d2901513808..ec3fdfaa913 100644 --- a/libs/libmysqlxx/src/tests/CMakeLists.txt +++ b/libs/libmysqlxx/src/tests/CMakeLists.txt @@ -1,2 +1,2 @@ add_executable (mysqlxx_test mysqlxx_test.cpp) -target_link_libraries (mysqlxx_test mysqlxx) +target_link_libraries (mysqlxx_test PRIVATE mysqlxx)