mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-27 18:12:02 +00:00
Improve jemalloc contrib
This commit is contained in:
Ivan Lezhankin
parent
48a2b46499
commit
528e42bc4c
@ -351,18 +351,12 @@ if (ENABLE_TESTS)
|
||||
endif ()
|
||||
|
||||
# Need to process before "contrib" dir:
|
||||
include (cmake/find/jemalloc.cmake)
|
||||
include (cmake/find/cctz.cmake)
|
||||
include (cmake/find/mysqlclient.cmake)
|
||||
|
||||
# When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc.
|
||||
|
||||
if (USE_JEMALLOC)
|
||||
message (STATUS "Link jemalloc: ${JEMALLOC_LIBRARIES}")
|
||||
set (MALLOC_LIBRARIES ${JEMALLOC_LIBRARIES})
|
||||
elseif (SANITIZE)
|
||||
message (STATUS "Will use ${SANITIZE} sanitizer.")
|
||||
elseif (OS_LINUX)
|
||||
if (OS_LINUX AND NOT ENABLE_JEMALLOC)
|
||||
message (WARNING "Non default allocator is disabled. This is not recommended for production Linux builds.")
|
||||
endif ()
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#include <common/argsToConfig.h>
|
||||
#include "argsToConfig.h"
|
||||
|
||||
#include <Poco/Util/Application.h>
|
||||
#include <Poco/Util/LayeredConfiguration.h>
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
#pragma once
|
||||
|
||||
#include <Poco/Util/Application.h>
|
||||
|
||||
namespace Poco::Util
|
||||
|
||||
@ -2,6 +2,5 @@
|
||||
|
||||
// .h autogenerated by cmake !
|
||||
|
||||
#cmakedefine01 USE_JEMALLOC
|
||||
#cmakedefine01 UNBUNDLED
|
||||
#cmakedefine01 WITH_COVERAGE
|
||||
|
||||
@ -1,16 +1,17 @@
|
||||
#include <common/coverage.h>
|
||||
#include "coverage.h"
|
||||
|
||||
#include <common/config_common.h>
|
||||
|
||||
#if WITH_COVERAGE
|
||||
|
||||
#include <unistd.h>
|
||||
#include <mutex>
|
||||
# include <mutex>
|
||||
# include <unistd.h>
|
||||
|
||||
#if defined(__clang__)
|
||||
# if defined(__clang__)
|
||||
extern "C" void __llvm_profile_dump();
|
||||
#elif defined(__GNUC__) || defined(__GNUG__)
|
||||
# elif defined(__GNUC__) || defined(__GNUG__)
|
||||
extern "C" void __gcov_exit();
|
||||
#endif
|
||||
# endif
|
||||
|
||||
#endif
|
||||
|
||||
@ -21,11 +22,11 @@ void dumpCoverageReportIfPossible()
|
||||
static std::mutex mutex;
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
#if defined(__clang__)
|
||||
# if defined(__clang__)
|
||||
__llvm_profile_dump();
|
||||
#elif defined(__GNUC__) || defined(__GNUG__)
|
||||
# elif defined(__GNUC__) || defined(__GNUG__)
|
||||
__gcov_exit();
|
||||
#endif
|
||||
# endif
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -1,46 +0,0 @@
|
||||
# https://github.com/bro/cmake/blob/master/FindJeMalloc.cmake
|
||||
#
|
||||
# - Try to find jemalloc headers and libraries.
|
||||
#
|
||||
# Usage of this module as follows:
|
||||
#
|
||||
# find_package(JeMalloc)
|
||||
#
|
||||
# Variables used by this module, they can change the default behaviour and need
|
||||
# to be set before calling find_package:
|
||||
#
|
||||
# JEMALLOC_ROOT_DIR Set this variable to the root installation of
|
||||
# jemalloc if the module has problems finding
|
||||
# the proper installation path.
|
||||
#
|
||||
# Variables defined by this module:
|
||||
#
|
||||
# JEMALLOC_FOUND System has jemalloc libs/headers
|
||||
# JEMALLOC_LIBRARIES The jemalloc library/libraries
|
||||
# JEMALLOC_INCLUDE_DIR The location of jemalloc headers
|
||||
|
||||
find_path(JEMALLOC_ROOT_DIR
|
||||
NAMES include/jemalloc/jemalloc.h
|
||||
)
|
||||
|
||||
find_library(JEMALLOC_LIBRARIES
|
||||
NAMES jemalloc
|
||||
HINTS ${JEMALLOC_ROOT_DIR}/lib
|
||||
)
|
||||
|
||||
find_path(JEMALLOC_INCLUDE_DIR
|
||||
NAMES jemalloc/jemalloc.h
|
||||
HINTS ${JEMALLOC_ROOT_DIR}/include
|
||||
)
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(JeMalloc DEFAULT_MSG
|
||||
JEMALLOC_LIBRARIES
|
||||
JEMALLOC_INCLUDE_DIR
|
||||
)
|
||||
|
||||
mark_as_advanced(
|
||||
JEMALLOC_ROOT_DIR
|
||||
JEMALLOC_LIBRARIES
|
||||
JEMALLOC_INCLUDE_DIR
|
||||
)
|
||||
@ -1,42 +0,0 @@
|
||||
if (OS_LINUX AND NOT SANITIZE AND NOT ARCH_32 AND NOT ARCH_PPC64LE)
|
||||
set(ENABLE_JEMALLOC_DEFAULT ${ENABLE_LIBRARIES})
|
||||
else ()
|
||||
set(ENABLE_JEMALLOC_DEFAULT 0)
|
||||
endif ()
|
||||
|
||||
option (ENABLE_JEMALLOC "Set to TRUE to use jemalloc" ${ENABLE_JEMALLOC_DEFAULT})
|
||||
if (OS_LINUX)
|
||||
option (USE_INTERNAL_JEMALLOC_LIBRARY "Set to FALSE to use system jemalloc library instead of bundled" ${NOT_UNBUNDLED})
|
||||
else()
|
||||
option (USE_INTERNAL_JEMALLOC_LIBRARY "Set to FALSE to use system jemalloc library instead of bundled" OFF)
|
||||
endif()
|
||||
|
||||
if (ENABLE_JEMALLOC)
|
||||
if (USE_INTERNAL_JEMALLOC_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/jemalloc/src/jemalloc.c")
|
||||
message (WARNING "submodule contrib/jemalloc is missing. to fix try run: \n git submodule update --init --recursive")
|
||||
set (USE_INTERNAL_JEMALLOC_LIBRARY 0)
|
||||
set (MISSING_INTERNAL_JEMALLOC_LIBRARY 1)
|
||||
endif ()
|
||||
|
||||
if (NOT USE_INTERNAL_JEMALLOC_LIBRARY)
|
||||
find_package (JeMalloc)
|
||||
endif ()
|
||||
|
||||
if ((NOT JEMALLOC_LIBRARIES OR NOT JEMALLOC_INCLUDE_DIR) AND NOT MISSING_INTERNAL_JEMALLOC_LIBRARY )
|
||||
set (JEMALLOC_LIBRARIES "jemalloc")
|
||||
set (JEMALLOC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/jemalloc-cmake/include" "${ClickHouse_SOURCE_DIR}/contrib/jemalloc-cmake/include_linux_x86_64")
|
||||
set (USE_INTERNAL_JEMALLOC_LIBRARY 1)
|
||||
endif ()
|
||||
|
||||
if (JEMALLOC_LIBRARIES)
|
||||
set (USE_JEMALLOC 1)
|
||||
elseif (NOT MISSING_INTERNAL_JEMALLOC_LIBRARY)
|
||||
message (FATAL_ERROR "ENABLE_JEMALLOC is set to true, but library was not found")
|
||||
endif ()
|
||||
|
||||
if (SANITIZE)
|
||||
message (FATAL_ERROR "ENABLE_JEMALLOC is set to true, but it cannot be used with sanitizers")
|
||||
endif ()
|
||||
|
||||
message (STATUS "Using jemalloc=${USE_JEMALLOC}: ${JEMALLOC_INCLUDE_DIR} : ${JEMALLOC_LIBRARIES}")
|
||||
endif ()
|
||||
4
contrib/CMakeLists.txt
vendored
4
contrib/CMakeLists.txt
vendored
@ -79,9 +79,7 @@ if (USE_INTERNAL_CCTZ_LIBRARY)
|
||||
add_subdirectory (cctz-cmake)
|
||||
endif ()
|
||||
|
||||
if (ENABLE_JEMALLOC AND USE_INTERNAL_JEMALLOC_LIBRARY)
|
||||
add_subdirectory (jemalloc-cmake)
|
||||
endif ()
|
||||
add_subdirectory (jemalloc-cmake)
|
||||
|
||||
if (USE_INTERNAL_CPUID_LIBRARY)
|
||||
add_subdirectory (libcpuid)
|
||||
|
||||
@ -1,70 +1,108 @@
|
||||
set(JEMALLOC_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/jemalloc)
|
||||
option (ENABLE_JEMALLOC "Enable jemalloc allocator" ${ENABLE_LIBRARIES})
|
||||
|
||||
set(SRCS
|
||||
${JEMALLOC_SOURCE_DIR}/src/arena.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/background_thread.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/base.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/bin.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/bitmap.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/ckh.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/ctl.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/div.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/extent.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/extent_dss.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/extent_mmap.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/hash.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/hook.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/jemalloc.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/large.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/log.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/malloc_io.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/mutex.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/mutex_pool.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/nstime.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/pages.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/prng.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/prof.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/rtree.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/sc.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/stats.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/sz.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/tcache.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/test_hooks.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/ticker.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/tsd.c
|
||||
${JEMALLOC_SOURCE_DIR}/src/witness.c
|
||||
)
|
||||
|
||||
if(OS_DARWIN)
|
||||
list(APPEND SRCS ${JEMALLOC_SOURCE_DIR}/src/zone.c)
|
||||
endif()
|
||||
|
||||
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
|
||||
if (SANITIZE)
|
||||
set (ENABLE_JEMALLOC FALSE)
|
||||
message (STATUS "jemalloc is disabled because of sanitizers")
|
||||
endif ()
|
||||
|
||||
add_library(jemalloc STATIC ${SRCS})
|
||||
if (ENABLE_JEMALLOC)
|
||||
option (USE_INTERNAL_JEMALLOC "Use internal jemalloc library" ${NOT_UNBUNDLED})
|
||||
|
||||
target_include_directories(jemalloc PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
|
||||
if (USE_INTERNAL_JEMALLOC)
|
||||
if (NOT OS_LINUX OR NOT (ARCH_AMD64 OR ARCH_ARM))
|
||||
message (FATAL_ERROR "Internal jemalloc can only be used on Linux with x86_64 or aarch64.")
|
||||
endif ()
|
||||
|
||||
if (ARCH_AMD64)
|
||||
target_include_directories(jemalloc PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include_linux_x86_64)
|
||||
elseif (ARCH_ARM)
|
||||
target_include_directories(jemalloc PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include_linux_aarch64)
|
||||
else ()
|
||||
message (FATAL_ERROR "jemalloc can only be used on x86_64 or aarch64.")
|
||||
endif ()
|
||||
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/jemalloc")
|
||||
|
||||
target_include_directories(jemalloc PRIVATE
|
||||
${JEMALLOC_SOURCE_DIR}/include)
|
||||
set (SRCS
|
||||
${LIBRARY_DIR}/src/arena.c
|
||||
${LIBRARY_DIR}/src/background_thread.c
|
||||
${LIBRARY_DIR}/src/base.c
|
||||
${LIBRARY_DIR}/src/bin.c
|
||||
${LIBRARY_DIR}/src/bitmap.c
|
||||
${LIBRARY_DIR}/src/ckh.c
|
||||
${LIBRARY_DIR}/src/ctl.c
|
||||
${LIBRARY_DIR}/src/div.c
|
||||
${LIBRARY_DIR}/src/extent.c
|
||||
${LIBRARY_DIR}/src/extent_dss.c
|
||||
${LIBRARY_DIR}/src/extent_mmap.c
|
||||
${LIBRARY_DIR}/src/hash.c
|
||||
${LIBRARY_DIR}/src/hook.c
|
||||
${LIBRARY_DIR}/src/jemalloc.c
|
||||
${LIBRARY_DIR}/src/large.c
|
||||
${LIBRARY_DIR}/src/log.c
|
||||
${LIBRARY_DIR}/src/malloc_io.c
|
||||
${LIBRARY_DIR}/src/mutex.c
|
||||
${LIBRARY_DIR}/src/mutex_pool.c
|
||||
${LIBRARY_DIR}/src/nstime.c
|
||||
${LIBRARY_DIR}/src/pages.c
|
||||
${LIBRARY_DIR}/src/prng.c
|
||||
${LIBRARY_DIR}/src/prof.c
|
||||
${LIBRARY_DIR}/src/rtree.c
|
||||
${LIBRARY_DIR}/src/sc.c
|
||||
${LIBRARY_DIR}/src/stats.c
|
||||
${LIBRARY_DIR}/src/sz.c
|
||||
${LIBRARY_DIR}/src/tcache.c
|
||||
${LIBRARY_DIR}/src/test_hooks.c
|
||||
${LIBRARY_DIR}/src/ticker.c
|
||||
${LIBRARY_DIR}/src/tsd.c
|
||||
${LIBRARY_DIR}/src/witness.c
|
||||
)
|
||||
if (OS_DARWIN)
|
||||
list(APPEND SRCS ${LIBRARY_DIR}/src/zone.c)
|
||||
endif ()
|
||||
|
||||
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
|
||||
add_library(jemalloc STATIC ${SRCS})
|
||||
target_include_directories(jemalloc PRIVATE ${LIBRARY_DIR}/include)
|
||||
target_include_directories(jemalloc PUBLIC include)
|
||||
if (ARCH_AMD64)
|
||||
target_include_directories(jemalloc PUBLIC include_linux_x86_64)
|
||||
elseif (ARCH_ARM)
|
||||
target_include_directories(jemalloc PUBLIC include_linux_aarch64)
|
||||
endif ()
|
||||
|
||||
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
|
||||
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1 -DJEMALLOC_PROF=1)
|
||||
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
|
||||
|
||||
if (USE_UNWIND)
|
||||
target_compile_definitions (jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1)
|
||||
target_link_libraries (jemalloc PRIVATE ${UNWIND_LIBRARIES})
|
||||
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
|
||||
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1 -DJEMALLOC_PROF=1)
|
||||
|
||||
if (USE_UNWIND)
|
||||
target_compile_definitions (jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1)
|
||||
target_link_libraries (jemalloc PRIVATE unwind)
|
||||
endif ()
|
||||
endif ()
|
||||
else ()
|
||||
find_library(LIBRARY_JEMALLOC jemalloc)
|
||||
find_path(INCLUDE_JEMALLOC jemalloc/jemalloc.h)
|
||||
|
||||
add_library (jemalloc STATIC IMPORTED)
|
||||
set_property (TARGET jemalloc PROPERTY IMPORTED_LOCATION ${LIBRARY_JEMALLOC})
|
||||
target_include_directories (jemalloc PUBLIC ${INCLUDE_JEMALLOC})
|
||||
|
||||
set (CMAKE_REQUIRED_LIBRARIES jemalloc)
|
||||
check_cxx_source_compiles (
|
||||
"
|
||||
#include <jemalloc/jemalloc.h>
|
||||
|
||||
int main() {
|
||||
free(mallocx(1, 0));
|
||||
}
|
||||
"
|
||||
EXTERNAL_JEMALLOC_WORKS
|
||||
)
|
||||
|
||||
if (NOT EXTERNAL_JEMALLOC_WORKS)
|
||||
message (FATAL_ERROR "jemalloc is unusable: ${LIBRARY_JEMALLOC} ${INCLUDE_JEMALLOC}")
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
target_compile_definitions(jemalloc PUBLIC USE_JEMALLOC=1)
|
||||
|
||||
message (STATUS "Using jemalloc")
|
||||
else ()
|
||||
add_library(jemalloc INTERFACE)
|
||||
target_compile_definitions(jemalloc INTERFACE USE_JEMALLOC=0)
|
||||
|
||||
message (STATUS "Not using jemalloc")
|
||||
endif ()
|
||||
|
||||
@ -256,7 +256,7 @@ add_library (clickhouse_malloc OBJECT src/Common/malloc.cpp)
|
||||
set_source_files_properties(src/Common/malloc.cpp PROPERTIES COMPILE_FLAGS "-fno-builtin")
|
||||
|
||||
add_library (clickhouse_new_delete STATIC src/Common/new_delete.cpp)
|
||||
target_link_libraries (clickhouse_new_delete PRIVATE clickhouse_common_io)
|
||||
target_link_libraries (clickhouse_new_delete PRIVATE clickhouse_common_io jemalloc)
|
||||
|
||||
if (OS_FREEBSD)
|
||||
target_compile_definitions (clickhouse_common_io PUBLIC CLOCK_MONOTONIC_COARSE=CLOCK_MONOTONIC_FAST)
|
||||
@ -303,10 +303,12 @@ add_object_library(clickhouse_processors_sources src/Processors/Sources)
|
||||
|
||||
if (MAKE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES)
|
||||
add_library (dbms STATIC ${dbms_headers} ${dbms_sources})
|
||||
target_link_libraries (dbms PRIVATE jemalloc)
|
||||
set (all_modules dbms)
|
||||
else()
|
||||
add_library (dbms SHARED ${dbms_headers} ${dbms_sources})
|
||||
target_link_libraries (dbms PUBLIC ${all_modules})
|
||||
target_link_libraries (clickhouse_interpreters PRIVATE jemalloc)
|
||||
list (APPEND all_modules dbms)
|
||||
# force all split libs to be linked
|
||||
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-as-needed")
|
||||
@ -555,18 +557,6 @@ if (USE_BROTLI)
|
||||
target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BROTLI_INCLUDE_DIR})
|
||||
endif()
|
||||
|
||||
if (USE_JEMALLOC)
|
||||
dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp
|
||||
target_include_directories (clickhouse_new_delete SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR})
|
||||
|
||||
if(NOT MAKE_STATIC_LIBRARIES AND ${JEMALLOC_LIBRARIES} MATCHES "${CMAKE_STATIC_LIBRARY_SUFFIX}$")
|
||||
# mallctl in dbms/src/Interpreters/AsynchronousMetrics.cpp
|
||||
# Actually we link JEMALLOC to almost all libraries.
|
||||
# This is just hotfix for some uninvestigated problem.
|
||||
target_link_libraries(clickhouse_interpreters PRIVATE ${JEMALLOC_LIBRARIES})
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR})
|
||||
target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR})
|
||||
|
||||
|
||||
@ -34,7 +34,6 @@ const char * auto_config_build[]
|
||||
"SPLIT_BINARY", "@CLICKHOUSE_SPLIT_BINARY@",
|
||||
"USE_EMBEDDED_COMPILER", "@USE_EMBEDDED_COMPILER@",
|
||||
"USE_GLIBC_COMPATIBILITY", "@GLIBC_COMPATIBILITY@",
|
||||
"USE_JEMALLOC", "@USE_JEMALLOC@",
|
||||
"USE_MIMALLOC", "@USE_MIMALLOC@",
|
||||
"USE_UNWIND", "@USE_UNWIND@",
|
||||
"USE_ICU", "@USE_ICU@",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user