mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-19 16:20:50 +00:00
Merge pull request #67098 from ClickHouse/add-numactl
Try calculating available memory if ClickHouse is bound to subset of NUMA nodes
This commit is contained in:
commit
2ea4bfe04d
3
.gitmodules
vendored
3
.gitmodules
vendored
@ -372,3 +372,6 @@
|
|||||||
[submodule "contrib/double-conversion"]
|
[submodule "contrib/double-conversion"]
|
||||||
path = contrib/double-conversion
|
path = contrib/double-conversion
|
||||||
url = https://github.com/ClickHouse/double-conversion.git
|
url = https://github.com/ClickHouse/double-conversion.git
|
||||||
|
[submodule "contrib/numactl"]
|
||||||
|
path = contrib/numactl
|
||||||
|
url = https://github.com/ClickHouse/numactl.git
|
||||||
|
@ -32,6 +32,7 @@ set (SRCS
|
|||||||
StringRef.cpp
|
StringRef.cpp
|
||||||
safeExit.cpp
|
safeExit.cpp
|
||||||
throwError.cpp
|
throwError.cpp
|
||||||
|
Numa.cpp
|
||||||
)
|
)
|
||||||
|
|
||||||
add_library (common ${SRCS})
|
add_library (common ${SRCS})
|
||||||
@ -46,6 +47,10 @@ if (TARGET ch_contrib::crc32_s390x)
|
|||||||
target_link_libraries(common PUBLIC ch_contrib::crc32_s390x)
|
target_link_libraries(common PUBLIC ch_contrib::crc32_s390x)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
if (TARGET ch_contrib::numactl)
|
||||||
|
target_link_libraries(common PUBLIC ch_contrib::numactl)
|
||||||
|
endif()
|
||||||
|
|
||||||
target_include_directories(common PUBLIC .. "${CMAKE_CURRENT_BINARY_DIR}/..")
|
target_include_directories(common PUBLIC .. "${CMAKE_CURRENT_BINARY_DIR}/..")
|
||||||
|
|
||||||
target_link_libraries (common
|
target_link_libraries (common
|
||||||
|
37
base/base/Numa.cpp
Normal file
37
base/base/Numa.cpp
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
#include <base/Numa.h>
|
||||||
|
|
||||||
|
#include "config.h"
|
||||||
|
|
||||||
|
#if USE_NUMACTL
|
||||||
|
# include <numa.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
namespace DB
|
||||||
|
{
|
||||||
|
|
||||||
|
std::optional<size_t> getNumaNodesTotalMemory()
|
||||||
|
{
|
||||||
|
std::optional<size_t> total_memory;
|
||||||
|
#if USE_NUMACTL
|
||||||
|
if (numa_available() != -1)
|
||||||
|
{
|
||||||
|
auto * membind = numa_get_membind();
|
||||||
|
if (!numa_bitmask_equal(membind, numa_all_nodes_ptr))
|
||||||
|
{
|
||||||
|
total_memory.emplace(0);
|
||||||
|
auto max_node = numa_max_node();
|
||||||
|
for (int i = 0; i <= max_node; ++i)
|
||||||
|
{
|
||||||
|
if (numa_bitmask_isbitset(membind, i))
|
||||||
|
*total_memory += numa_node_size(i, nullptr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
numa_bitmask_free(membind);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
||||||
|
return total_memory;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
12
base/base/Numa.h
Normal file
12
base/base/Numa.h
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <optional>
|
||||||
|
|
||||||
|
namespace DB
|
||||||
|
{
|
||||||
|
|
||||||
|
/// return total memory of NUMA nodes the process is bound to
|
||||||
|
/// if NUMA is not supported or process can use all nodes, std::nullopt is returned
|
||||||
|
std::optional<size_t> getNumaNodesTotalMemory();
|
||||||
|
|
||||||
|
}
|
@ -2,15 +2,14 @@
|
|||||||
|
|
||||||
#include <base/cgroupsv2.h>
|
#include <base/cgroupsv2.h>
|
||||||
#include <base/getPageSize.h>
|
#include <base/getPageSize.h>
|
||||||
|
#include <base/Numa.h>
|
||||||
|
|
||||||
#include <fstream>
|
#include <fstream>
|
||||||
#include <stdexcept>
|
|
||||||
|
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
#include <sys/param.h>
|
#include <sys/param.h>
|
||||||
|
|
||||||
|
|
||||||
namespace
|
namespace
|
||||||
{
|
{
|
||||||
|
|
||||||
@ -63,6 +62,9 @@ uint64_t getMemoryAmountOrZero()
|
|||||||
|
|
||||||
uint64_t memory_amount = num_pages * page_size;
|
uint64_t memory_amount = num_pages * page_size;
|
||||||
|
|
||||||
|
if (auto total_numa_memory = DB::getNumaNodesTotalMemory(); total_numa_memory.has_value())
|
||||||
|
memory_amount = *total_numa_memory;
|
||||||
|
|
||||||
/// Respect the memory limit set by cgroups v2.
|
/// Respect the memory limit set by cgroups v2.
|
||||||
auto limit_v2 = getCgroupsV2MemoryLimit();
|
auto limit_v2 = getCgroupsV2MemoryLimit();
|
||||||
if (limit_v2.has_value() && *limit_v2 < memory_amount)
|
if (limit_v2.has_value() && *limit_v2 < memory_amount)
|
||||||
|
2
contrib/CMakeLists.txt
vendored
2
contrib/CMakeLists.txt
vendored
@ -230,6 +230,8 @@ add_contrib (libssh-cmake libssh)
|
|||||||
|
|
||||||
add_contrib (prometheus-protobufs-cmake prometheus-protobufs prometheus-protobufs-gogo)
|
add_contrib (prometheus-protobufs-cmake prometheus-protobufs prometheus-protobufs-gogo)
|
||||||
|
|
||||||
|
add_contrib(numactl-cmake numactl)
|
||||||
|
|
||||||
# Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
|
# Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
|
||||||
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
|
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
|
||||||
# in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually,
|
# in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually,
|
||||||
|
1
contrib/numactl
vendored
Submodule
1
contrib/numactl
vendored
Submodule
@ -0,0 +1 @@
|
|||||||
|
Subproject commit 8d13d63a05f0c3cd88bf777cbb61541202b7da08
|
30
contrib/numactl-cmake/CMakeLists.txt
Normal file
30
contrib/numactl-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
if (NOT (
|
||||||
|
OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64 OR ARCH_LOONGARCH64))
|
||||||
|
)
|
||||||
|
if (ENABLE_NUMACTL)
|
||||||
|
message (${RECONFIGURE_MESSAGE_LEVEL}
|
||||||
|
"numactl is disabled implicitly because the OS or architecture is not supported. Use -DENABLE_NUMACTL=0")
|
||||||
|
endif ()
|
||||||
|
set (ENABLE_NUMACTL OFF)
|
||||||
|
else()
|
||||||
|
option (ENABLE_NUMACTL "Enable numactl" ${ENABLE_LIBRARIES})
|
||||||
|
endif()
|
||||||
|
|
||||||
|
if (NOT ENABLE_NUMACTL)
|
||||||
|
message (STATUS "Not using numactl")
|
||||||
|
return()
|
||||||
|
endif ()
|
||||||
|
|
||||||
|
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/numactl")
|
||||||
|
|
||||||
|
set (SRCS
|
||||||
|
"${LIBRARY_DIR}/libnuma.c"
|
||||||
|
"${LIBRARY_DIR}/syscall.c"
|
||||||
|
)
|
||||||
|
|
||||||
|
add_library(_numactl ${SRCS})
|
||||||
|
|
||||||
|
target_include_directories(_numactl SYSTEM PRIVATE include)
|
||||||
|
target_include_directories(_numactl SYSTEM PUBLIC "${LIBRARY_DIR}")
|
||||||
|
|
||||||
|
add_library(ch_contrib::numactl ALIAS _numactl)
|
82
contrib/numactl-cmake/include/config.h
Normal file
82
contrib/numactl-cmake/include/config.h
Normal file
@ -0,0 +1,82 @@
|
|||||||
|
/* config.h. Generated from config.h.in by configure. */
|
||||||
|
/* config.h.in. Generated from configure.ac by autoheader. */
|
||||||
|
|
||||||
|
/* Checking for symver attribute */
|
||||||
|
#define HAVE_ATTRIBUTE_SYMVER 0
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <dlfcn.h> header file. */
|
||||||
|
#define HAVE_DLFCN_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <inttypes.h> header file. */
|
||||||
|
#define HAVE_INTTYPES_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <stdint.h> header file. */
|
||||||
|
#define HAVE_STDINT_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <stdio.h> header file. */
|
||||||
|
#define HAVE_STDIO_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <stdlib.h> header file. */
|
||||||
|
#define HAVE_STDLIB_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <strings.h> header file. */
|
||||||
|
#define HAVE_STRINGS_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <string.h> header file. */
|
||||||
|
#define HAVE_STRING_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <sys/stat.h> header file. */
|
||||||
|
#define HAVE_SYS_STAT_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <sys/types.h> header file. */
|
||||||
|
#define HAVE_SYS_TYPES_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <unistd.h> header file. */
|
||||||
|
#define HAVE_UNISTD_H 1
|
||||||
|
|
||||||
|
/* Define to the sub-directory where libtool stores uninstalled libraries. */
|
||||||
|
#define LT_OBJDIR ".libs/"
|
||||||
|
|
||||||
|
/* Name of package */
|
||||||
|
#define PACKAGE "numactl"
|
||||||
|
|
||||||
|
/* Define to the address where bug reports for this package should be sent. */
|
||||||
|
#define PACKAGE_BUGREPORT ""
|
||||||
|
|
||||||
|
/* Define to the full name of this package. */
|
||||||
|
#define PACKAGE_NAME "numactl"
|
||||||
|
|
||||||
|
/* Define to the full name and version of this package. */
|
||||||
|
#define PACKAGE_STRING "numactl 2.1"
|
||||||
|
|
||||||
|
/* Define to the one symbol short name of this package. */
|
||||||
|
#define PACKAGE_TARNAME "numactl"
|
||||||
|
|
||||||
|
/* Define to the home page for this package. */
|
||||||
|
#define PACKAGE_URL ""
|
||||||
|
|
||||||
|
/* Define to the version of this package. */
|
||||||
|
#define PACKAGE_VERSION "2.1"
|
||||||
|
|
||||||
|
/* Define to 1 if all of the C89 standard headers exist (not just the ones
|
||||||
|
required in a freestanding environment). This macro is provided for
|
||||||
|
backward compatibility; new code need not use it. */
|
||||||
|
#define STDC_HEADERS 1
|
||||||
|
|
||||||
|
/* If the compiler supports a TLS storage class define it to that here */
|
||||||
|
#define TLS __thread
|
||||||
|
|
||||||
|
/* Version number of package */
|
||||||
|
#define VERSION "2.1"
|
||||||
|
|
||||||
|
/* Number of bits in a file offset, on hosts where this is settable. */
|
||||||
|
/* #undef _FILE_OFFSET_BITS */
|
||||||
|
|
||||||
|
/* Define to 1 on platforms where this makes off_t a 64-bit type. */
|
||||||
|
/* #undef _LARGE_FILES */
|
||||||
|
|
||||||
|
/* Number of bits in time_t, on hosts where this is settable. */
|
||||||
|
/* #undef _TIME_BITS */
|
||||||
|
|
||||||
|
/* Define to 1 on platforms where this makes time_t a 64-bit type. */
|
||||||
|
/* #undef __MINGW_USE_VC2005_COMPAT */
|
@ -13,6 +13,7 @@ entry="/usr/share/clickhouse-test/performance/scripts/entrypoint.sh"
|
|||||||
# https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt
|
# https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt
|
||||||
# Double-escaped backslashes are a tribute to the engineering wonder of docker --
|
# Double-escaped backslashes are a tribute to the engineering wonder of docker --
|
||||||
# it gives '/bin/sh: 1: [bash,: not found' otherwise.
|
# it gives '/bin/sh: 1: [bash,: not found' otherwise.
|
||||||
|
numactl --hardware
|
||||||
node=$(( RANDOM % $(numactl --hardware | sed -n 's/^.*available:\(.*\)nodes.*$/\1/p') ));
|
node=$(( RANDOM % $(numactl --hardware | sed -n 's/^.*available:\(.*\)nodes.*$/\1/p') ));
|
||||||
echo Will bind to NUMA node $node;
|
echo Will bind to NUMA node $node;
|
||||||
numactl --cpunodebind=$node --membind=$node $entry
|
numactl --cpunodebind=$node --membind=$node $entry
|
||||||
|
@ -19,6 +19,7 @@
|
|||||||
#include <base/getMemoryAmount.h>
|
#include <base/getMemoryAmount.h>
|
||||||
#include <base/scope_guard.h>
|
#include <base/scope_guard.h>
|
||||||
#include <base/safeExit.h>
|
#include <base/safeExit.h>
|
||||||
|
#include <base/Numa.h>
|
||||||
#include <Poco/Net/NetException.h>
|
#include <Poco/Net/NetException.h>
|
||||||
#include <Poco/Net/TCPServerParams.h>
|
#include <Poco/Net/TCPServerParams.h>
|
||||||
#include <Poco/Net/TCPServer.h>
|
#include <Poco/Net/TCPServer.h>
|
||||||
@ -311,6 +312,12 @@ try
|
|||||||
|
|
||||||
MainThreadStatus::getInstance();
|
MainThreadStatus::getInstance();
|
||||||
|
|
||||||
|
if (auto total_numa_memory = getNumaNodesTotalMemory(); total_numa_memory.has_value())
|
||||||
|
{
|
||||||
|
LOG_INFO(
|
||||||
|
log, "Keeper is bound to a subset of NUMA nodes. Total memory of all available nodes: {}", ReadableSize(*total_numa_memory));
|
||||||
|
}
|
||||||
|
|
||||||
#if !defined(NDEBUG) || !defined(__OPTIMIZE__)
|
#if !defined(NDEBUG) || !defined(__OPTIMIZE__)
|
||||||
LOG_WARNING(log, "Keeper was built in debug mode. It will work slowly.");
|
LOG_WARNING(log, "Keeper was built in debug mode. It will work slowly.");
|
||||||
#endif
|
#endif
|
||||||
|
@ -22,6 +22,7 @@
|
|||||||
#include <base/coverage.h>
|
#include <base/coverage.h>
|
||||||
#include <base/getFQDNOrHostName.h>
|
#include <base/getFQDNOrHostName.h>
|
||||||
#include <base/safeExit.h>
|
#include <base/safeExit.h>
|
||||||
|
#include <base/Numa.h>
|
||||||
#include <Common/PoolId.h>
|
#include <Common/PoolId.h>
|
||||||
#include <Common/MemoryTracker.h>
|
#include <Common/MemoryTracker.h>
|
||||||
#include <Common/ClickHouseRevision.h>
|
#include <Common/ClickHouseRevision.h>
|
||||||
@ -140,6 +141,7 @@
|
|||||||
# include <azure/core/diagnostics/logger.hpp>
|
# include <azure/core/diagnostics/logger.hpp>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
||||||
#include <incbin.h>
|
#include <incbin.h>
|
||||||
/// A minimal file used when the server is run without installation
|
/// A minimal file used when the server is run without installation
|
||||||
INCBIN(resource_embedded_xml, SOURCE_DIR "/programs/server/embedded.xml");
|
INCBIN(resource_embedded_xml, SOURCE_DIR "/programs/server/embedded.xml");
|
||||||
@ -754,6 +756,12 @@ try
|
|||||||
setenv("OPENSSL_CONF", config_dir.c_str(), true); /// NOLINT
|
setenv("OPENSSL_CONF", config_dir.c_str(), true); /// NOLINT
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (auto total_numa_memory = getNumaNodesTotalMemory(); total_numa_memory.has_value())
|
||||||
|
{
|
||||||
|
LOG_INFO(
|
||||||
|
log, "ClickHouse is bound to a subset of NUMA nodes. Total memory of all available nodes: {}", ReadableSize(*total_numa_memory));
|
||||||
|
}
|
||||||
|
|
||||||
registerInterpreters();
|
registerInterpreters();
|
||||||
registerFunctions();
|
registerFunctions();
|
||||||
registerAggregateFunctions();
|
registerAggregateFunctions();
|
||||||
|
@ -64,6 +64,7 @@
|
|||||||
#cmakedefine01 USE_LIBARCHIVE
|
#cmakedefine01 USE_LIBARCHIVE
|
||||||
#cmakedefine01 USE_POCKETFFT
|
#cmakedefine01 USE_POCKETFFT
|
||||||
#cmakedefine01 USE_PROMETHEUS_PROTOBUFS
|
#cmakedefine01 USE_PROMETHEUS_PROTOBUFS
|
||||||
|
#cmakedefine01 USE_NUMACTL
|
||||||
|
|
||||||
/// This is needed for .incbin in assembly. For some reason, include paths don't work there in presence of LTO.
|
/// This is needed for .incbin in assembly. For some reason, include paths don't work there in presence of LTO.
|
||||||
/// That's why we use absolute paths.
|
/// That's why we use absolute paths.
|
||||||
|
@ -173,5 +173,8 @@ endif()
|
|||||||
if (TARGET ch_contrib::prometheus_protobufs)
|
if (TARGET ch_contrib::prometheus_protobufs)
|
||||||
set(USE_PROMETHEUS_PROTOBUFS 1)
|
set(USE_PROMETHEUS_PROTOBUFS 1)
|
||||||
endif()
|
endif()
|
||||||
|
if (TARGET ch_contrib::numactl)
|
||||||
|
set(USE_NUMACTL 1)
|
||||||
|
endif()
|
||||||
|
|
||||||
set(SOURCE_DIR ${PROJECT_SOURCE_DIR})
|
set(SOURCE_DIR ${PROJECT_SOURCE_DIR})
|
||||||
|
Loading…
Reference in New Issue
Block a user