diff --git a/cmake/find_mimalloc.cmake b/cmake/find_mimalloc.cmake index 6e3f24625b6..1820421379f 100644 --- a/cmake/find_mimalloc.cmake +++ b/cmake/find_mimalloc.cmake @@ -1,5 +1,5 @@ if (OS_LINUX AND NOT SANITIZE AND NOT ARCH_ARM AND NOT ARCH_32 AND NOT ARCH_PPC64LE) - option (ENABLE_MIMALLOC "Set to FALSE to disable usage of mimalloc for internal ClickHouse caches" ${NOT_UNBUNDLED}) + option (ENABLE_MIMALLOC "Set to FALSE to disable usage of mimalloc for internal ClickHouse caches" FALSE) endif () if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include/mimalloc.h") @@ -8,6 +8,8 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include/mimalloc.h") endif () if (ENABLE_MIMALLOC) + message (FATAL_ERROR "Mimalloc is not production ready. (Disable with cmake -D ENABLE_MIMALLOC=0). If you want to use mimalloc, you must manually remove this message.") + set (MIMALLOC_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include) set (USE_MIMALLOC 1) set (MIMALLOC_LIBRARY mimalloc-static) diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index b4382755ba3..b825342ee5f 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -178,11 +178,16 @@ StringRef ColumnArray::serializeValueIntoArena(size_t n, Arena & arena, char con char * pos = arena.allocContinue(sizeof(array_size), begin); memcpy(pos, &array_size, sizeof(array_size)); - size_t values_size = 0; - for (size_t i = 0; i < array_size; ++i) - values_size += getData().serializeValueIntoArena(offset + i, arena, begin).size; + StringRef res(pos, sizeof(array_size)); - return StringRef(begin, sizeof(array_size) + values_size); + for (size_t i = 0; i < array_size; ++i) + { + auto value_ref = getData().serializeValueIntoArena(offset + i, arena, begin); + res.data = value_ref.data - res.size; + res.size += value_ref.size; + } + + return res; } diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp index 3ae692552e8..c5cdffba84d 100644 --- a/dbms/src/Columns/ColumnNullable.cpp +++ b/dbms/src/Columns/ColumnNullable.cpp @@ -103,12 +103,13 @@ StringRef ColumnNullable::serializeValueIntoArena(size_t n, Arena & arena, char auto pos = arena.allocContinue(s, begin); memcpy(pos, &arr[n], s); - size_t nested_size = 0; + if (arr[n]) + return StringRef(pos, s); - if (arr[n] == 0) - nested_size = getNestedColumn().serializeValueIntoArena(n, arena, begin).size; + auto nested_ref = getNestedColumn().serializeValueIntoArena(n, arena, begin); - return StringRef{begin, s + nested_size}; + /// serializeValueIntoArena may reallocate memory. Have to use ptr from nested_ref.data and move it back. + return StringRef(nested_ref.data - s, nested_ref.size + s); } const char * ColumnNullable::deserializeAndInsertFromArena(const char * pos) diff --git a/dbms/src/Columns/ColumnTuple.cpp b/dbms/src/Columns/ColumnTuple.cpp index f7a95c1ac9f..3ad7f007edf 100644 --- a/dbms/src/Columns/ColumnTuple.cpp +++ b/dbms/src/Columns/ColumnTuple.cpp @@ -142,11 +142,15 @@ void ColumnTuple::popBack(size_t n) StringRef ColumnTuple::serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const { - size_t values_size = 0; + StringRef res(begin, 0); for (auto & column : columns) - values_size += column->serializeValueIntoArena(n, arena, begin).size; + { + auto value_ref = column->serializeValueIntoArena(n, arena, begin); + res.data = value_ref.data - res.size; + res.size += value_ref.size; + } - return StringRef(begin, values_size); + return res; } const char * ColumnTuple::deserializeAndInsertFromArena(const char * pos) diff --git a/dbms/src/Columns/ColumnUnique.h b/dbms/src/Columns/ColumnUnique.h index febdeaafa95..0c5efd8058d 100644 --- a/dbms/src/Columns/ColumnUnique.h +++ b/dbms/src/Columns/ColumnUnique.h @@ -300,19 +300,19 @@ StringRef ColumnUnique::serializeValueIntoArena(size_t n, Arena & ar { if (is_nullable) { - const UInt8 null_flag = 1; - const UInt8 not_null_flag = 0; + static constexpr auto s = sizeof(UInt8); - auto pos = arena.allocContinue(sizeof(null_flag), begin); - auto & flag = (n == getNullValueIndex() ? null_flag : not_null_flag); - memcpy(pos, &flag, sizeof(flag)); + auto pos = arena.allocContinue(s, begin); + UInt8 flag = (n == getNullValueIndex() ? 1 : 0); + unalignedStore(pos, flag); - size_t nested_size = 0; + if (n == getNullValueIndex()) + return StringRef(pos, s); - if (n != getNullValueIndex()) - nested_size = column_holder->serializeValueIntoArena(n, arena, begin).size; + auto nested_ref = column_holder->serializeValueIntoArena(n, arena, begin); - return StringRef(pos, sizeof(null_flag) + nested_size); + /// serializeValueIntoArena may reallocate memory. Have to use ptr from nested_ref.data and move it back. + return StringRef(nested_ref.data - s, nested_ref.size + s); } return column_holder->serializeValueIntoArena(n, arena, begin); diff --git a/dbms/src/Common/MiAllocator.cpp b/dbms/src/Common/MiAllocator.cpp index 456609374ee..cafa6c135f7 100644 --- a/dbms/src/Common/MiAllocator.cpp +++ b/dbms/src/Common/MiAllocator.cpp @@ -5,15 +5,33 @@ #include "MiAllocator.h" #include +#include +#include +#include + namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_ALLOCATE_MEMORY; +} void * MiAllocator::alloc(size_t size, size_t alignment) { + void * ptr; if (alignment == 0) - return mi_malloc(size); + { + ptr = mi_malloc(size); + if (!ptr) + DB::throwFromErrno("MiAllocator: Cannot allocate in mimalloc " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } else - return mi_malloc_aligned(size, alignment); + { + ptr = mi_malloc_aligned(size, alignment); + if (!ptr) + DB::throwFromErrno("MiAllocator: Cannot allocate in mimalloc (mi_malloc_aligned) " + formatReadableSizeWithBinarySuffix(size) + " with alignment " + toString(alignment) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } + return ptr; } void MiAllocator::free(void * buf, size_t) @@ -32,10 +50,21 @@ void * MiAllocator::realloc(void * old_ptr, size_t, size_t new_size, size_t alig return nullptr; } - if (alignment == 0) - return mi_realloc(old_ptr, alignment); + void * ptr; - return mi_realloc_aligned(old_ptr, new_size, alignment); + if (alignment == 0) + { + ptr = mi_realloc(old_ptr, alignment); + if (!ptr) + DB::throwFromErrno("MiAllocator: Cannot reallocate in mimalloc " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } + else + { + ptr = mi_realloc_aligned(old_ptr, new_size, alignment); + if (!ptr) + DB::throwFromErrno("MiAllocator: Cannot reallocate in mimalloc (mi_realloc_aligned) " + formatReadableSizeWithBinarySuffix(size) + " with alignment " + toString(alignment) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } + return ptr; } } diff --git a/dbms/src/Common/tests/CMakeLists.txt b/dbms/src/Common/tests/CMakeLists.txt index fb28dd14a43..83d69b2c8f2 100644 --- a/dbms/src/Common/tests/CMakeLists.txt +++ b/dbms/src/Common/tests/CMakeLists.txt @@ -75,3 +75,6 @@ target_link_libraries (cow_compositions PRIVATE clickhouse_common_io) add_executable (stopwatch stopwatch.cpp) target_link_libraries (stopwatch PRIVATE clickhouse_common_io) + +add_executable (mi_malloc_test mi_malloc_test.cpp) +target_link_libraries (mi_malloc_test PRIVATE clickhouse_common_io) diff --git a/dbms/src/Common/tests/mi_malloc_test.cpp b/dbms/src/Common/tests/mi_malloc_test.cpp new file mode 100644 index 00000000000..6a565b71ae6 --- /dev/null +++ b/dbms/src/Common/tests/mi_malloc_test.cpp @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include +#include + +#include + +//#undef USE_MIMALLOC +//#define USE_MIMALLOC 0 + +#if USE_MIMALLOC + +#include +#define malloc mi_malloc +#define free mi_free + +#else + +#include + +#endif + + +size_t total_size{0}; + +struct Allocation +{ + void * ptr = nullptr; + size_t size = 0; + + Allocation() {} + + Allocation(size_t size) + : size(size) + { + ptr = malloc(size); + if (!ptr) + throw std::runtime_error("Cannot allocate memory"); + total_size += size; + } + + ~Allocation() + { + if (ptr) + { + free(ptr); + total_size -= size; + } + ptr = nullptr; + } + + Allocation(const Allocation &) = delete; + + Allocation(Allocation && rhs) + { + ptr = rhs.ptr; + size = rhs.size; + rhs.ptr = nullptr; + rhs.size = 0; + } +}; + + +int main(int, char **) +{ + std::vector allocations; + + constexpr size_t limit = 100000000; + constexpr size_t min_alloc_size = 65536; + constexpr size_t max_alloc_size = 10000000; + + std::mt19937 rng; + auto distribution = std::uniform_int_distribution(min_alloc_size, max_alloc_size); + + size_t total_allocations = 0; + + while (true) + { + size_t size = distribution(rng); + + while (total_size + size > limit) + allocations.pop_back(); + + allocations.emplace_back(size); + + ++total_allocations; + if (total_allocations % (1ULL << 20) == 0) + std::cerr << "Total allocations: " << total_allocations << "\n"; + } +} diff --git a/dbms/tests/performance/uniq.xml b/dbms/tests/performance/uniq.xml index af41d1c79e5..c44a4e2ca58 100644 --- a/dbms/tests/performance/uniq.xml +++ b/dbms/tests/performance/uniq.xml @@ -46,8 +46,6 @@ uniqCombined(16) uniqCombined(17) uniqCombined(18) - uniqCombined(19) - uniqCombined(20) uniqUpTo(3) uniqUpTo(5) uniqUpTo(10) diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md index 7c302884631..f96507320a5 100644 --- a/docs/en/interfaces/third-party/integrations.md +++ b/docs/en/interfaces/third-party/integrations.md @@ -14,6 +14,7 @@ - [clickhousedb_fdw](https://github.com/Percona-Lab/clickhousedb_fdw) - [infi.clickhouse_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (uses [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm)) - [pg2ch](https://github.com/mkabilov/pg2ch) + - [clickhouse_fdw](https://github.com/adjust/clickhouse_fdw) - [MSSQL](https://en.wikipedia.org/wiki/Microsoft_SQL_Server) - [ClickHouseMigrator](https://github.com/zlzforever/ClickHouseMigrator) - Message queues