Merge branch 'master' into fixed-error-with-query-id-printing

cmake/find_mimalloc.cmake

@@ -1,5 +1,5 @@
 if (OS_LINUX AND NOT SANITIZE AND NOT ARCH_ARM AND NOT ARCH_32 AND NOT ARCH_PPC64LE)
-    option (ENABLE_MIMALLOC "Set to FALSE to disable usage of mimalloc for internal ClickHouse caches" ${NOT_UNBUNDLED})
+    option (ENABLE_MIMALLOC "Set to FALSE to disable usage of mimalloc for internal ClickHouse caches" FALSE)
 endif ()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include/mimalloc.h")
@@ -8,6 +8,8 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include/mimalloc.h")
 endif ()
 
 if (ENABLE_MIMALLOC)
+    message (FATAL_ERROR "Mimalloc is not production ready. (Disable with cmake -D ENABLE_MIMALLOC=0). If you want to use mimalloc, you must manually remove this message.")
+
     set (MIMALLOC_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include)
     set (USE_MIMALLOC 1)
     set (MIMALLOC_LIBRARY mimalloc-static)

dbms/src/Columns/ColumnArray.cpp

@@ -178,11 +178,16 @@ StringRef ColumnArray::serializeValueIntoArena(size_t n, Arena & arena, char con
     char * pos = arena.allocContinue(sizeof(array_size), begin);
     memcpy(pos, &array_size, sizeof(array_size));
 
-    size_t values_size = 0;
-    for (size_t i = 0; i < array_size; ++i)
-        values_size += getData().serializeValueIntoArena(offset + i, arena, begin).size;
+    StringRef res(pos, sizeof(array_size));
 
-    return StringRef(begin, sizeof(array_size) + values_size);
+    for (size_t i = 0; i < array_size; ++i)
+    {
+        auto value_ref = getData().serializeValueIntoArena(offset + i, arena, begin);
+        res.data = value_ref.data - res.size;
+        res.size += value_ref.size;
+    }
+
+    return res;
 }
 
 

dbms/src/Columns/ColumnNullable.cpp

@@ -103,12 +103,13 @@ StringRef ColumnNullable::serializeValueIntoArena(size_t n, Arena & arena, char
     auto pos = arena.allocContinue(s, begin);
     memcpy(pos, &arr[n], s);
 
-    size_t nested_size = 0;
+    if (arr[n])
+        return StringRef(pos, s);
 
-    if (arr[n] == 0)
-        nested_size = getNestedColumn().serializeValueIntoArena(n, arena, begin).size;
+    auto nested_ref = getNestedColumn().serializeValueIntoArena(n, arena, begin);
 
-    return StringRef{begin, s + nested_size};
+    /// serializeValueIntoArena may reallocate memory. Have to use ptr from nested_ref.data and move it back.
+    return StringRef(nested_ref.data - s, nested_ref.size + s);
 }
 
 const char * ColumnNullable::deserializeAndInsertFromArena(const char * pos)

dbms/src/Columns/ColumnTuple.cpp

@@ -142,11 +142,15 @@ void ColumnTuple::popBack(size_t n)
 
 StringRef ColumnTuple::serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const
 {
-    size_t values_size = 0;
+    StringRef res(begin, 0);
     for (auto & column : columns)
-        values_size += column->serializeValueIntoArena(n, arena, begin).size;
+    {
+        auto value_ref = column->serializeValueIntoArena(n, arena, begin);
+        res.data = value_ref.data - res.size;
+        res.size += value_ref.size;
+    }
 
-    return StringRef(begin, values_size);
+    return res;
 }
 
 const char * ColumnTuple::deserializeAndInsertFromArena(const char * pos)

dbms/src/Columns/ColumnUnique.h

@@ -300,19 +300,19 @@ StringRef ColumnUnique<ColumnType>::serializeValueIntoArena(size_t n, Arena & ar
 {
     if (is_nullable)
     {
-        const UInt8 null_flag = 1;
-        const UInt8 not_null_flag = 0;
+        static constexpr auto s = sizeof(UInt8);
 
-        auto pos = arena.allocContinue(sizeof(null_flag), begin);
-        auto & flag = (n == getNullValueIndex() ? null_flag : not_null_flag);
-        memcpy(pos, &flag, sizeof(flag));
+        auto pos = arena.allocContinue(s, begin);
+        UInt8 flag = (n == getNullValueIndex() ? 1 : 0);
+        unalignedStore<UInt8>(pos, flag);
 
-        size_t nested_size = 0;
+        if (n == getNullValueIndex())
+            return StringRef(pos, s);
 
-        if (n != getNullValueIndex())
-            nested_size = column_holder->serializeValueIntoArena(n, arena, begin).size;
+        auto nested_ref = column_holder->serializeValueIntoArena(n, arena, begin);
 
-        return StringRef(pos, sizeof(null_flag) + nested_size);
+        /// serializeValueIntoArena may reallocate memory. Have to use ptr from nested_ref.data and move it back.
+        return StringRef(nested_ref.data - s, nested_ref.size + s);
     }
 
     return column_holder->serializeValueIntoArena(n, arena, begin);

dbms/src/Common/MiAllocator.cpp

@@ -5,15 +5,33 @@
 #include "MiAllocator.h"
 #include <mimalloc.h>
 
+#include <Common/Exception.h>
+#include <Common/formatReadable.h>
+#include <IO/WriteHelpers.h>
+
 namespace DB
 {
+namespace ErrorCodes
+{
+    extern const int CANNOT_ALLOCATE_MEMORY;
+}
 
 void * MiAllocator::alloc(size_t size, size_t alignment)
 {
+    void * ptr;
     if (alignment == 0)
-        return mi_malloc(size);
+    {
+        ptr = mi_malloc(size);
+        if (!ptr)
+            DB::throwFromErrno("MiAllocator: Cannot allocate in mimalloc " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
+    }
     else
-        return mi_malloc_aligned(size, alignment);
+    {
+        ptr = mi_malloc_aligned(size, alignment);
+        if (!ptr)
+            DB::throwFromErrno("MiAllocator: Cannot allocate in mimalloc (mi_malloc_aligned) " + formatReadableSizeWithBinarySuffix(size) + " with alignment " + toString(alignment) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
+    }
+    return ptr;
 }
 
 void MiAllocator::free(void * buf, size_t)
@@ -32,10 +50,21 @@ void * MiAllocator::realloc(void * old_ptr, size_t, size_t new_size, size_t alig
         return nullptr;
     }
 
-    if (alignment == 0)
-        return mi_realloc(old_ptr, alignment);
+    void * ptr;
 
-    return mi_realloc_aligned(old_ptr, new_size, alignment);
+    if (alignment == 0)
+    {
+        ptr = mi_realloc(old_ptr, alignment);
+        if (!ptr)
+            DB::throwFromErrno("MiAllocator: Cannot reallocate in mimalloc " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
+    }
+    else
+    {
+        ptr = mi_realloc_aligned(old_ptr, new_size, alignment);
+        if (!ptr)
+            DB::throwFromErrno("MiAllocator: Cannot reallocate in mimalloc (mi_realloc_aligned) " + formatReadableSizeWithBinarySuffix(size) + " with alignment " + toString(alignment) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
+    }
+    return ptr;
 }
 
 }

dbms/src/Common/tests/CMakeLists.txt

@@ -75,3 +75,6 @@ target_link_libraries (cow_compositions PRIVATE clickhouse_common_io)
 
 add_executable (stopwatch stopwatch.cpp)
 target_link_libraries (stopwatch PRIVATE clickhouse_common_io)
+
+add_executable (mi_malloc_test mi_malloc_test.cpp)
+target_link_libraries (mi_malloc_test PRIVATE clickhouse_common_io)

dbms/src/Common/tests/mi_malloc_test.cpp

@@ -0,0 +1,92 @@
+#include <map>
+#include <vector>
+#include <cstdint>
+#include <random>
+#include <stdexcept>
+#include <iostream>
+
+#include <Common/config.h>
+
+//#undef USE_MIMALLOC
+//#define USE_MIMALLOC 0
+
+#if USE_MIMALLOC
+
+#include <mimalloc.h>
+#define malloc mi_malloc
+#define free mi_free
+
+#else
+
+#include <stdlib.h>
+
+#endif
+
+
+size_t total_size{0};
+
+struct Allocation
+{
+    void * ptr = nullptr;
+    size_t size = 0;
+
+    Allocation() {}
+
+    Allocation(size_t size)
+        : size(size)
+    {
+        ptr = malloc(size);
+        if (!ptr)
+            throw std::runtime_error("Cannot allocate memory");
+        total_size += size;
+    }
+
+    ~Allocation()
+    {
+        if (ptr)
+        {
+            free(ptr);
+            total_size -= size;
+        }
+        ptr = nullptr;
+    }
+
+    Allocation(const Allocation &) = delete;
+
+    Allocation(Allocation && rhs)
+    {
+        ptr = rhs.ptr;
+        size = rhs.size;
+        rhs.ptr = nullptr;
+        rhs.size = 0;
+    }
+};
+
+
+int main(int, char **)
+{
+    std::vector<Allocation> allocations;
+
+    constexpr size_t limit = 100000000;
+    constexpr size_t min_alloc_size = 65536;
+    constexpr size_t max_alloc_size = 10000000;
+
+    std::mt19937 rng;
+    auto distribution = std::uniform_int_distribution(min_alloc_size, max_alloc_size);
+
+    size_t total_allocations = 0;
+
+    while (true)
+    {
+        size_t size = distribution(rng);
+
+        while (total_size + size > limit)
+            allocations.pop_back();
+
+        allocations.emplace_back(size);
+
+        ++total_allocations;
+        if (total_allocations % (1ULL << 20) == 0)
+            std::cerr << "Total allocations: " << total_allocations << "\n";
+    }
+}

dbms/tests/performance/uniq.xml

@@ -46,8 +46,6 @@
                <value>uniqCombined(16)</value>
                <value>uniqCombined(17)</value>
                <value>uniqCombined(18)</value>
-               <value>uniqCombined(19)</value>
-               <value>uniqCombined(20)</value>
                <value>uniqUpTo(3)</value>
                <value>uniqUpTo(5)</value>
                <value>uniqUpTo(10)</value>

docs/en/interfaces/third-party/integrations.md

@@ -14,6 +14,7 @@
         - [clickhousedb_fdw](https://github.com/Percona-Lab/clickhousedb_fdw)
         - [infi.clickhouse_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (uses [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm))
         - [pg2ch](https://github.com/mkabilov/pg2ch)
+        - [clickhouse_fdw](https://github.com/adjust/clickhouse_fdw)
     - [MSSQL](https://en.wikipedia.org/wiki/Microsoft_SQL_Server)
         - [ClickHouseMigrator](https://github.com/zlzforever/ClickHouseMigrator)
 - Message queues