mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-27 18:12:02 +00:00
Merge pull request #57074 from ClickHouse/less-allocation-in-arenas
Avoid excessive allocation in Arena
This commit is contained in:
commit
d72bc854d1
@ -11,7 +11,6 @@
|
|||||||
#include <Common/memcmpSmall.h>
|
#include <Common/memcmpSmall.h>
|
||||||
#include <Common/assert_cast.h>
|
#include <Common/assert_cast.h>
|
||||||
#include <Core/Field.h>
|
#include <Core/Field.h>
|
||||||
#include <Common/Arena.h>
|
|
||||||
|
|
||||||
|
|
||||||
class Collator;
|
class Collator;
|
||||||
@ -20,6 +19,8 @@ class Collator;
|
|||||||
namespace DB
|
namespace DB
|
||||||
{
|
{
|
||||||
|
|
||||||
|
class Arena;
|
||||||
|
|
||||||
/** Column for String values.
|
/** Column for String values.
|
||||||
*/
|
*/
|
||||||
class ColumnString final : public COWHelper<IColumn, ColumnString>
|
class ColumnString final : public COWHelper<IColumn, ColumnString>
|
||||||
|
137
src/Columns/IColumnDummy.cpp
Normal file
137
src/Columns/IColumnDummy.cpp
Normal file
@ -0,0 +1,137 @@
|
|||||||
|
#include <Common/Arena.h>
|
||||||
|
#include <Columns/IColumnDummy.h>
|
||||||
|
#include <Columns/ColumnsCommon.h>
|
||||||
|
|
||||||
|
|
||||||
|
namespace DB
|
||||||
|
{
|
||||||
|
|
||||||
|
namespace ErrorCodes
|
||||||
|
{
|
||||||
|
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
|
||||||
|
extern const int NOT_IMPLEMENTED;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
Field IColumnDummy::operator[](size_t) const
|
||||||
|
{
|
||||||
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get value from {}", getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
void IColumnDummy::get(size_t, Field &) const
|
||||||
|
{
|
||||||
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get value from {}", getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
void IColumnDummy::insert(const Field &)
|
||||||
|
{
|
||||||
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot insert element into {}", getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
bool IColumnDummy::isDefaultAt(size_t) const
|
||||||
|
{
|
||||||
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "isDefaultAt is not implemented for {}", getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
StringRef IColumnDummy::serializeValueIntoArena(size_t /*n*/, Arena & arena, char const *& begin, const UInt8 *) const
|
||||||
|
{
|
||||||
|
/// Has to put one useless byte into Arena, because serialization into zero number of bytes is ambiguous.
|
||||||
|
char * res = arena.allocContinue(1, begin);
|
||||||
|
*res = 0;
|
||||||
|
return { res, 1 };
|
||||||
|
}
|
||||||
|
|
||||||
|
const char * IColumnDummy::deserializeAndInsertFromArena(const char * pos)
|
||||||
|
{
|
||||||
|
++s;
|
||||||
|
return pos + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char * IColumnDummy::skipSerializedInArena(const char * pos) const
|
||||||
|
{
|
||||||
|
return pos;
|
||||||
|
}
|
||||||
|
|
||||||
|
ColumnPtr IColumnDummy::filter(const Filter & filt, ssize_t /*result_size_hint*/) const
|
||||||
|
{
|
||||||
|
size_t bytes = countBytesInFilter(filt);
|
||||||
|
return cloneDummy(bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
void IColumnDummy::expand(const IColumn::Filter & mask, bool inverted)
|
||||||
|
{
|
||||||
|
size_t bytes = countBytesInFilter(mask);
|
||||||
|
if (inverted)
|
||||||
|
bytes = mask.size() - bytes;
|
||||||
|
s = bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
ColumnPtr IColumnDummy::permute(const Permutation & perm, size_t limit) const
|
||||||
|
{
|
||||||
|
if (s != perm.size())
|
||||||
|
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of permutation doesn't match size of column.");
|
||||||
|
|
||||||
|
return cloneDummy(limit ? std::min(s, limit) : s);
|
||||||
|
}
|
||||||
|
|
||||||
|
ColumnPtr IColumnDummy::index(const IColumn & indexes, size_t limit) const
|
||||||
|
{
|
||||||
|
if (indexes.size() < limit)
|
||||||
|
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of indexes is less than required.");
|
||||||
|
|
||||||
|
return cloneDummy(limit ? limit : s);
|
||||||
|
}
|
||||||
|
|
||||||
|
void IColumnDummy::getPermutation(IColumn::PermutationSortDirection /*direction*/, IColumn::PermutationSortStability /*stability*/,
|
||||||
|
size_t /*limit*/, int /*nan_direction_hint*/, Permutation & res) const
|
||||||
|
{
|
||||||
|
res.resize(s);
|
||||||
|
for (size_t i = 0; i < s; ++i)
|
||||||
|
res[i] = i;
|
||||||
|
}
|
||||||
|
|
||||||
|
ColumnPtr IColumnDummy::replicate(const Offsets & offsets) const
|
||||||
|
{
|
||||||
|
if (s != offsets.size())
|
||||||
|
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of offsets doesn't match size of column.");
|
||||||
|
|
||||||
|
return cloneDummy(offsets.back());
|
||||||
|
}
|
||||||
|
|
||||||
|
MutableColumns IColumnDummy::scatter(ColumnIndex num_columns, const Selector & selector) const
|
||||||
|
{
|
||||||
|
if (s != selector.size())
|
||||||
|
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of selector doesn't match size of column.");
|
||||||
|
|
||||||
|
std::vector<size_t> counts(num_columns);
|
||||||
|
for (auto idx : selector)
|
||||||
|
++counts[idx];
|
||||||
|
|
||||||
|
MutableColumns res(num_columns);
|
||||||
|
for (size_t i = 0; i < num_columns; ++i)
|
||||||
|
res[i] = cloneResized(counts[i]);
|
||||||
|
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
double IColumnDummy::getRatioOfDefaultRows(double) const
|
||||||
|
{
|
||||||
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getRatioOfDefaultRows is not supported for {}", getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
UInt64 IColumnDummy::getNumberOfDefaultRows() const
|
||||||
|
{
|
||||||
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getNumberOfDefaultRows is not supported for {}", getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
void IColumnDummy::getIndicesOfNonDefaultRows(Offsets &, size_t, size_t) const
|
||||||
|
{
|
||||||
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getIndicesOfNonDefaultRows is not supported for {}", getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
void IColumnDummy::gather(ColumnGathererStream &)
|
||||||
|
{
|
||||||
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method gather is not supported for {}", getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -1,21 +1,12 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <Common/Arena.h>
|
|
||||||
#include <Common/PODArray.h>
|
|
||||||
#include <Columns/IColumn.h>
|
#include <Columns/IColumn.h>
|
||||||
#include <Columns/ColumnsCommon.h>
|
|
||||||
#include <Core/Field.h>
|
|
||||||
|
|
||||||
|
|
||||||
namespace DB
|
namespace DB
|
||||||
{
|
{
|
||||||
|
|
||||||
namespace ErrorCodes
|
class Arena;
|
||||||
{
|
|
||||||
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
|
|
||||||
extern const int NOT_IMPLEMENTED;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/** Base class for columns-constants that contain a value that is not in the `Field`.
|
/** Base class for columns-constants that contain a value that is not in the `Field`.
|
||||||
* Not a full-fledged column and is used in a special way.
|
* Not a full-fledged column and is used in a special way.
|
||||||
@ -42,10 +33,10 @@ public:
|
|||||||
|
|
||||||
bool hasEqualValues() const override { return true; }
|
bool hasEqualValues() const override { return true; }
|
||||||
|
|
||||||
Field operator[](size_t) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get value from {}", getName()); }
|
Field operator[](size_t) const override;
|
||||||
void get(size_t, Field &) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get value from {}", getName()); }
|
void get(size_t, Field &) const override;
|
||||||
void insert(const Field &) override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot insert element into {}", getName()); }
|
void insert(const Field &) override;
|
||||||
bool isDefaultAt(size_t) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "isDefaultAt is not implemented for {}", getName()); }
|
bool isDefaultAt(size_t) const override;
|
||||||
|
|
||||||
StringRef getDataAt(size_t) const override
|
StringRef getDataAt(size_t) const override
|
||||||
{
|
{
|
||||||
@ -57,24 +48,9 @@ public:
|
|||||||
++s;
|
++s;
|
||||||
}
|
}
|
||||||
|
|
||||||
StringRef serializeValueIntoArena(size_t /*n*/, Arena & arena, char const *& begin, const UInt8 *) const override
|
StringRef serializeValueIntoArena(size_t /*n*/, Arena & arena, char const *& begin, const UInt8 *) const override;
|
||||||
{
|
const char * deserializeAndInsertFromArena(const char * pos) override;
|
||||||
/// Has to put one useless byte into Arena, because serialization into zero number of bytes is ambiguous.
|
const char * skipSerializedInArena(const char * pos) const override;
|
||||||
char * res = arena.allocContinue(1, begin);
|
|
||||||
*res = 0;
|
|
||||||
return { res, 1 };
|
|
||||||
}
|
|
||||||
|
|
||||||
const char * deserializeAndInsertFromArena(const char * pos) override
|
|
||||||
{
|
|
||||||
++s;
|
|
||||||
return pos + 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
const char * skipSerializedInArena(const char * pos) const override
|
|
||||||
{
|
|
||||||
return pos;
|
|
||||||
}
|
|
||||||
|
|
||||||
void updateHashWithValue(size_t /*n*/, SipHash & /*hash*/) const override
|
void updateHashWithValue(size_t /*n*/, SipHash & /*hash*/) const override
|
||||||
{
|
{
|
||||||
@ -98,90 +74,30 @@ public:
|
|||||||
s += length;
|
s += length;
|
||||||
}
|
}
|
||||||
|
|
||||||
ColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override
|
ColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override;
|
||||||
{
|
|
||||||
size_t bytes = countBytesInFilter(filt);
|
|
||||||
return cloneDummy(bytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
void expand(const IColumn::Filter & mask, bool inverted) override
|
void expand(const IColumn::Filter & mask, bool inverted) override;
|
||||||
{
|
|
||||||
size_t bytes = countBytesInFilter(mask);
|
|
||||||
if (inverted)
|
|
||||||
bytes = mask.size() - bytes;
|
|
||||||
s = bytes;
|
|
||||||
}
|
|
||||||
|
|
||||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override
|
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||||
{
|
|
||||||
if (s != perm.size())
|
|
||||||
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of permutation doesn't match size of column.");
|
|
||||||
|
|
||||||
return cloneDummy(limit ? std::min(s, limit) : s);
|
ColumnPtr index(const IColumn & indexes, size_t limit) const override;
|
||||||
}
|
|
||||||
|
|
||||||
ColumnPtr index(const IColumn & indexes, size_t limit) const override
|
|
||||||
{
|
|
||||||
if (indexes.size() < limit)
|
|
||||||
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of indexes is less than required.");
|
|
||||||
|
|
||||||
return cloneDummy(limit ? limit : s);
|
|
||||||
}
|
|
||||||
|
|
||||||
void getPermutation(IColumn::PermutationSortDirection /*direction*/, IColumn::PermutationSortStability /*stability*/,
|
void getPermutation(IColumn::PermutationSortDirection /*direction*/, IColumn::PermutationSortStability /*stability*/,
|
||||||
size_t /*limit*/, int /*nan_direction_hint*/, Permutation & res) const override
|
size_t /*limit*/, int /*nan_direction_hint*/, Permutation & res) const override;
|
||||||
{
|
|
||||||
res.resize(s);
|
|
||||||
for (size_t i = 0; i < s; ++i)
|
|
||||||
res[i] = i;
|
|
||||||
}
|
|
||||||
|
|
||||||
void updatePermutation(IColumn::PermutationSortDirection /*direction*/, IColumn::PermutationSortStability /*stability*/,
|
void updatePermutation(IColumn::PermutationSortDirection /*direction*/, IColumn::PermutationSortStability /*stability*/,
|
||||||
size_t, int, Permutation &, EqualRanges&) const override {}
|
size_t, int, Permutation &, EqualRanges&) const override
|
||||||
|
|
||||||
ColumnPtr replicate(const Offsets & offsets) const override
|
|
||||||
{
|
{
|
||||||
if (s != offsets.size())
|
|
||||||
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of offsets doesn't match size of column.");
|
|
||||||
|
|
||||||
return cloneDummy(offsets.back());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
|
ColumnPtr replicate(const Offsets & offsets) const override;
|
||||||
{
|
|
||||||
if (s != selector.size())
|
|
||||||
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of selector doesn't match size of column.");
|
|
||||||
|
|
||||||
std::vector<size_t> counts(num_columns);
|
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
|
||||||
for (auto idx : selector)
|
|
||||||
++counts[idx];
|
|
||||||
|
|
||||||
MutableColumns res(num_columns);
|
double getRatioOfDefaultRows(double) const override;
|
||||||
for (size_t i = 0; i < num_columns; ++i)
|
UInt64 getNumberOfDefaultRows() const override;
|
||||||
res[i] = cloneResized(counts[i]);
|
void getIndicesOfNonDefaultRows(Offsets &, size_t, size_t) const override;
|
||||||
|
void gather(ColumnGathererStream &) override;
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
double getRatioOfDefaultRows(double) const override
|
|
||||||
{
|
|
||||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getRatioOfDefaultRows is not supported for {}", getName());
|
|
||||||
}
|
|
||||||
|
|
||||||
UInt64 getNumberOfDefaultRows() const override
|
|
||||||
{
|
|
||||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getNumberOfDefaultRows is not supported for {}", getName());
|
|
||||||
}
|
|
||||||
|
|
||||||
void getIndicesOfNonDefaultRows(Offsets &, size_t, size_t) const override
|
|
||||||
{
|
|
||||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getIndicesOfNonDefaultRows is not supported for {}", getName());
|
|
||||||
}
|
|
||||||
|
|
||||||
void gather(ColumnGathererStream &) override
|
|
||||||
{
|
|
||||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method gather is not supported for {}", getName());
|
|
||||||
}
|
|
||||||
|
|
||||||
void getExtremes(Field &, Field &) const override
|
void getExtremes(Field &, Field &) const override
|
||||||
{
|
{
|
||||||
|
@ -5,13 +5,14 @@
|
|||||||
#include <vector>
|
#include <vector>
|
||||||
#include <boost/noncopyable.hpp>
|
#include <boost/noncopyable.hpp>
|
||||||
#include <Core/Defines.h>
|
#include <Core/Defines.h>
|
||||||
#if __has_include(<sanitizer/asan_interface.h>) && defined(ADDRESS_SANITIZER)
|
|
||||||
# include <sanitizer/asan_interface.h>
|
|
||||||
#endif
|
|
||||||
#include <Common/memcpySmall.h>
|
#include <Common/memcpySmall.h>
|
||||||
#include <Common/ProfileEvents.h>
|
#include <Common/ProfileEvents.h>
|
||||||
#include <Common/Allocator.h>
|
#include <Common/Allocator.h>
|
||||||
|
|
||||||
|
#if __has_include(<sanitizer/asan_interface.h>) && defined(ADDRESS_SANITIZER)
|
||||||
|
# include <sanitizer/asan_interface.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
namespace ProfileEvents
|
namespace ProfileEvents
|
||||||
{
|
{
|
||||||
@ -39,13 +40,36 @@ private:
|
|||||||
/// Contiguous MemoryChunk of memory and pointer to free space inside it. Member of single-linked list.
|
/// Contiguous MemoryChunk of memory and pointer to free space inside it. Member of single-linked list.
|
||||||
struct alignas(16) MemoryChunk : private Allocator<false> /// empty base optimization
|
struct alignas(16) MemoryChunk : private Allocator<false> /// empty base optimization
|
||||||
{
|
{
|
||||||
char * begin;
|
char * begin = nullptr;
|
||||||
char * pos;
|
char * pos = nullptr;
|
||||||
char * end; /// does not include padding.
|
char * end = nullptr; /// does not include padding.
|
||||||
|
|
||||||
MemoryChunk * prev;
|
std::unique_ptr<MemoryChunk> prev;
|
||||||
|
|
||||||
MemoryChunk(size_t size_, MemoryChunk * prev_)
|
MemoryChunk()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
void swap(MemoryChunk & other)
|
||||||
|
{
|
||||||
|
std::swap(begin, other.begin);
|
||||||
|
std::swap(pos, other.pos);
|
||||||
|
std::swap(end, other.end);
|
||||||
|
prev.swap(other.prev);
|
||||||
|
}
|
||||||
|
|
||||||
|
MemoryChunk(MemoryChunk && other)
|
||||||
|
{
|
||||||
|
*this = std::move(other);
|
||||||
|
}
|
||||||
|
|
||||||
|
MemoryChunk & operator=(MemoryChunk && other)
|
||||||
|
{
|
||||||
|
swap(other);
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
MemoryChunk(size_t size_)
|
||||||
{
|
{
|
||||||
ProfileEvents::increment(ProfileEvents::ArenaAllocChunks);
|
ProfileEvents::increment(ProfileEvents::ArenaAllocChunks);
|
||||||
ProfileEvents::increment(ProfileEvents::ArenaAllocBytes, size_);
|
ProfileEvents::increment(ProfileEvents::ArenaAllocBytes, size_);
|
||||||
@ -53,13 +77,15 @@ private:
|
|||||||
begin = reinterpret_cast<char *>(Allocator<false>::alloc(size_));
|
begin = reinterpret_cast<char *>(Allocator<false>::alloc(size_));
|
||||||
pos = begin;
|
pos = begin;
|
||||||
end = begin + size_ - pad_right;
|
end = begin + size_ - pad_right;
|
||||||
prev = prev_;
|
|
||||||
|
|
||||||
ASAN_POISON_MEMORY_REGION(begin, size_);
|
ASAN_POISON_MEMORY_REGION(begin, size_);
|
||||||
}
|
}
|
||||||
|
|
||||||
~MemoryChunk()
|
~MemoryChunk()
|
||||||
{
|
{
|
||||||
|
if (empty())
|
||||||
|
return;
|
||||||
|
|
||||||
/// We must unpoison the memory before returning to the allocator,
|
/// We must unpoison the memory before returning to the allocator,
|
||||||
/// because the allocator might not have asan integration, and the
|
/// because the allocator might not have asan integration, and the
|
||||||
/// memory would stay poisoned forever. If the allocator supports
|
/// memory would stay poisoned forever. If the allocator supports
|
||||||
@ -67,21 +93,21 @@ private:
|
|||||||
ASAN_UNPOISON_MEMORY_REGION(begin, size());
|
ASAN_UNPOISON_MEMORY_REGION(begin, size());
|
||||||
|
|
||||||
Allocator<false>::free(begin, size());
|
Allocator<false>::free(begin, size());
|
||||||
|
|
||||||
delete prev;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool empty() const { return begin == end;}
|
||||||
size_t size() const { return end + pad_right - begin; }
|
size_t size() const { return end + pad_right - begin; }
|
||||||
size_t remaining() const { return end - pos; }
|
size_t remaining() const { return end - pos; }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
size_t initial_size;
|
||||||
size_t growth_factor;
|
size_t growth_factor;
|
||||||
size_t linear_growth_threshold;
|
size_t linear_growth_threshold;
|
||||||
|
|
||||||
/// Last contiguous MemoryChunk of memory.
|
/// Last contiguous MemoryChunk of memory.
|
||||||
MemoryChunk * head;
|
MemoryChunk head;
|
||||||
size_t allocated_bytes;
|
size_t allocated_bytes = 0;
|
||||||
size_t used_bytes;
|
size_t used_bytes = 0;
|
||||||
size_t page_size;
|
size_t page_size;
|
||||||
|
|
||||||
static size_t roundUpToPageSize(size_t s, size_t page_size)
|
static size_t roundUpToPageSize(size_t s, size_t page_size)
|
||||||
@ -95,9 +121,13 @@ private:
|
|||||||
{
|
{
|
||||||
size_t size_after_grow = 0;
|
size_t size_after_grow = 0;
|
||||||
|
|
||||||
if (head->size() < linear_growth_threshold)
|
if (head.empty())
|
||||||
{
|
{
|
||||||
size_after_grow = std::max(min_next_size, head->size() * growth_factor);
|
size_after_grow = std::max(min_next_size, initial_size);
|
||||||
|
}
|
||||||
|
else if (head.size() < linear_growth_threshold)
|
||||||
|
{
|
||||||
|
size_after_grow = std::max(min_next_size, head.size() * growth_factor);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -119,8 +149,18 @@ private:
|
|||||||
/// Add next contiguous MemoryChunk of memory with size not less than specified.
|
/// Add next contiguous MemoryChunk of memory with size not less than specified.
|
||||||
void NO_INLINE addMemoryChunk(size_t min_size)
|
void NO_INLINE addMemoryChunk(size_t min_size)
|
||||||
{
|
{
|
||||||
head = new MemoryChunk(nextSize(min_size + pad_right), head);
|
size_t next_size = nextSize(min_size + pad_right);
|
||||||
allocated_bytes += head->size();
|
if (head.empty())
|
||||||
|
{
|
||||||
|
head = MemoryChunk(next_size);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
auto chunk = std::make_unique<MemoryChunk>(next_size);
|
||||||
|
head.swap(*chunk);
|
||||||
|
head.prev = std::move(chunk);
|
||||||
|
}
|
||||||
|
allocated_bytes += head.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
friend class ArenaAllocator;
|
friend class ArenaAllocator;
|
||||||
@ -128,29 +168,23 @@ private:
|
|||||||
|
|
||||||
public:
|
public:
|
||||||
explicit Arena(size_t initial_size_ = 4096, size_t growth_factor_ = 2, size_t linear_growth_threshold_ = 128 * 1024 * 1024)
|
explicit Arena(size_t initial_size_ = 4096, size_t growth_factor_ = 2, size_t linear_growth_threshold_ = 128 * 1024 * 1024)
|
||||||
: growth_factor(growth_factor_)
|
: initial_size(initial_size_)
|
||||||
|
, growth_factor(growth_factor_)
|
||||||
, linear_growth_threshold(linear_growth_threshold_)
|
, linear_growth_threshold(linear_growth_threshold_)
|
||||||
, head(new MemoryChunk(initial_size_, nullptr))
|
|
||||||
, allocated_bytes(head->size())
|
|
||||||
, used_bytes(0)
|
|
||||||
, page_size(static_cast<size_t>(::getPageSize()))
|
, page_size(static_cast<size_t>(::getPageSize()))
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
~Arena()
|
|
||||||
{
|
|
||||||
delete head;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Get piece of memory, without alignment.
|
/// Get piece of memory, without alignment.
|
||||||
|
/// Note: we expect it will return a non-nullptr even if the size is zero.
|
||||||
char * alloc(size_t size)
|
char * alloc(size_t size)
|
||||||
{
|
{
|
||||||
used_bytes += size;
|
used_bytes += size;
|
||||||
if (unlikely(static_cast<std::ptrdiff_t>(size) > head->end - head->pos))
|
if (unlikely(head.empty() || static_cast<std::ptrdiff_t>(size) > head.end - head.pos))
|
||||||
addMemoryChunk(size);
|
addMemoryChunk(size);
|
||||||
|
|
||||||
char * res = head->pos;
|
char * res = head.pos;
|
||||||
head->pos += size;
|
head.pos += size;
|
||||||
ASAN_UNPOISON_MEMORY_REGION(res, size + pad_right);
|
ASAN_UNPOISON_MEMORY_REGION(res, size + pad_right);
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
@ -161,14 +195,14 @@ public:
|
|||||||
used_bytes += size;
|
used_bytes += size;
|
||||||
do
|
do
|
||||||
{
|
{
|
||||||
void * head_pos = head->pos;
|
void * head_pos = head.pos;
|
||||||
size_t space = head->end - head->pos;
|
size_t space = head.end - head.pos;
|
||||||
|
|
||||||
auto * res = static_cast<char *>(std::align(alignment, size, head_pos, space));
|
auto * res = static_cast<char *>(std::align(alignment, size, head_pos, space));
|
||||||
if (res)
|
if (res)
|
||||||
{
|
{
|
||||||
head->pos = static_cast<char *>(head_pos);
|
head.pos = static_cast<char *>(head_pos);
|
||||||
head->pos += size;
|
head.pos += size;
|
||||||
ASAN_UNPOISON_MEMORY_REGION(res, size + pad_right);
|
ASAN_UNPOISON_MEMORY_REGION(res, size + pad_right);
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
@ -191,9 +225,9 @@ public:
|
|||||||
void * rollback(size_t size)
|
void * rollback(size_t size)
|
||||||
{
|
{
|
||||||
used_bytes -= size;
|
used_bytes -= size;
|
||||||
head->pos -= size;
|
head.pos -= size;
|
||||||
ASAN_POISON_MEMORY_REGION(head->pos, size + pad_right);
|
ASAN_POISON_MEMORY_REGION(head.pos, size + pad_right);
|
||||||
return head->pos;
|
return head.pos;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Begin or expand a contiguous range of memory.
|
/** Begin or expand a contiguous range of memory.
|
||||||
@ -234,10 +268,10 @@ public:
|
|||||||
// This method only works for extending the last allocation. For lack of
|
// This method only works for extending the last allocation. For lack of
|
||||||
// original size, check a weaker condition: that 'begin' is at least in
|
// original size, check a weaker condition: that 'begin' is at least in
|
||||||
// the current MemoryChunk.
|
// the current MemoryChunk.
|
||||||
assert(range_start >= head->begin);
|
assert(range_start >= head.begin);
|
||||||
assert(range_start < head->end);
|
assert(range_start < head.end);
|
||||||
|
|
||||||
if (head->pos + additional_bytes <= head->end)
|
if (head.pos + additional_bytes <= head.end)
|
||||||
{
|
{
|
||||||
// The new size fits into the last MemoryChunk, so just alloc the
|
// The new size fits into the last MemoryChunk, so just alloc the
|
||||||
// additional size. We can alloc without alignment here, because it
|
// additional size. We can alloc without alignment here, because it
|
||||||
@ -254,7 +288,7 @@ public:
|
|||||||
// solved not by complicating this method, but by rethinking the
|
// solved not by complicating this method, but by rethinking the
|
||||||
// approach to memory management for aggregate function states, so that
|
// approach to memory management for aggregate function states, so that
|
||||||
// we can provide a proper realloc().
|
// we can provide a proper realloc().
|
||||||
const size_t existing_bytes = head->pos - range_start;
|
const size_t existing_bytes = head.pos - range_start;
|
||||||
const size_t new_bytes = existing_bytes + additional_bytes;
|
const size_t new_bytes = existing_bytes + additional_bytes;
|
||||||
const char * old_range = range_start;
|
const char * old_range = range_start;
|
||||||
|
|
||||||
@ -317,12 +351,11 @@ public:
|
|||||||
/// yourself having to use this method, probably you're doing something wrong.
|
/// yourself having to use this method, probably you're doing something wrong.
|
||||||
size_t remainingSpaceInCurrentMemoryChunk() const
|
size_t remainingSpaceInCurrentMemoryChunk() const
|
||||||
{
|
{
|
||||||
return head->remaining();
|
return head.remaining();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
using ArenaPtr = std::shared_ptr<Arena>;
|
using ArenaPtr = std::shared_ptr<Arena>;
|
||||||
using Arenas = std::vector<ArenaPtr>;
|
using Arenas = std::vector<ArenaPtr>;
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -20,7 +20,7 @@ public:
|
|||||||
char const * data = reinterpret_cast<char *>(buf);
|
char const * data = reinterpret_cast<char *>(buf);
|
||||||
|
|
||||||
// Invariant should be maintained: new_size > old_size
|
// Invariant should be maintained: new_size > old_size
|
||||||
if (data + old_size == arena->head->pos)
|
if (data + old_size == arena->head.pos)
|
||||||
{
|
{
|
||||||
// Consecutive optimization
|
// Consecutive optimization
|
||||||
arena->allocContinue(new_size - old_size, data);
|
arena->allocContinue(new_size - old_size, data);
|
||||||
@ -59,7 +59,7 @@ public:
|
|||||||
{
|
{
|
||||||
char const * data = reinterpret_cast<char *>(buf);
|
char const * data = reinterpret_cast<char *>(buf);
|
||||||
|
|
||||||
if (data + old_size == arena->head->pos)
|
if (data + old_size == arena->head.pos)
|
||||||
{
|
{
|
||||||
arena->allocContinue(new_size - old_size, data, alignment);
|
arena->allocContinue(new_size - old_size, data, alignment);
|
||||||
return reinterpret_cast<void *>(const_cast<char *>(data));
|
return reinterpret_cast<void *>(const_cast<char *>(data));
|
||||||
|
@ -11,11 +11,9 @@
|
|||||||
#include <DataTypes/DataTypeAggregateFunction.h>
|
#include <DataTypes/DataTypeAggregateFunction.h>
|
||||||
#include <DataTypes/DataTypeNullable.h>
|
#include <DataTypes/DataTypeNullable.h>
|
||||||
#include <DataTypes/DataTypeLowCardinality.h>
|
#include <DataTypes/DataTypeLowCardinality.h>
|
||||||
#include <Columns/ColumnArray.h>
|
|
||||||
#include <Columns/ColumnTuple.h>
|
#include <Columns/ColumnTuple.h>
|
||||||
#include <Columns/ColumnSparse.h>
|
#include <Columns/ColumnSparse.h>
|
||||||
#include <Formats/NativeWriter.h>
|
#include <Formats/NativeWriter.h>
|
||||||
#include <IO/WriteBufferFromFile.h>
|
|
||||||
#include <Compression/CompressedWriteBuffer.h>
|
#include <Compression/CompressedWriteBuffer.h>
|
||||||
#include <Interpreters/Aggregator.h>
|
#include <Interpreters/Aggregator.h>
|
||||||
#include <AggregateFunctions/Combinators/AggregateFunctionArray.h>
|
#include <AggregateFunctions/Combinators/AggregateFunctionArray.h>
|
||||||
@ -23,7 +21,6 @@
|
|||||||
#include <IO/Operators.h>
|
#include <IO/Operators.h>
|
||||||
#include <Interpreters/JIT/compileFunction.h>
|
#include <Interpreters/JIT/compileFunction.h>
|
||||||
#include <Interpreters/JIT/CompiledExpressionCache.h>
|
#include <Interpreters/JIT/CompiledExpressionCache.h>
|
||||||
#include <Core/ProtocolDefines.h>
|
|
||||||
#include <Disks/TemporaryFileOnDisk.h>
|
#include <Disks/TemporaryFileOnDisk.h>
|
||||||
#include <Interpreters/TemporaryDataOnDisk.h>
|
#include <Interpreters/TemporaryDataOnDisk.h>
|
||||||
#include <Common/Stopwatch.h>
|
#include <Common/Stopwatch.h>
|
||||||
@ -37,13 +34,13 @@
|
|||||||
#include <Common/typeid_cast.h>
|
#include <Common/typeid_cast.h>
|
||||||
#include <Common/assert_cast.h>
|
#include <Common/assert_cast.h>
|
||||||
#include <Common/JSONBuilder.h>
|
#include <Common/JSONBuilder.h>
|
||||||
#include <Common/filesystemHelpers.h>
|
|
||||||
#include <Common/scope_guard_safe.h>
|
#include <Common/scope_guard_safe.h>
|
||||||
|
|
||||||
#include <Parsers/ASTSelectQuery.h>
|
#include <Parsers/ASTSelectQuery.h>
|
||||||
|
|
||||||
#include <Interpreters/AggregationUtils.h>
|
#include <Interpreters/AggregationUtils.h>
|
||||||
|
|
||||||
|
|
||||||
namespace ProfileEvents
|
namespace ProfileEvents
|
||||||
{
|
{
|
||||||
extern const Event ExternalAggregationWritePart;
|
extern const Event ExternalAggregationWritePart;
|
||||||
@ -1123,7 +1120,9 @@ void NO_INLINE Aggregator::executeImplBatch(
|
|||||||
return;
|
return;
|
||||||
|
|
||||||
/// For all rows.
|
/// For all rows.
|
||||||
AggregateDataPtr place = aggregates_pool->alloc(0);
|
|
||||||
|
/// This pointer is unused, but the logic will compare it for nullptr to check if the cell is set.
|
||||||
|
AggregateDataPtr place = reinterpret_cast<AggregateDataPtr>(0x1);
|
||||||
if (all_keys_are_const)
|
if (all_keys_are_const)
|
||||||
{
|
{
|
||||||
state.emplaceKey(method.data, 0, *aggregates_pool).setMapped(place);
|
state.emplaceKey(method.data, 0, *aggregates_pool).setMapped(place);
|
||||||
|
@ -5,11 +5,8 @@
|
|||||||
#include <Interpreters/Context.h>
|
#include <Interpreters/Context.h>
|
||||||
#include <Interpreters/ExpressionAnalyzer.h>
|
#include <Interpreters/ExpressionAnalyzer.h>
|
||||||
#include <Interpreters/TreeRewriter.h>
|
#include <Interpreters/TreeRewriter.h>
|
||||||
#include <Interpreters/evaluateConstantExpression.h>
|
|
||||||
|
|
||||||
#include <Parsers/ASTFunction.h>
|
#include <Parsers/ASTFunction.h>
|
||||||
#include <Parsers/ASTInsertQuery.h>
|
|
||||||
#include <Parsers/ASTLiteral.h>
|
|
||||||
|
|
||||||
#include <Processors/ISource.h>
|
#include <Processors/ISource.h>
|
||||||
|
|
||||||
|
@ -49,5 +49,5 @@ Check total_bytes/total_rows for Set
|
|||||||
2048 50
|
2048 50
|
||||||
2048 100
|
2048 100
|
||||||
Check total_bytes/total_rows for Join
|
Check total_bytes/total_rows for Join
|
||||||
10240 50
|
1 50
|
||||||
10240 100
|
1 100
|
||||||
|
@ -134,7 +134,7 @@ DROP TABLE check_system_tables;
|
|||||||
|
|
||||||
SELECT 'Check total_bytes/total_rows for Join';
|
SELECT 'Check total_bytes/total_rows for Join';
|
||||||
CREATE TABLE check_system_tables Engine=Join(ANY, LEFT, number) AS SELECT * FROM numbers(50);
|
CREATE TABLE check_system_tables Engine=Join(ANY, LEFT, number) AS SELECT * FROM numbers(50);
|
||||||
SELECT total_bytes, total_rows FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase();
|
SELECT total_bytes BETWEEN 5000 AND 15000, total_rows FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase();
|
||||||
INSERT INTO check_system_tables SELECT number+50 FROM numbers(50);
|
INSERT INTO check_system_tables SELECT number+50 FROM numbers(50);
|
||||||
SELECT total_bytes, total_rows FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase();
|
SELECT total_bytes BETWEEN 5000 AND 15000, total_rows FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase();
|
||||||
DROP TABLE check_system_tables;
|
DROP TABLE check_system_tables;
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
test_dictionary_hashed 1000000 0.4768 33558760
|
test_dictionary_hashed 1000000 0.4768 34000000
|
||||||
test_dictionary_hashed_load_factor 1000000 0.9537 16781544
|
test_dictionary_hashed_load_factor 1000000 0.9537 17000000
|
||||||
test_dictionary_sparse_hashed 1000000 0.4768 20975848
|
test_dictionary_sparse_hashed 1000000 0.4768 21000000
|
||||||
test_dictionary_sparse_hashed_load_factor 1000000 0.9537 10490088
|
test_dictionary_sparse_hashed_load_factor 1000000 0.9537 10000000
|
||||||
|
@ -31,7 +31,7 @@ LIFETIME(0);
|
|||||||
|
|
||||||
SYSTEM RELOAD DICTIONARY test_dictionary_{{layout}};
|
SYSTEM RELOAD DICTIONARY test_dictionary_{{layout}};
|
||||||
SYSTEM RELOAD DICTIONARY test_dictionary_{{layout}}_load_factor;
|
SYSTEM RELOAD DICTIONARY test_dictionary_{{layout}}_load_factor;
|
||||||
SELECT name, element_count, round(load_factor, 4), bytes_allocated FROM system.dictionaries WHERE database = currentDatabase() ORDER BY name;
|
SELECT name, element_count, round(load_factor, 4), round(bytes_allocated, -6) FROM system.dictionaries WHERE database = currentDatabase() ORDER BY name;
|
||||||
|
|
||||||
DROP DICTIONARY IF EXISTS test_dictionary_{{layout}};
|
DROP DICTIONARY IF EXISTS test_dictionary_{{layout}};
|
||||||
DROP DICTIONARY IF EXISTS test_dictionary_{{layout}}_load_factor;
|
DROP DICTIONARY IF EXISTS test_dictionary_{{layout}}_load_factor;
|
||||||
|
Loading…
Reference in New Issue
Block a user