thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-21 17:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

dbms: fixed excessive memory allocation when splitting single-level block in Aggregator [#METR-17000].

Browse Source
This commit is contained in:
Alexey Milovidov 2015-12-05 10:01:18 +03:00
parent 2e08b4e816
commit 7fa1a57165
24 changed files with 104 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dbms/include/DB/Columns/ColumnAggregateFunction.h
View File

@ -211,7 +211,7 @@ public:
memcpy(&data[old_size], &src_concrete.getData()[start], length * sizeof(data[0])); memcpy(&data[old_size], &src_concrete.getData()[start], length * sizeof(data[0]));
} }
ColumnPtr filter(const Filter & filter) const override ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override
{ {
size_t size = getData().size(); size_t size = getData().size();
if (size != filter.size()) if (size != filter.size())
@ -225,7 +225,9 @@ public:
auto & res_data = res_->getData(); auto & res_data = res_->getData();
res_data.reserve(size); if (result_size_hint)
res_data.reserve(result_size_hint > 0 ? result_size_hint : size);
for (size_t i = 0; i < size; ++i) for (size_t i = 0; i < size; ++i)
if (filter[i]) if (filter[i])
res_data.push_back(getData()[i]); res_data.push_back(getData()[i]);

8
dbms/include/DB/Columns/ColumnArray.h
View File

@ -175,7 +175,7 @@ public:
getOffsets().push_back(getOffsets().size() == 0 ? 0 : getOffsets().back()); getOffsets().push_back(getOffsets().size() == 0 ? 0 : getOffsets().back());
} }
ColumnPtr filter(const Filter & filt) const override; ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
ColumnPtr permute(const Permutation & perm, size_t limit) const override; ColumnPtr permute(const Permutation & perm, size_t limit) const override;
@ -310,10 +310,10 @@ private:
/// Специализации для функции filter. /// Специализации для функции filter.
template <typename T> template <typename T>
ColumnPtr filterNumber(const Filter & filt) const; ColumnPtr filterNumber(const Filter & filt, ssize_t result_size_hint) const;
ColumnPtr filterString(const Filter & filt) const; ColumnPtr filterString(const Filter & filt, ssize_t result_size_hint) const;
ColumnPtr filterGeneric(const Filter & filt) const; ColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const;
}; };

2
dbms/include/DB/Columns/ColumnConst.h
View File

@ -105,7 +105,7 @@ public:
throw Exception("Method deserializeAndInsertFromArena is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED); throw Exception("Method deserializeAndInsertFromArena is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
} }
ColumnPtr filter(const Filter & filt) const override ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override
{ {
if (s != filt.size()) if (s != filt.size())
throw Exception("Size of filter doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); throw Exception("Size of filter doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);

11
dbms/include/DB/Columns/ColumnFixedString.h
View File

@ -189,7 +189,7 @@ public:
memcpy(&chars[old_size], &src_concrete.chars[start * n], length * n); memcpy(&chars[old_size], &src_concrete.chars[start * n], length * n);
} }
ColumnPtr filter(const IColumn::Filter & filt) const override ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override
{ {
size_t col_size = size(); size_t col_size = size();
if (col_size != filt.size()) if (col_size != filt.size())
@ -197,7 +197,9 @@ public:
ColumnFixedString * res_ = new ColumnFixedString(n); ColumnFixedString * res_ = new ColumnFixedString(n);
ColumnPtr res = res_; ColumnPtr res = res_;
res_->chars.reserve(chars.size());
if (result_size_hint)
res_->chars.reserve(result_size_hint > 0 ? result_size_hint * n : chars.size());
size_t offset = 0; size_t offset = 0;
for (size_t i = 0; i < col_size; ++i, offset += n) for (size_t i = 0; i < col_size; ++i, offset += n)
@ -276,6 +278,11 @@ public:
max = String(); max = String();
} }
void reserve(size_t size) override
{
chars.reserve(n * size);
};
Chars_t & getChars() { return chars; } Chars_t & getChars() { return chars; }
const Chars_t & getChars() const { return chars; } const Chars_t & getChars() const { return chars; }

4
dbms/include/DB/Columns/ColumnString.h
View File

@ -178,7 +178,7 @@ public:
} }
} }
ColumnPtr filter(const Filter & filt) const override ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override
{ {
if (offsets.size() == 0) if (offsets.size() == 0)
return new ColumnString; return new ColumnString;
@ -189,7 +189,7 @@ public:
Chars_t & res_chars = res->chars; Chars_t & res_chars = res->chars;
Offsets_t & res_offsets = res->offsets; Offsets_t & res_offsets = res->offsets;
filterArraysImpl<UInt8>(chars, offsets, res_chars, res_offsets, filt); filterArraysImpl<UInt8>(chars, offsets, res_chars, res_offsets, filt, result_size_hint);
return res_; return res_;
} }

4
dbms/include/DB/Columns/ColumnTuple.h
View File

@ -123,12 +123,12 @@ public:
start, length); start, length);
} }
ColumnPtr filter(const Filter & filt) const override ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override
{ {
Block res_block = data.cloneEmpty(); Block res_block = data.cloneEmpty();
for (size_t i = 0; i < columns.size(); ++i) for (size_t i = 0; i < columns.size(); ++i)
res_block.unsafeGetByPosition(i).column = data.unsafeGetByPosition(i).column->filter(filt); res_block.unsafeGetByPosition(i).column = data.unsafeGetByPosition(i).column->filter(filt, result_size_hint);
return new ColumnTuple(res_block); return new ColumnTuple(res_block);
} }

6
dbms/include/DB/Columns/ColumnVector.h
View File

@ -270,7 +270,7 @@ public:
memcpy(&data[old_size], &src_vec.data[start], length * sizeof(data[0])); memcpy(&data[old_size], &src_vec.data[start], length * sizeof(data[0]));
} }
ColumnPtr filter(const IColumn::Filter & filt) const override ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override
{ {
size_t size = data.size(); size_t size = data.size();
if (size != filt.size()) if (size != filt.size())
@ -279,7 +279,9 @@ public:
Self * res_ = new Self; Self * res_ = new Self;
ColumnPtr res = res_; ColumnPtr res = res_;
typename Self::Container_t & res_data = res_->getData(); typename Self::Container_t & res_data = res_->getData();
res_data.reserve(size);
if (result_size_hint)
res_data.reserve(result_size_hint > 0 ? result_size_hint : size);
/** Чуть более оптимизированная версия. /** Чуть более оптимизированная версия.
* Исходит из допущения, что часто куски последовательно идущих значений * Исходит из допущения, что часто куски последовательно идущих значений

2
dbms/include/DB/Columns/ColumnsCommon.h
View File

@ -17,6 +17,6 @@ template <typename T>
void filterArraysImpl( void filterArraysImpl(
const PODArray<T> & src_elems, const IColumn::Offsets_t & src_offsets, const PODArray<T> & src_elems, const IColumn::Offsets_t & src_offsets,
PODArray<T> & res_elems, IColumn::Offsets_t & res_offsets, PODArray<T> & res_elems, IColumn::Offsets_t & res_offsets,
const IColumn::Filter & filt); const IColumn::Filter & filt, ssize_t result_size_hint);
} }

5
dbms/include/DB/Columns/IColumn.h
View File

@ -176,9 +176,12 @@ public:
/** Оставить только значения, соответствующие фильтру. /** Оставить только значения, соответствующие фильтру.
* Используется для операции WHERE / HAVING. * Используется для операции WHERE / HAVING.
* Если result_size_hint > 0, то сделать reserve этого размера у результата;
* если 0, то не делать reserve,
* иначе сделать reserve по размеру исходного столбца.
*/ */
typedef PODArray<UInt8> Filter; typedef PODArray<UInt8> Filter;
virtual SharedPtr<IColumn> filter(const Filter & filt) const = 0; virtual SharedPtr<IColumn> filter(const Filter & filt, ssize_t result_size_hint) const = 0;
/** Переставить значения местами, используя указанную перестановку. /** Переставить значения местами, используя указанную перестановку.
* Используется при сортировке. * Используется при сортировке.

2
dbms/include/DB/Columns/IColumnDummy.h
View File

@ -50,7 +50,7 @@ public:
s += length; s += length;
} }
ColumnPtr filter(const Filter & filt) const override ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override
{ {
return cloneDummy(countBytesInFilter(filt)); return cloneDummy(countBytesInFilter(filt));
} }

2
dbms/include/DB/Functions/FunctionsHigherOrder.h
View File

@ -77,7 +77,7 @@ struct ArrayFilterImpl
} }
const IColumn::Filter & filter = column_filter->getData(); const IColumn::Filter & filter = column_filter->getData();
ColumnPtr filtered = array->getData().filter(filter); ColumnPtr filtered = array->getData().filter(filter, -1);
const IColumn::Offsets_t & in_offsets = array->getOffsets(); const IColumn::Offsets_t & in_offsets = array->getOffsets();
ColumnArray::ColumnOffsets_t * column_offsets = new ColumnArray::ColumnOffsets_t(in_offsets.size()); ColumnArray::ColumnOffsets_t * column_offsets = new ColumnArray::ColumnOffsets_t(in_offsets.size());

5
dbms/include/DB/Storages/Distributed/DistributedBlockOutputStream.h
View File

@ -124,12 +124,15 @@ private:
auto filters = createFilters(block); auto filters = createFilters(block);
const auto num_shards = storage.cluster.getShardsInfo().size(); const auto num_shards = storage.cluster.getShardsInfo().size();
ssize_t size_hint = ((block.rowsInFirstColumn() + num_shards - 1) / num_shards) * 1.1; /// Число 1.1 выбрано наугад.
for (size_t i = 0; i < num_shards; ++i) for (size_t i = 0; i < num_shards; ++i)
{ {
auto target_block = block.cloneEmpty(); auto target_block = block.cloneEmpty();
for (size_t col = 0; col < num_cols; ++col) for (size_t col = 0; col < num_cols; ++col)
target_block.getByPosition(col).column = columns[col]->filter(filters[i]); target_block.getByPosition(col).column = columns[col]->filter(filters[i], size_hint);
if (target_block.rowsInFirstColumn()) if (target_block.rowsInFirstColumn())
writeImpl(target_block, i); writeImpl(target_block, i);

2
dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h
View File

@ -324,7 +324,7 @@ protected:
ColumnWithTypeAndName & column = res.getByPosition(i); ColumnWithTypeAndName & column = res.getByPosition(i);
if (column.name == prewhere_column && res.columns() > 1) if (column.name == prewhere_column && res.columns() > 1)
continue; continue;
column.column = column.column->filter(column_name_set.count(column.name) ? post_filter : pre_filter); column.column = column.column->filter(column_name_set.count(column.name) ? post_filter : pre_filter, -1);
rows = column.column->size(); rows = column.column->size();
} }

2
dbms/include/DB/Storages/MergeTree/MergeTreeReadPool.h
View File

@ -150,7 +150,7 @@ public:
per_part_remove_prewhere_column[part_idx], per_part_should_reorder[part_idx]); per_part_remove_prewhere_column[part_idx], per_part_should_reorder[part_idx]);
} }
public: private:
std::vector<std::size_t> fillPerPartInfo( std::vector<std::size_t> fillPerPartInfo(
RangesInDataParts & parts, const ExpressionActionsPtr & prewhere_actions, const String & prewhere_column_name, RangesInDataParts & parts, const ExpressionActionsPtr & prewhere_actions, const String & prewhere_column_name,
const bool check_columns) const bool check_columns)

2
dbms/include/DB/Storages/MergeTree/MergeTreeThreadBlockInputStream.h
View File

@ -255,7 +255,7 @@ private:
if (col.name == prewhere_column && res.columns() > 1) if (col.name == prewhere_column && res.columns() > 1)
continue; continue;
col.column = col.column =
col.column->filter(task->column_name_set.count(col.name) ? post_filter : pre_filter); col.column->filter(task->column_name_set.count(col.name) ? post_filter : pre_filter, -1);
rows = col.column->size(); rows = col.column->size();
} }

55
dbms/src/Columns/ColumnArray.cpp
View File

@ -41,24 +41,24 @@ void ColumnArray::insertRangeFrom(const IColumn & src, size_t start, size_t leng
} }
ColumnPtr ColumnArray::filter(const Filter & filt) const ColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const
{ {
if (typeid_cast<const ColumnUInt8 *>(data.get())) return filterNumber<UInt8>(filt); if (typeid_cast<const ColumnUInt8 *>(data.get())) return filterNumber<UInt8>(filt, result_size_hint);
if (typeid_cast<const ColumnUInt16 *>(data.get())) return filterNumber<UInt16>(filt); if (typeid_cast<const ColumnUInt16 *>(data.get())) return filterNumber<UInt16>(filt, result_size_hint);
if (typeid_cast<const ColumnUInt32 *>(data.get())) return filterNumber<UInt32>(filt); if (typeid_cast<const ColumnUInt32 *>(data.get())) return filterNumber<UInt32>(filt, result_size_hint);
if (typeid_cast<const ColumnUInt64 *>(data.get())) return filterNumber<UInt64>(filt); if (typeid_cast<const ColumnUInt64 *>(data.get())) return filterNumber<UInt64>(filt, result_size_hint);
if (typeid_cast<const ColumnInt8 *>(data.get())) return filterNumber<Int8>(filt); if (typeid_cast<const ColumnInt8 *>(data.get())) return filterNumber<Int8>(filt, result_size_hint);
if (typeid_cast<const ColumnInt16 *>(data.get())) return filterNumber<Int16>(filt); if (typeid_cast<const ColumnInt16 *>(data.get())) return filterNumber<Int16>(filt, result_size_hint);
if (typeid_cast<const ColumnInt32 *>(data.get())) return filterNumber<Int32>(filt); if (typeid_cast<const ColumnInt32 *>(data.get())) return filterNumber<Int32>(filt, result_size_hint);
if (typeid_cast<const ColumnInt64 *>(data.get())) return filterNumber<Int64>(filt); if (typeid_cast<const ColumnInt64 *>(data.get())) return filterNumber<Int64>(filt, result_size_hint);
if (typeid_cast<const ColumnFloat32 *>(data.get())) return filterNumber<Float32>(filt); if (typeid_cast<const ColumnFloat32 *>(data.get())) return filterNumber<Float32>(filt, result_size_hint);
if (typeid_cast<const ColumnFloat64 *>(data.get())) return filterNumber<Float64>(filt); if (typeid_cast<const ColumnFloat64 *>(data.get())) return filterNumber<Float64>(filt, result_size_hint);
if (typeid_cast<const ColumnString *>(data.get())) return filterString(filt); if (typeid_cast<const ColumnString *>(data.get())) return filterString(filt, result_size_hint);
return filterGeneric(filt); return filterGeneric(filt, result_size_hint);
} }
template <typename T> template <typename T>
ColumnPtr ColumnArray::filterNumber(const Filter & filt) const ColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const
{ {
if (getOffsets().size() == 0) if (getOffsets().size() == 0)
return new ColumnArray(data); return new ColumnArray(data);
@ -69,11 +69,11 @@ ColumnPtr ColumnArray::filterNumber(const Filter & filt) const
PODArray<T> & res_elems = static_cast<ColumnVector<T> &>(res->getData()).getData(); PODArray<T> & res_elems = static_cast<ColumnVector<T> &>(res->getData()).getData();
Offsets_t & res_offsets = res->getOffsets(); Offsets_t & res_offsets = res->getOffsets();
filterArraysImpl<T>(static_cast<const ColumnVector<T> &>(*data).getData(), getOffsets(), res_elems, res_offsets, filt); filterArraysImpl<T>(static_cast<const ColumnVector<T> &>(*data).getData(), getOffsets(), res_elems, res_offsets, filt, result_size_hint);
return res_; return res_;
} }
ColumnPtr ColumnArray::filterString(const Filter & filt) const ColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hint) const
{ {
size_t col_size = getOffsets().size(); size_t col_size = getOffsets().size();
if (col_size != filt.size()) if (col_size != filt.size())
@ -94,9 +94,12 @@ ColumnPtr ColumnArray::filterString(const Filter & filt) const
Offsets_t & res_string_offsets = typeid_cast<ColumnString &>(res->getData()).getOffsets(); Offsets_t & res_string_offsets = typeid_cast<ColumnString &>(res->getData()).getOffsets();
Offsets_t & res_offsets = res->getOffsets(); Offsets_t & res_offsets = res->getOffsets();
res_chars.reserve(src_chars.size()); if (result_size_hint < 0) /// Остальные случаи не рассматриваем.
res_string_offsets.reserve(src_string_offsets.size()); {
res_offsets.reserve(col_size); res_chars.reserve(src_chars.size());
res_string_offsets.reserve(src_string_offsets.size());
res_offsets.reserve(col_size);
}
Offset_t prev_src_offset = 0; Offset_t prev_src_offset = 0;
Offset_t prev_src_string_offset = 0; Offset_t prev_src_string_offset = 0;
@ -139,7 +142,7 @@ ColumnPtr ColumnArray::filterString(const Filter & filt) const
return res_; return res_;
} }
ColumnPtr ColumnArray::filterGeneric(const Filter & filt) const ColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const
{ {
size_t size = getOffsets().size(); size_t size = getOffsets().size();
if (size != filt.size()) if (size != filt.size())
@ -159,10 +162,18 @@ ColumnPtr ColumnArray::filterGeneric(const Filter & filt) const
ColumnArray * res_ = new ColumnArray(data); ColumnArray * res_ = new ColumnArray(data);
ColumnPtr res = res_; ColumnPtr res = res_;
res_->data = data->filter(nested_filt);
ssize_t nested_result_size_hint = 0;
if (result_size_hint < 0)
nested_result_size_hint = result_size_hint;
else if (result_size_hint && result_size_hint < 1000000000 && data->size() < 1000000000) /// Избегаем переполнения.
nested_result_size_hint = result_size_hint * data->size() / size;
res_->data = data->filter(nested_filt, nested_result_size_hint);
Offsets_t & res_offsets = res_->getOffsets(); Offsets_t & res_offsets = res_->getOffsets();
res_offsets.reserve(size); if (result_size_hint)
res_offsets.reserve(result_size_hint > 0 ? result_size_hint : size);
size_t current_offset = 0; size_t current_offset = 0;
for (size_t i = 0; i < size; ++i) for (size_t i = 0; i < size; ++i)

33
dbms/src/Columns/ColumnsCommon.cpp
View File

@ -47,14 +47,21 @@ template <typename T>
void filterArraysImpl( void filterArraysImpl(
const PODArray<T> & src_elems, const IColumn::Offsets_t & src_offsets, const PODArray<T> & src_elems, const IColumn::Offsets_t & src_offsets,
PODArray<T> & res_elems, IColumn::Offsets_t & res_offsets, PODArray<T> & res_elems, IColumn::Offsets_t & res_offsets,
const IColumn::Filter & filt) const IColumn::Filter & filt, ssize_t result_size_hint)
{ {
const size_t size = src_offsets.size(); const size_t size = src_offsets.size();
if (size != filt.size()) if (size != filt.size())
throw Exception("Size of filter doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); throw Exception("Size of filter doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
res_elems.reserve(src_elems.size()); if (result_size_hint)
res_offsets.reserve(size); {
res_offsets.reserve(result_size_hint > 0 ? result_size_hint : size);
if (result_size_hint < 0)
res_elems.reserve(src_elems.size());
else if (result_size_hint < 1000000000 && src_elems.size() < 1000000000) /// Избегаем переполнения.
res_elems.reserve(result_size_hint * src_elems.size() / size);
}
IColumn::Offset_t current_src_offset = 0; IColumn::Offset_t current_src_offset = 0;
@ -150,24 +157,24 @@ void filterArraysImpl(
/// Явные инстанцирования - чтобы не размещать реализацию функции выше в заголовочном файле. /// Явные инстанцирования - чтобы не размещать реализацию функции выше в заголовочном файле.
template void filterArraysImpl<UInt8>( template void filterArraysImpl<UInt8>(
const PODArray<UInt8> &, const IColumn::Offsets_t &, PODArray<UInt8> &, IColumn::Offsets_t &, const IColumn::Filter &); const PODArray<UInt8> &, const IColumn::Offsets_t &, PODArray<UInt8> &, IColumn::Offsets_t &, const IColumn::Filter &, ssize_t);
template void filterArraysImpl<UInt16>( template void filterArraysImpl<UInt16>(
const PODArray<UInt16> &, const IColumn::Offsets_t &, PODArray<UInt16> &, IColumn::Offsets_t &, const IColumn::Filter &); const PODArray<UInt16> &, const IColumn::Offsets_t &, PODArray<UInt16> &, IColumn::Offsets_t &, const IColumn::Filter &, ssize_t);
template void filterArraysImpl<UInt32>( template void filterArraysImpl<UInt32>(
const PODArray<UInt32> &, const IColumn::Offsets_t &, PODArray<UInt32> &, IColumn::Offsets_t &, const IColumn::Filter &); const PODArray<UInt32> &, const IColumn::Offsets_t &, PODArray<UInt32> &, IColumn::Offsets_t &, const IColumn::Filter &, ssize_t);
template void filterArraysImpl<UInt64>( template void filterArraysImpl<UInt64>(
const PODArray<UInt64> &, const IColumn::Offsets_t &, PODArray<UInt64> &, IColumn::Offsets_t &, const IColumn::Filter &); const PODArray<UInt64> &, const IColumn::Offsets_t &, PODArray<UInt64> &, IColumn::Offsets_t &, const IColumn::Filter &, ssize_t);
template void filterArraysImpl<Int8>( template void filterArraysImpl<Int8>(
const PODArray<Int8> &, const IColumn::Offsets_t &, PODArray<Int8> &, IColumn::Offsets_t &, const IColumn::Filter &); const PODArray<Int8> &, const IColumn::Offsets_t &, PODArray<Int8> &, IColumn::Offsets_t &, const IColumn::Filter &, ssize_t);
template void filterArraysImpl<Int16>( template void filterArraysImpl<Int16>(
const PODArray<Int16> &, const IColumn::Offsets_t &, PODArray<Int16> &, IColumn::Offsets_t &, const IColumn::Filter &); const PODArray<Int16> &, const IColumn::Offsets_t &, PODArray<Int16> &, IColumn::Offsets_t &, const IColumn::Filter &, ssize_t);
template void filterArraysImpl<Int32>( template void filterArraysImpl<Int32>(
const PODArray<Int32> &, const IColumn::Offsets_t &, PODArray<Int32> &, IColumn::Offsets_t &, const IColumn::Filter &); const PODArray<Int32> &, const IColumn::Offsets_t &, PODArray<Int32> &, IColumn::Offsets_t &, const IColumn::Filter &, ssize_t);
template void filterArraysImpl<Int64>( template void filterArraysImpl<Int64>(
const PODArray<Int64> &, const IColumn::Offsets_t &, PODArray<Int64> &, IColumn::Offsets_t &, const IColumn::Filter &); const PODArray<Int64> &, const IColumn::Offsets_t &, PODArray<Int64> &, IColumn::Offsets_t &, const IColumn::Filter &, ssize_t);
template void filterArraysImpl<Float32>( template void filterArraysImpl<Float32>(
const PODArray<Float32> &, const IColumn::Offsets_t &, PODArray<Float32> &, IColumn::Offsets_t &, const IColumn::Filter &); const PODArray<Float32> &, const IColumn::Offsets_t &, PODArray<Float32> &, IColumn::Offsets_t &, const IColumn::Filter &, ssize_t);
template void filterArraysImpl<Float64>( template void filterArraysImpl<Float64>(
const PODArray<Float64> &, const IColumn::Offsets_t &, PODArray<Float64> &, IColumn::Offsets_t &, const IColumn::Filter &); const PODArray<Float64> &, const IColumn::Offsets_t &, PODArray<Float64> &, IColumn::Offsets_t &, const IColumn::Filter &, ssize_t);
} }

2
dbms/src/Common/VirtualColumnUtils.cpp
View File

@ -169,7 +169,7 @@ bool filterBlockWithQuery(ASTPtr query, Block & block, const Context & context)
for (size_t i = 0; i < block.columns(); ++i) for (size_t i = 0; i < block.columns(); ++i)
{ {
ColumnPtr & column = block.getByPosition(i).column; ColumnPtr & column = block.getByPosition(i).column;
column = column->filter(filter); column = column->filter(filter, -1);
} }
return true; return true;

2
dbms/src/DataStreams/CollapsingFinalBlockInputStream.cpp
View File

@ -153,7 +153,7 @@ Block CollapsingFinalBlockInputStream::readImpl()
Block block = merging_block->block; Block block = merging_block->block;
for (size_t i = 0; i < block.columns(); ++i) for (size_t i = 0; i < block.columns(); ++i)
block.getByPosition(i).column = block.getByPosition(i).column->filter(merging_block->filter); block.getByPosition(i).column = block.getByPosition(i).column->filter(merging_block->filter, -1);
output_blocks.pop_back(); output_blocks.pop_back();
delete merging_block; delete merging_block;

2
dbms/src/DataStreams/DistinctBlockInputStream.cpp
View File

@ -107,7 +107,7 @@ Block DistinctBlockInputStream::readImpl()
size_t all_columns = block.columns(); size_t all_columns = block.columns();
for (size_t i = 0; i < all_columns; ++i) for (size_t i = 0; i < all_columns; ++i)
block.getByPosition(i).column = block.getByPosition(i).column->filter(filter); block.getByPosition(i).column = block.getByPosition(i).column->filter(filter, -1);
return block; return block;
} }

4
dbms/src/DataStreams/FilterBlockInputStream.cpp
View File

@ -77,7 +77,7 @@ Block FilterBlockInputStream::readImpl()
if (first_non_constant_column != static_cast<size_t>(filter_column)) if (first_non_constant_column != static_cast<size_t>(filter_column))
{ {
ColumnWithTypeAndName & current_column = res.getByPosition(first_non_constant_column); ColumnWithTypeAndName & current_column = res.getByPosition(first_non_constant_column);
current_column.column = current_column.column->filter(filter); current_column.column = current_column.column->filter(filter, -1);
filtered_rows = current_column.column->size(); filtered_rows = current_column.column->size();
} }
else else
@ -116,7 +116,7 @@ Block FilterBlockInputStream::readImpl()
if (current_column.column->isConst()) if (current_column.column->isConst())
current_column.column = current_column.column->cut(0, filtered_rows); current_column.column = current_column.column->cut(0, filtered_rows);
else else
current_column.column = current_column.column->filter(filter); current_column.column = current_column.column->filter(filter, -1);
} }
return res; return res;

2
dbms/src/DataStreams/TotalsHavingBlockInputStream.cpp
View File

@ -107,7 +107,7 @@ Block TotalsHavingBlockInputStream::readImpl()
for (size_t i = 0; i < columns; ++i) for (size_t i = 0; i < columns; ++i)
{ {
ColumnWithTypeAndName & current_column = finalized.getByPosition(i); ColumnWithTypeAndName & current_column = finalized.getByPosition(i);
current_column.column = current_column.column->filter(filter); current_column.column = current_column.column->filter(filter, -1);
if (current_column.column->empty()) if (current_column.column->empty())
{ {
finalized.clear(); finalized.clear();

5
dbms/src/Interpreters/Aggregator.cpp
View File

@ -2206,6 +2206,9 @@ void NO_INLINE Aggregator::convertBlockToTwoLevelImpl(
filter[i] = 1; filter[i] = 1;
} }
ssize_t size_hint = ((source.rowsInFirstColumn() + method.data.NUM_BUCKETS - 1)
/ method.data.NUM_BUCKETS) * 1.1; /// Число 1.1 выбрано наугад.
for (size_t bucket = 0, size = destinations.size(); bucket < size; ++bucket) for (size_t bucket = 0, size = destinations.size(); bucket < size; ++bucket)
{ {
const auto & filter = filters[bucket]; const auto & filter = filters[bucket];
@ -2219,7 +2222,7 @@ void NO_INLINE Aggregator::convertBlockToTwoLevelImpl(
for (size_t j = 0; j < columns; ++j) for (size_t j = 0; j < columns; ++j)
{ {
const ColumnWithTypeAndName & src_col = source.unsafeGetByPosition(j); const ColumnWithTypeAndName & src_col = source.unsafeGetByPosition(j);
dst.insert({src_col.column->filter(filter), src_col.type, src_col.name}); dst.insert({src_col.column->filter(filter, size_hint), src_col.type, src_col.name});
/** Вставленные в блок столбцы типа ColumnAggregateFunction будут владеть состояниями агрегатных функций /** Вставленные в блок столбцы типа ColumnAggregateFunction будут владеть состояниями агрегатных функций
* путём удержания SharedPtr-а на исходный столбец. См. ColumnAggregateFunction.h * путём удержания SharedPtr-а на исходный столбец. См. ColumnAggregateFunction.h

2
dbms/src/Interpreters/Join.cpp
View File

@ -740,7 +740,7 @@ void Join::joinBlockImpl(Block & block, const Maps & maps) const
/// Если ANY INNER|RIGHT JOIN - фильтруем все столбцы кроме новых. /// Если ANY INNER|RIGHT JOIN - фильтруем все столбцы кроме новых.
if (filter) if (filter)
for (size_t i = 0; i < existing_columns; ++i) for (size_t i = 0; i < existing_columns; ++i)
block.getByPosition(i).column = block.getByPosition(i).column->filter(*filter); block.getByPosition(i).column = block.getByPosition(i).column->filter(*filter, -1);
/// Если ALL ... JOIN - размножаем все столбцы кроме новых. /// Если ALL ... JOIN - размножаем все столбцы кроме новых.
if (offsets_to_replicate) if (offsets_to_replicate)