thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 07:31:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3909 from yandex/fix-use-after-free-in-array-enumerate

Browse Source 
Fix use after free in arrayEnumerateUniq and -Dense function.
This commit is contained in:
alexey-milovidov 2018-12-23 06:11:43 +03:00 committed by GitHub
commit a03c28d0e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 176 additions and 459 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
dbms/src/Columns/ColumnArray.cpp
View File

@ -322,17 +322,11 @@ bool ColumnArray::hasEqualOffsets(const ColumnArray & other) const
ColumnPtr ColumnArray::convertToFullColumnIfConst() const
{
ColumnPtr new_data;
if (ColumnPtr full_column = getData().convertToFullColumnIfConst())
new_data = full_column;
else
new_data = data;
return ColumnArray::create(new_data, offsets);
/// It is possible to have an array with constant data and non-constant offsets.
/// Example is the result of expression: replicate('hello', [1])
return ColumnArray::create(data->convertToFullColumnIfConst(), offsets);
}
void ColumnArray::getExtremes(Field & min, Field & max) const
{
min = Array();

3
dbms/src/Columns/ColumnNullable.cpp
View File

@ -22,8 +22,7 @@ ColumnNullable::ColumnNullable(MutableColumnPtr && nested_column_, MutableColumn
: nested_column(std::move(nested_column_)), null_map(std::move(null_map_))
{
/// ColumnNullable cannot have constant nested column. But constant argument could be passed. Materialize it.
if (ColumnPtr nested_column_materialized = getNestedColumn().convertToFullColumnIfConst())
nested_column = nested_column_materialized;
nested_column = getNestedColumn().convertToFullColumnIfConst();
if (!getNestedColumn().canBeInsideNullable())
throw Exception{getNestedColumn().getName() + " cannot be inside Nullable column", ErrorCodes::ILLEGAL_COLUMN};

2
dbms/src/Columns/IColumn.h
View File

@ -45,7 +45,7 @@ public:
/** If column isn't constant, returns nullptr (or itself).
* If column is constant, transforms constant to full column (if column type allows such tranform) and return it.
*/
virtual Ptr convertToFullColumnIfConst() const { return {}; }
virtual Ptr convertToFullColumnIfConst() const { return getPtr(); }
/// If column isn't ColumnLowCardinality, return itself.
/// If column is ColumnLowCardinality, transforms is to full column.

7
dbms/src/DataStreams/NativeBlockOutputStream.cpp
View File

@ -46,12 +46,7 @@ void NativeBlockOutputStream::writeData(const IDataType & type, const ColumnPtr
/** If there are columns-constants - then we materialize them.
* (Since the data type does not know how to serialize / deserialize constants.)
*/
ColumnPtr full_column;
if (ColumnPtr converted = column->convertToFullColumnIfConst())
full_column = converted;
else
full_column = column;
ColumnPtr full_column = column->convertToFullColumnIfConst();
IDataType::SerializeBinaryBulkSettings settings;
settings.getter = [&ostr](IDataType::SubstreamPath) -> WriteBuffer * { return &ostr; };

5
dbms/src/DataStreams/TotalsHavingBlockInputStream.cpp
View File

@ -127,10 +127,7 @@ Block TotalsHavingBlockInputStream::readImpl()
expression->execute(finalized);
size_t filter_column_pos = finalized.getPositionByName(filter_column_name);
ColumnPtr filter_column_ptr = finalized.safeGetByPosition(filter_column_pos).column;
if (ColumnPtr materialized = filter_column_ptr->convertToFullColumnIfConst())
filter_column_ptr = materialized;
ColumnPtr filter_column_ptr = finalized.safeGetByPosition(filter_column_pos).column->convertToFullColumnIfConst();
FilterDescription filter_description(*filter_column_ptr);

4
dbms/src/DataStreams/materializeBlock.cpp
View File

@ -14,9 +14,7 @@ Block materializeBlock(const Block & block)
for (size_t i = 0; i < columns; ++i)
{
auto & element = res.getByPosition(i);
auto & src = element.column;
if (ColumnPtr converted = src->convertToFullColumnIfConst())
src = converted;
element.column = element.column->convertToFullColumnIfConst();
}
return res;

20
dbms/src/Functions/FunctionsExternalDictionaries.h
View File

@ -346,11 +346,8 @@ private:
String attr_name = attr_name_col->getValue<String>();
const ColumnWithTypeAndName & key_col_with_type = block.getByPosition(arguments[2]);
ColumnPtr key_col = key_col_with_type.column;
/// Functions in external dictionaries only support full-value (not constant) columns with keys.
if (ColumnPtr key_col_materialized = key_col_with_type.column->convertToFullColumnIfConst())
key_col = key_col_materialized;
ColumnPtr key_col = key_col_with_type.column->convertToFullColumnIfConst();
if (checkColumn<ColumnTuple>(key_col.get()))
{
@ -578,11 +575,8 @@ private:
String attr_name = attr_name_col->getValue<String>();
const ColumnWithTypeAndName & key_col_with_type = block.getByPosition(arguments[2]);
ColumnPtr key_col = key_col_with_type.column;
/// Functions in external dictionaries only support full-value (not constant) columns with keys.
if (ColumnPtr key_col_materialized = key_col_with_type.column->convertToFullColumnIfConst())
key_col = key_col_materialized;
ColumnPtr key_col = key_col_with_type.column->convertToFullColumnIfConst();
const auto & key_columns = typeid_cast<const ColumnTuple &>(*key_col).getColumns();
const auto & key_types = static_cast<const DataTypeTuple &>(*key_col_with_type.type).getElements();
@ -813,11 +807,9 @@ private:
String attr_name = attr_name_col->getValue<String>();
const ColumnWithTypeAndName & key_col_with_type = block.getByPosition(arguments[2]);
ColumnPtr key_col = key_col_with_type.column;
/// Functions in external dictionaries only support full-value (not constant) columns with keys.
if (ColumnPtr key_col_materialized = key_col_with_type.column->convertToFullColumnIfConst())
key_col = key_col_materialized;
ColumnPtr key_col = key_col_with_type.column->convertToFullColumnIfConst();
if (checkColumn<ColumnTuple>(key_col.get()))
{
@ -1079,11 +1071,9 @@ private:
String attr_name = attr_name_col->getValue<String>();
const ColumnWithTypeAndName & key_col_with_type = block.getByPosition(arguments[2]);
ColumnPtr key_col = key_col_with_type.column;
/// Functions in external dictionaries only support full-value (not constant) columns with keys.
if (ColumnPtr key_col_materialized = key_col_with_type.column->convertToFullColumnIfConst())
key_col = key_col_materialized;
ColumnPtr key_col = key_col_with_type.column->convertToFullColumnIfConst();
const auto & key_columns = typeid_cast<const ColumnTuple &>(*key_col).getColumns();
const auto & key_types = static_cast<const DataTypeTuple &>(*key_col_with_type.type).getElements();
@ -1691,7 +1681,7 @@ static const PaddedPODArray<T> & getColumnDataAsPaddedPODArray(const IColumn & c
}
}
const auto full_column = column.isColumnConst() ? column.convertToFullColumnIfConst() : column.getPtr();
const auto full_column = column.convertToFullColumnIfConst();
// With type conversion and const columns we need to use backup storage here
const auto size = full_column->size();

3
dbms/src/Functions/FunctionsFindCluster.h
View File

@ -227,8 +227,7 @@ protected:
bool executeOperationTyped(const IColumn * in_untyped, PaddedPODArray<OutputType> & dst, const IColumn * centroids_array_untyped)
{
const auto maybe_const = in_untyped->convertToFullColumnIfConst();
if (maybe_const)
in_untyped = maybe_const.get();
in_untyped = maybe_const.get();
const auto in_vector = checkAndGetColumn<ColumnVector<InputType>>(in_untyped);
if (in_vector)

9
dbms/src/Functions/IFunction.cpp
View File

@ -157,10 +157,7 @@ ColumnPtr wrapInNullable(const ColumnPtr & src, const Block & block, const Colum
if (!result_null_map_column)
return makeNullable(src);
if (src_not_nullable->isColumnConst())
return ColumnNullable::create(src_not_nullable->convertToFullColumnIfConst(), result_null_map_column);
else
return ColumnNullable::create(src_not_nullable, result_null_map_column);
return ColumnNullable::create(src_not_nullable->convertToFullColumnIfConst(), result_null_map_column);
}
@ -431,9 +428,7 @@ void PreparedFunctionImpl::execute(Block & block, const ColumnNumbers & args, si
executeWithoutLowCardinalityColumns(block_without_low_cardinality, args, result, block_without_low_cardinality.rows(), dry_run);
auto & keys = block_without_low_cardinality.safeGetByPosition(result).column;
if (auto full_column = keys->convertToFullColumnIfConst())
keys = full_column;
auto keys = block_without_low_cardinality.safeGetByPosition(result).column->convertToFullColumnIfConst();
auto res_mut_dictionary = DataTypeLowCardinality::createColumnUnique(*res_low_cardinality_type->getDictionaryType());
ColumnPtr res_indexes = res_mut_dictionary->uniqueInsertRangeFrom(*keys, 0, keys->size());

3
dbms/src/Functions/array.cpp
View File

@ -69,8 +69,7 @@ public:
if (!arg.type->equals(*elem_type))
preprocessed_column = castColumn(arg, elem_type, context);
if (ColumnPtr materialized_column = preprocessed_column->convertToFullColumnIfConst())
preprocessed_column = materialized_column;
preprocessed_column = preprocessed_column->convertToFullColumnIfConst();
columns_holder[i] = std::move(preprocessed_column);
columns[i] = columns_holder[i].get();

220
dbms/src/Functions/arrayEnumerateExtended.h
View File

@ -61,21 +61,10 @@ private:
static constexpr size_t INITIAL_SIZE_DEGREE = 9;
template <typename T>
bool executeNumber(const ColumnArray * array, const IColumn * null_map, ColumnUInt32::Container & res_values);
bool executeString(const ColumnArray * array, const IColumn * null_map, ColumnUInt32::Container & res_values);
bool execute128bit(
const ColumnArray::Offsets & offsets,
const ColumnRawPtrs & columns,
const ColumnRawPtrs & null_maps,
ColumnUInt32::Container & res_values,
bool has_nullable_columns);
void executeHashed(
const ColumnArray::Offsets & offsets,
const ColumnRawPtrs & columns,
ColumnUInt32::Container & res_values);
bool executeNumber(const ColumnArray::Offsets & offsets, const IColumn & data, const NullMap * null_map, ColumnUInt32::Container & res_values);
bool executeString(const ColumnArray::Offsets & offsets, const IColumn & data, const NullMap * null_map, ColumnUInt32::Container & res_values);
bool execute128bit(const ColumnArray::Offsets & offsets, const ColumnRawPtrs & columns, ColumnUInt32::Container & res_values);
bool executeHashed(const ColumnArray::Offsets & offsets, const ColumnRawPtrs & columns, ColumnUInt32::Container & res_values);
};
@ -83,14 +72,14 @@ template <typename Derived>
void FunctionArrayEnumerateExtended<Derived>::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/)
{
const ColumnArray::Offsets * offsets = nullptr;
ColumnRawPtrs data_columns;
data_columns.reserve(arguments.size());
size_t num_arguments = arguments.size();
ColumnRawPtrs data_columns(num_arguments);
bool has_nullable_columns = false;
for (size_t i = 0; i < arguments.size(); ++i)
Columns array_holders;
ColumnPtr offsets_column;
for (size_t i = 0; i < num_arguments; ++i)
{
ColumnPtr array_ptr = block.getByPosition(arguments[i]).column;
const ColumnPtr & array_ptr = block.getByPosition(arguments[i]).column;
const ColumnArray * array = checkAndGetColumn<ColumnArray>(array_ptr.get());
if (!array)
{
@ -100,101 +89,84 @@ void FunctionArrayEnumerateExtended<Derived>::executeImpl(Block & block, const C
throw Exception("Illegal column " + block.getByPosition(arguments[i]).column->getName()
+ " of " + toString(i + 1) + "-th argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
array_ptr = const_array->convertToFullColumn();
array = checkAndGetColumn<ColumnArray>(array_ptr.get());
array_holders.emplace_back(const_array->convertToFullColumn());
array = checkAndGetColumn<ColumnArray>(array_holders.back().get());
}
const ColumnArray::Offsets & offsets_i = array->getOffsets();
if (i == 0)
{
offsets = &offsets_i;
offsets_column = array->getOffsetsPtr();
}
else if (offsets_i != *offsets)
throw Exception("Lengths of all arrays passed to " + getName() + " must be equal.",
ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH);
auto * array_data = &array->getData();
data_columns.push_back(array_data);
data_columns[i] = array_data;
}
size_t num_columns = data_columns.size();
ColumnRawPtrs original_data_columns(num_columns);
ColumnRawPtrs null_maps(num_columns);
const NullMap * null_map = nullptr;
for (size_t i = 0; i < num_columns; ++i)
for (size_t i = 0; i < num_arguments; ++i)
{
original_data_columns[i] = data_columns[i];
if (data_columns[i]->isColumnNullable())
{
has_nullable_columns = true;
const auto & nullable_col = static_cast<const ColumnNullable &>(*data_columns[i]);
data_columns[i] = &nullable_col.getNestedColumn();
null_maps[i] = &nullable_col.getNullMapColumn();
if (num_arguments == 1)
data_columns[i] = &nullable_col.getNestedColumn();
null_map = &nullable_col.getNullMapData();
break;
}
else
null_maps[i] = nullptr;
}
const ColumnArray * first_array = checkAndGetColumn<ColumnArray>(block.getByPosition(arguments.at(0)).column.get());
const IColumn * first_null_map = null_maps[0];
auto res_nested = ColumnUInt32::create();
ColumnUInt32::Container & res_values = res_nested->getData();
if (!offsets->empty())
res_values.resize(offsets->back());
if (num_columns == 1)
if (num_arguments == 1)
{
if (!(executeNumber<UInt8>(first_array, first_null_map, res_values)
|| executeNumber<UInt16>(first_array, first_null_map, res_values)
|| executeNumber<UInt32>(first_array, first_null_map, res_values)
|| executeNumber<UInt64>(first_array, first_null_map, res_values)
|| executeNumber<Int8>(first_array, first_null_map, res_values)
|| executeNumber<Int16>(first_array, first_null_map, res_values)
|| executeNumber<Int32>(first_array, first_null_map, res_values)
|| executeNumber<Int64>(first_array, first_null_map, res_values)
|| executeNumber<Float32>(first_array, first_null_map, res_values)
|| executeNumber<Float64>(first_array, first_null_map, res_values)
|| executeString (first_array, first_null_map, res_values)))
executeHashed(*offsets, original_data_columns, res_values);
executeNumber<UInt8>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<UInt16>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<UInt32>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<UInt64>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<Int8>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<Int16>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<Int32>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<Int64>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<Float32>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<Float64>(*offsets, *data_columns[0], null_map, res_values)
|| executeString(*offsets, *data_columns[0], null_map, res_values)
|| executeHashed(*offsets, data_columns, res_values);
}
else
{
if (!execute128bit(*offsets, data_columns, null_maps, res_values, has_nullable_columns))
executeHashed(*offsets, original_data_columns, res_values);
execute128bit(*offsets, data_columns, res_values)
|| executeHashed(*offsets, data_columns, res_values);
}
block.getByPosition(result).column = ColumnArray::create(std::move(res_nested), first_array->getOffsetsPtr());
block.getByPosition(result).column = ColumnArray::create(std::move(res_nested), offsets_column);
}
template <typename Derived>
template <typename T>
bool FunctionArrayEnumerateExtended<Derived>::executeNumber(const ColumnArray * array, const IColumn * null_map, ColumnUInt32::Container & res_values)
bool FunctionArrayEnumerateExtended<Derived>::executeNumber(
const ColumnArray::Offsets & offsets, const IColumn & data, const NullMap * null_map, ColumnUInt32::Container & res_values)
{
const IColumn * inner_col;
const auto & array_data = array->getData();
if (array_data.isColumnNullable())
{
const auto & nullable_col = static_cast<const ColumnNullable &>(array_data);
inner_col = &nullable_col.getNestedColumn();
}
else
inner_col = &array_data;
const ColumnVector<T> * nested = checkAndGetColumn<ColumnVector<T>>(inner_col);
if (!nested)
const ColumnVector<T> * data_concrete = checkAndGetColumn<ColumnVector<T>>(&data);
if (!data_concrete)
return false;
const ColumnArray::Offsets & offsets = array->getOffsets();
const typename ColumnVector<T>::Container & values = nested->getData();
const auto & values = data_concrete->getData();
using ValuesToIndices = ClearableHashMap<T, UInt32, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>;
const PaddedPODArray<UInt8> * null_map_data = nullptr;
if (null_map)
null_map_data = &static_cast<const ColumnUInt8 *>(null_map)->getData();
ValuesToIndices indices;
size_t prev_off = 0;
if constexpr (std::is_same_v<Derived, FunctionArrayEnumerateUniq>)
@ -207,7 +179,7 @@ bool FunctionArrayEnumerateExtended<Derived>::executeNumber(const ColumnArray *
size_t off = offsets[i];
for (size_t j = prev_off; j < off; ++j)
{
if (null_map_data && ((*null_map_data)[j] == 1))
if (null_map && (*null_map)[j])
res_values[j] = ++null_count;
else
res_values[j] = ++indices[values[j]];
@ -226,7 +198,7 @@ bool FunctionArrayEnumerateExtended<Derived>::executeNumber(const ColumnArray *
size_t off = offsets[i];
for (size_t j = prev_off; j < off; ++j)
{
if (null_map_data && ((*null_map_data)[j] == 1))
if (null_map && (*null_map)[j])
{
if (!null_index)
null_index = ++rank;
@ -247,32 +219,17 @@ bool FunctionArrayEnumerateExtended<Derived>::executeNumber(const ColumnArray *
}
template <typename Derived>
bool FunctionArrayEnumerateExtended<Derived>::executeString(const ColumnArray * array, const IColumn * null_map, ColumnUInt32::Container & res_values)
bool FunctionArrayEnumerateExtended<Derived>::executeString(
const ColumnArray::Offsets & offsets, const IColumn & data, const NullMap * null_map, ColumnUInt32::Container & res_values)
{
const IColumn * inner_col;
const auto & array_data = array->getData();
if (array_data.isColumnNullable())
{
const auto & nullable_col = static_cast<const ColumnNullable &>(array_data);
inner_col = &nullable_col.getNestedColumn();
}
else
inner_col = &array_data;
const ColumnString * nested = checkAndGetColumn<ColumnString>(inner_col);
if (!nested)
const ColumnString * values = checkAndGetColumn<ColumnString>(&data);
if (!values)
return false;
const ColumnArray::Offsets & offsets = array->getOffsets();
size_t prev_off = 0;
using ValuesToIndices = ClearableHashMap<StringRef, UInt32, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
const PaddedPODArray<UInt8> * null_map_data = nullptr;
if (null_map)
null_map_data = &static_cast<const ColumnUInt8 *>(null_map)->getData();
ValuesToIndices indices;
if constexpr (std::is_same_v<Derived, FunctionArrayEnumerateUniq>)
{
@ -284,10 +241,10 @@ bool FunctionArrayEnumerateExtended<Derived>::executeString(const ColumnArray *
size_t off = offsets[i];
for (size_t j = prev_off; j < off; ++j)
{
if (null_map_data && ((*null_map_data)[j] == 1))
if (null_map && (*null_map)[j])
res_values[j] = ++null_count;
else
res_values[j] = ++indices[nested->getDataAt(j)];
res_values[j] = ++indices[values->getDataAt(j)];
}
prev_off = off;
}
@ -303,7 +260,7 @@ bool FunctionArrayEnumerateExtended<Derived>::executeString(const ColumnArray *
size_t off = offsets[i];
for (size_t j = prev_off; j < off; ++j)
{
if (null_map_data && ((*null_map_data)[j] == 1))
if (null_map && (*null_map)[j])
{
if (!null_index)
null_index = ++rank;
@ -311,7 +268,7 @@ bool FunctionArrayEnumerateExtended<Derived>::executeString(const ColumnArray *
}
else
{
auto & idx = indices[nested->getDataAt(j)];
auto & idx = indices[values->getDataAt(j)];
if (!idx)
idx = ++rank;
res_values[j] = idx;
@ -327,9 +284,7 @@ template <typename Derived>
bool FunctionArrayEnumerateExtended<Derived>::execute128bit(
const ColumnArray::Offsets & offsets,
const ColumnRawPtrs & columns,
const ColumnRawPtrs & null_maps,
ColumnUInt32::Container & res_values,
bool has_nullable_columns)
ColumnUInt32::Container & res_values)
{
size_t count = columns.size();
size_t keys_bytes = 0;
@ -342,8 +297,6 @@ bool FunctionArrayEnumerateExtended<Derived>::execute128bit(
key_sizes[j] = columns[j]->sizeOfValueIfFixed();
keys_bytes += key_sizes[j];
}
if (has_nullable_columns)
keys_bytes += std::tuple_size<KeysNullMap<UInt128>>::value;
if (keys_bytes > 16)
return false;
@ -361,29 +314,7 @@ bool FunctionArrayEnumerateExtended<Derived>::execute128bit(
indices.clear();
size_t off = offsets[i];
for (size_t j = prev_off; j < off; ++j)
{
if (has_nullable_columns)
{
KeysNullMap<UInt128> bitmap{};
for (size_t i = 0; i < columns.size(); ++i)
{
if (null_maps[i])
{
const auto & null_map = static_cast<const ColumnUInt8 &>(*null_maps[i]).getData();
if (null_map[j] == 1)
{
size_t bucket = i / 8;
size_t offset = i % 8;
bitmap[bucket] |= UInt8(1) << offset;
}
}
}
res_values[j] = ++indices[packFixed<UInt128>(j, count, columns, key_sizes, bitmap)];
}
else
res_values[j] = ++indices[packFixed<UInt128>(j, count, columns, key_sizes)];
}
res_values[j] = ++indices[packFixed<UInt128>(j, count, columns, key_sizes)];
prev_off = off;
}
}
@ -397,35 +328,10 @@ bool FunctionArrayEnumerateExtended<Derived>::execute128bit(
size_t rank = 0;
for (size_t j = prev_off; j < off; ++j)
{
if (has_nullable_columns)
{
KeysNullMap<UInt128> bitmap{};
for (size_t i = 0; i < columns.size(); ++i)
{
if (null_maps[i])
{
const auto & null_map = static_cast<const ColumnUInt8 &>(*null_maps[i]).getData();
if (null_map[j] == 1)
{
size_t bucket = i / 8;
size_t offset = i % 8;
bitmap[bucket] |= UInt8(1) << offset;
}
}
}
auto &idx = indices[packFixed<UInt128>(j, count, columns, key_sizes, bitmap)];
if (!idx)
idx = ++rank;
res_values[j] = idx;
}
else
{
auto &idx = indices[packFixed<UInt128>(j, count, columns, key_sizes)];;
if (!idx)
idx = ++rank;
res_values[j] = idx;
}
auto &idx = indices[packFixed<UInt128>(j, count, columns, key_sizes)];;
if (!idx)
idx = ++rank;
res_values[j] = idx;
}
prev_off = off;
}
@ -435,7 +341,7 @@ bool FunctionArrayEnumerateExtended<Derived>::execute128bit(
}
template <typename Derived>
void FunctionArrayEnumerateExtended<Derived>::executeHashed(
bool FunctionArrayEnumerateExtended<Derived>::executeHashed(
const ColumnArray::Offsets & offsets,
const ColumnRawPtrs & columns,
ColumnUInt32::Container & res_values)
@ -479,6 +385,8 @@ void FunctionArrayEnumerateExtended<Derived>::executeHashed(
prev_off = off;
}
}
return true;
}
}

12
dbms/src/Functions/arrayIndex.h
View File

@ -838,15 +838,9 @@ private:
null_map_data, nullptr);
else
{
/// If item_arg is tuple and have constants.
if (ColumnPtr materialized_tuple = item_arg.convertToFullColumnIfConst())
ArrayIndexGenericImpl<IndexConv, false>::vector(
col_nested, col_array->getOffsets(), *materialized_tuple, col_res->getData(),
null_map_data, null_map_item);
else
ArrayIndexGenericImpl<IndexConv, false>::vector(
col_nested, col_array->getOffsets(), item_arg, col_res->getData(),
null_map_data, null_map_item);
ArrayIndexGenericImpl<IndexConv, false>::vector(
col_nested, col_array->getOffsets(), *item_arg.convertToFullColumnIfConst(), col_res->getData(),
null_map_data, null_map_item);
}
block.getByPosition(result).column = std::move(col_res);

4
dbms/src/Functions/arrayMap.cpp
View File

@ -23,9 +23,7 @@ struct ArrayMapImpl
static ColumnPtr execute(const ColumnArray & array, ColumnPtr mapped)
{
return mapped->isColumnConst()
? ColumnArray::create(mapped->convertToFullColumnIfConst(), array.getOffsetsPtr())
: ColumnArray::create(mapped, array.getOffsetsPtr());
return ColumnArray::create(mapped->convertToFullColumnIfConst(), array.getOffsetsPtr());
}
};

196
dbms/src/Functions/arrayUniq.cpp
View File

@ -63,37 +63,23 @@ private:
static constexpr size_t INITIAL_SIZE_DEGREE = 9;
template <typename T>
bool executeNumber(const ColumnArray * array, const IColumn * null_map, ColumnUInt32::Container & res_values);
bool executeString(const ColumnArray * array, const IColumn * null_map, ColumnUInt32::Container & res_values);
bool execute128bit(
const ColumnArray::Offsets & offsets,
const ColumnRawPtrs & columns,
const ColumnRawPtrs & null_maps,
ColumnUInt32::Container & res_values,
bool has_nullable_columns);
void executeHashed(
const ColumnArray::Offsets & offsets,
const ColumnRawPtrs & columns,
ColumnUInt32::Container & res_values);
bool executeNumber(const ColumnArray::Offsets & offsets, const IColumn & data, const NullMap * null_map, ColumnUInt32::Container & res_values);
bool executeString(const ColumnArray::Offsets & offsets, const IColumn & data, const NullMap * null_map, ColumnUInt32::Container & res_values);
bool execute128bit(const ColumnArray::Offsets & offsets, const ColumnRawPtrs & columns, ColumnUInt32::Container & res_values);
bool executeHashed(const ColumnArray::Offsets & offsets, const ColumnRawPtrs & columns, ColumnUInt32::Container & res_values);
};
void FunctionArrayUniq::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/)
{
Columns array_columns(arguments.size());
const ColumnArray::Offsets * offsets = nullptr;
ColumnRawPtrs data_columns(arguments.size());
ColumnRawPtrs original_data_columns(arguments.size());
ColumnRawPtrs null_maps(arguments.size());
size_t num_arguments = arguments.size();
ColumnRawPtrs data_columns(num_arguments);
bool has_nullable_columns = false;
for (size_t i = 0; i < arguments.size(); ++i)
Columns array_holders;
for (size_t i = 0; i < num_arguments; ++i)
{
ColumnPtr array_ptr = block.getByPosition(arguments[i]).column;
const ColumnPtr & array_ptr = block.getByPosition(arguments[i]).column;
const ColumnArray * array = checkAndGetColumn<ColumnArray>(array_ptr.get());
if (!array)
{
@ -101,14 +87,12 @@ void FunctionArrayUniq::executeImpl(Block & block, const ColumnNumbers & argumen
block.getByPosition(arguments[i]).column.get());
if (!const_array)
throw Exception("Illegal column " + block.getByPosition(arguments[i]).column->getName()
+ " of " + toString(i + 1) + getOrdinalSuffix(i + 1) + " argument of function " + getName(),
+ " of " + toString(i + 1) + "-th argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
array_ptr = const_array->convertToFullColumn();
array = static_cast<const ColumnArray *>(array_ptr.get());
array_holders.emplace_back(const_array->convertToFullColumn());
array = checkAndGetColumn<ColumnArray>(array_holders.back().get());
}
array_columns[i] = array_ptr;
const ColumnArray::Offsets & offsets_i = array->getOffsets();
if (i == 0)
offsets = &offsets_i;
@ -116,78 +100,65 @@ void FunctionArrayUniq::executeImpl(Block & block, const ColumnNumbers & argumen
throw Exception("Lengths of all arrays passed to " + getName() + " must be equal.",
ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH);
data_columns[i] = &array->getData();
original_data_columns[i] = data_columns[i];
if (data_columns[i]->isColumnNullable())
{
has_nullable_columns = true;
const auto & nullable_col = static_cast<const ColumnNullable &>(*data_columns[i]);
data_columns[i] = &nullable_col.getNestedColumn();
null_maps[i] = &nullable_col.getNullMapColumn();
}
else
null_maps[i] = nullptr;
auto * array_data = &array->getData();
data_columns[i] = array_data;
}
const ColumnArray * first_array = static_cast<const ColumnArray *>(array_columns[0].get());
const IColumn * first_null_map = null_maps[0];
auto res = ColumnUInt32::create();
const NullMap * null_map = nullptr;
for (size_t i = 0; i < num_arguments; ++i)
{
if (data_columns[i]->isColumnNullable())
{
const auto & nullable_col = static_cast<const ColumnNullable &>(*data_columns[i]);
if (num_arguments == 1)
data_columns[i] = &nullable_col.getNestedColumn();
null_map = &nullable_col.getNullMapData();
break;
}
}
auto res = ColumnUInt32::create();
ColumnUInt32::Container & res_values = res->getData();
res_values.resize(offsets->size());
if (arguments.size() == 1)
if (num_arguments == 1)
{
if (!(executeNumber<UInt8>(first_array, first_null_map, res_values)
|| executeNumber<UInt16>(first_array, first_null_map, res_values)
|| executeNumber<UInt32>(first_array, first_null_map, res_values)
|| executeNumber<UInt64>(first_array, first_null_map, res_values)
|| executeNumber<Int8>(first_array, first_null_map, res_values)
|| executeNumber<Int16>(first_array, first_null_map, res_values)
|| executeNumber<Int32>(first_array, first_null_map, res_values)
|| executeNumber<Int64>(first_array, first_null_map, res_values)
|| executeNumber<Float32>(first_array, first_null_map, res_values)
|| executeNumber<Float64>(first_array, first_null_map, res_values)
|| executeString(first_array, first_null_map, res_values)))
executeHashed(*offsets, original_data_columns, res_values);
executeNumber<UInt8>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<UInt16>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<UInt32>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<UInt64>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<Int8>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<Int16>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<Int32>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<Int64>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<Float32>(*offsets, *data_columns[0], null_map, res_values)
|| executeNumber<Float64>(*offsets, *data_columns[0], null_map, res_values)
|| executeString(*offsets, *data_columns[0], null_map, res_values)
|| executeHashed(*offsets, data_columns, res_values);
}
else
{
if (!execute128bit(*offsets, data_columns, null_maps, res_values, has_nullable_columns))
executeHashed(*offsets, original_data_columns, res_values);
execute128bit(*offsets, data_columns, res_values)
|| executeHashed(*offsets, data_columns, res_values);
}
block.getByPosition(result).column = std::move(res);
}
template <typename T>
bool FunctionArrayUniq::executeNumber(const ColumnArray * array, const IColumn * null_map, ColumnUInt32::Container & res_values)
bool FunctionArrayUniq::executeNumber(const ColumnArray::Offsets & offsets, const IColumn & data, const NullMap * null_map, ColumnUInt32::Container & res_values)
{
const IColumn * inner_col;
const auto & array_data = array->getData();
if (array_data.isColumnNullable())
{
const auto & nullable_col = static_cast<const ColumnNullable &>(array_data);
inner_col = &nullable_col.getNestedColumn();
}
else
inner_col = &array_data;
const ColumnVector<T> * nested = checkAndGetColumn<ColumnVector<T>>(inner_col);
const ColumnVector<T> * nested = checkAndGetColumn<ColumnVector<T>>(&data);
if (!nested)
return false;
const ColumnArray::Offsets & offsets = array->getOffsets();
const typename ColumnVector<T>::Container & values = nested->getData();
const auto & values = nested->getData();
using Set = ClearableHashSet<T, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>;
const PaddedPODArray<UInt8> * null_map_data = nullptr;
if (null_map)
null_map_data = &static_cast<const ColumnUInt8 *>(null_map)->getData();
Set set;
ColumnArray::Offset prev_off = 0;
for (size_t i = 0; i < offsets.size(); ++i)
@ -197,7 +168,7 @@ bool FunctionArrayUniq::executeNumber(const ColumnArray * array, const IColumn *
ColumnArray::Offset off = offsets[i];
for (ColumnArray::Offset j = prev_off; j < off; ++j)
{
if (null_map_data && ((*null_map_data)[j] == 1))
if (null_map && (*null_map)[j])
found_null = true;
else
set.insert(values[j]);
@ -209,31 +180,15 @@ bool FunctionArrayUniq::executeNumber(const ColumnArray * array, const IColumn *
return true;
}
bool FunctionArrayUniq::executeString(const ColumnArray * array, const IColumn * null_map, ColumnUInt32::Container & res_values)
bool FunctionArrayUniq::executeString(const ColumnArray::Offsets & offsets, const IColumn & data, const NullMap * null_map, ColumnUInt32::Container & res_values)
{
const IColumn * inner_col;
const auto & array_data = array->getData();
if (array_data.isColumnNullable())
{
const auto & nullable_col = static_cast<const ColumnNullable &>(array_data);
inner_col = &nullable_col.getNestedColumn();
}
else
inner_col = &array_data;
const ColumnString * nested = checkAndGetColumn<ColumnString>(inner_col);
const ColumnString * nested = checkAndGetColumn<ColumnString>(&data);
if (!nested)
return false;
const ColumnArray::Offsets & offsets = array->getOffsets();
using Set = ClearableHashSet<StringRef, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
const PaddedPODArray<UInt8> * null_map_data = nullptr;
if (null_map)
null_map_data = &static_cast<const ColumnUInt8 *>(null_map)->getData();
Set set;
ColumnArray::Offset prev_off = 0;
for (size_t i = 0; i < offsets.size(); ++i)
@ -243,7 +198,7 @@ bool FunctionArrayUniq::executeString(const ColumnArray * array, const IColumn *
ColumnArray::Offset off = offsets[i];
for (ColumnArray::Offset j = prev_off; j < off; ++j)
{
if (null_map_data && ((*null_map_data)[j] == 1))
if (null_map && (*null_map)[j])
found_null = true;
else
set.insert(nested->getDataAt(j));
@ -259,9 +214,7 @@ bool FunctionArrayUniq::executeString(const ColumnArray * array, const IColumn *
bool FunctionArrayUniq::execute128bit(
const ColumnArray::Offsets & offsets,
const ColumnRawPtrs & columns,
const ColumnRawPtrs & null_maps,
ColumnUInt32::Container & res_values,
bool has_nullable_columns)
ColumnUInt32::Container & res_values)
{
size_t count = columns.size();
size_t keys_bytes = 0;
@ -274,8 +227,6 @@ bool FunctionArrayUniq::execute128bit(
key_sizes[j] = columns[j]->sizeOfValueIfFixed();
keys_bytes += key_sizes[j];
}
if (has_nullable_columns)
keys_bytes += std::tuple_size<KeysNullMap<UInt128>>::value;
if (keys_bytes > 16)
return false;
@ -283,19 +234,6 @@ bool FunctionArrayUniq::execute128bit(
using Set = ClearableHashSet<UInt128, UInt128HashCRC32, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>;
/// Suppose that, for a given row, each of the N columns has an array whose length is M.
/// Denote arr_i each of these arrays (1 <= i <= N). Then the following is performed:
///
/// col1 ... colN
///
/// arr_1[1], ..., arr_N[1] -> pack into a binary blob b1
/// .
/// .
/// .
/// arr_1[M], ..., arr_N[M] -> pack into a binary blob bM
///
/// Each binary blob is inserted into a hash table.
///
Set set;
ColumnArray::Offset prev_off = 0;
for (ColumnArray::Offset i = 0; i < offsets.size(); ++i)
@ -303,29 +241,7 @@ bool FunctionArrayUniq::execute128bit(
set.clear();
ColumnArray::Offset off = offsets[i];
for (ColumnArray::Offset j = prev_off; j < off; ++j)
{
if (has_nullable_columns)
{
KeysNullMap<UInt128> bitmap{};
for (ColumnArray::Offset i = 0; i < columns.size(); ++i)
{
if (null_maps[i])
{
const auto & null_map = static_cast<const ColumnUInt8 &>(*null_maps[i]).getData();
if (null_map[j] == 1)
{
ColumnArray::Offset bucket = i / 8;
ColumnArray::Offset offset = i % 8;
bitmap[bucket] |= UInt8(1) << offset;
}
}
}
set.insert(packFixed<UInt128>(j, count, columns, key_sizes, bitmap));
}
else
set.insert(packFixed<UInt128>(j, count, columns, key_sizes));
}
set.insert(packFixed<UInt128>(j, count, columns, key_sizes));
res_values[i] = set.size();
prev_off = off;
@ -334,7 +250,7 @@ bool FunctionArrayUniq::execute128bit(
return true;
}
void FunctionArrayUniq::executeHashed(
bool FunctionArrayUniq::executeHashed(
const ColumnArray::Offsets & offsets,
const ColumnRawPtrs & columns,
ColumnUInt32::Container & res_values)
@ -356,6 +272,8 @@ void FunctionArrayUniq::executeHashed(
res_values[i] = set.size();
prev_off = off;
}
return true;
}

4
dbms/src/Functions/if.cpp
View File

@ -638,9 +638,7 @@ private:
static ColumnPtr materializeColumnIfConst(const ColumnPtr & column)
{
if (ColumnPtr res = column->convertToFullColumnIfConst())
return res;
return column;
return column->convertToFullColumnIfConst();
}
static ColumnPtr makeNullableColumnIfNot(const ColumnPtr & column)

6
dbms/src/Functions/materialize.cpp
View File

@ -34,11 +34,7 @@ public:
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override
{
const auto & src = block.getByPosition(arguments[0]).column;
if (ColumnPtr converted = src->convertToFullColumnIfConst())
block.getByPosition(result).column = converted;
else
block.getByPosition(result).column = src;
block.getByPosition(result).column = block.getByPosition(arguments[0]).column->convertToFullColumnIfConst();
}
};

11
dbms/src/Functions/tuple.cpp
View File

@ -65,14 +65,11 @@ public:
Columns tuple_columns(tuple_size);
for (size_t i = 0; i < tuple_size; ++i)
{
tuple_columns[i] = block.getByPosition(arguments[i]).column;
/** If tuple is mixed of constant and not constant columns,
* convert all to non-constant columns,
* because many places in code expect all non-constant columns in non-constant tuple.
*/
if (ColumnPtr converted = tuple_columns[i]->convertToFullColumnIfConst())
tuple_columns[i] = converted;
* convert all to non-constant columns,
* because many places in code expect all non-constant columns in non-constant tuple.
*/
tuple_columns[i] = block.getByPosition(arguments[i]).column->convertToFullColumnIfConst();
}
block.getByPosition(result).column = ColumnTuple::create(tuple_columns);
}

18
dbms/src/Interpreters/Aggregator.cpp
View File

@ -772,13 +772,8 @@ bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & re
/// Remember the columns we will work with
for (size_t i = 0; i < params.keys_size; ++i)
{
key_columns[i] = block.safeGetByPosition(params.keys[i]).column.get();
if (ColumnPtr converted = key_columns[i]->convertToFullColumnIfConst())
{
materialized_columns.push_back(converted);
key_columns[i] = materialized_columns.back().get();
}
materialized_columns.push_back(block.safeGetByPosition(params.keys[i]).column->convertToFullColumnIfConst());
key_columns[i] = materialized_columns.back().get();
if (const auto * low_cardinality_column = typeid_cast<const ColumnLowCardinality *>(key_columns[i]))
{
@ -797,13 +792,8 @@ bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & re
{
for (size_t j = 0; j < aggregate_columns[i].size(); ++j)
{
aggregate_columns[i][j] = block.safeGetByPosition(params.aggregates[i].arguments[j]).column.get();
if (ColumnPtr converted = aggregate_columns[i][j]->convertToFullColumnIfConst())
{
materialized_columns.push_back(converted);
aggregate_columns[i][j] = materialized_columns.back().get();
}
materialized_columns.push_back(block.safeGetByPosition(params.aggregates[i].arguments[j]).column->convertToFullColumnIfConst());
aggregate_columns[i][j] = materialized_columns.back().get();
if (auto * col_low_cardinality = typeid_cast<const ColumnLowCardinality *>(aggregate_columns[i][j]))
{

20
dbms/src/Interpreters/ExpressionActions.cpp
View File

@ -375,10 +375,7 @@ void ExpressionAction::execute(Block & block, bool dry_run) const
if (array_joined_columns.empty())
throw Exception("No arrays to join", ErrorCodes::LOGICAL_ERROR);
ColumnPtr any_array_ptr = block.getByName(*array_joined_columns.begin()).column;
if (ColumnPtr converted = any_array_ptr->convertToFullColumnIfConst())
any_array_ptr = converted;
ColumnPtr any_array_ptr = block.getByName(*array_joined_columns.begin()).column->convertToFullColumnIfConst();
const ColumnArray * any_array = typeid_cast<const ColumnArray *>(&*any_array_ptr);
if (!any_array)
throw Exception("ARRAY JOIN of not array: " + *array_joined_columns.begin(), ErrorCodes::TYPE_MISMATCH);
@ -416,10 +413,10 @@ void ExpressionAction::execute(Block & block, bool dry_run) const
Block tmp_block{src_col, column_of_max_length, {{}, src_col.type, {}}};
function_arrayResize->build({src_col, column_of_max_length})->execute(tmp_block, {0, 1}, 2, rows);
any_array_ptr = src_col.column = tmp_block.safeGetByPosition(2).column;
src_col.column = tmp_block.safeGetByPosition(2).column;
any_array_ptr = src_col.column->convertToFullColumnIfConst();
}
if (ColumnPtr converted = any_array_ptr->convertToFullColumnIfConst())
any_array_ptr = converted;
any_array = typeid_cast<const ColumnArray *>(&*any_array_ptr);
}
else if (array_join_is_left && !unaligned_array_join)
@ -434,10 +431,7 @@ void ExpressionAction::execute(Block & block, bool dry_run) const
non_empty_array_columns[name] = tmp_block.safeGetByPosition(1).column;
}
any_array_ptr = non_empty_array_columns.begin()->second;
if (ColumnPtr converted = any_array_ptr->convertToFullColumnIfConst())
any_array_ptr = converted;
any_array_ptr = non_empty_array_columns.begin()->second->convertToFullColumnIfConst();
any_array = &typeid_cast<const ColumnArray &>(*any_array_ptr);
}
@ -452,9 +446,7 @@ void ExpressionAction::execute(Block & block, bool dry_run) const
throw Exception("ARRAY JOIN of not array: " + current.name, ErrorCodes::TYPE_MISMATCH);
ColumnPtr array_ptr = (array_join_is_left && !unaligned_array_join) ? non_empty_array_columns[current.name] : current.column;
if (ColumnPtr converted = array_ptr->convertToFullColumnIfConst())
array_ptr = converted;
array_ptr = array_ptr->convertToFullColumnIfConst();
const ColumnArray & array = typeid_cast<const ColumnArray &>(*array_ptr);
if (!unaligned_array_join && !array.hasEqualOffsets(typeid_cast<const ColumnArray &>(*any_array_ptr)))

27
dbms/src/Interpreters/Join.cpp
View File

@ -437,14 +437,8 @@ bool Join::insertFromBlock(const Block & block)
/// Memoize key columns to work.
for (size_t i = 0; i < keys_size; ++i)
{
materialized_columns.emplace_back(recursiveRemoveLowCardinality(block.getByName(key_names_right[i]).column));
materialized_columns.emplace_back(recursiveRemoveLowCardinality(block.getByName(key_names_right[i]).column->convertToFullColumnIfConst()));
key_columns[i] = materialized_columns.back().get();
if (ColumnPtr converted = key_columns[i]->convertToFullColumnIfConst())
{
materialized_columns.emplace_back(converted);
key_columns[i] = materialized_columns.back().get();
}
}
/// We will insert to the map only keys, where all components are not NULL.
@ -483,11 +477,7 @@ bool Join::insertFromBlock(const Block & block)
/// Rare case, when joined columns are constant. To avoid code bloat, simply materialize them.
for (size_t i = 0; i < size; ++i)
{
ColumnPtr col = stored_block->safeGetByPosition(i).column;
if (ColumnPtr converted = col->convertToFullColumnIfConst())
stored_block->safeGetByPosition(i).column = converted;
}
stored_block->safeGetByPosition(i).column = stored_block->safeGetByPosition(i).column->convertToFullColumnIfConst();
/// In case of LEFT and FULL joins, if use_nulls, convert joined columns to Nullable.
if (use_nulls && (kind == ASTTableJoin::Kind::Left || kind == ASTTableJoin::Kind::Full))
@ -685,14 +675,8 @@ void Join::joinBlockImpl(
/// Memoize key columns to work with.
for (size_t i = 0; i < keys_size; ++i)
{
materialized_columns.emplace_back(recursiveRemoveLowCardinality(block.getByName(key_names_left[i]).column));
materialized_columns.emplace_back(recursiveRemoveLowCardinality(block.getByName(key_names_left[i]).column->convertToFullColumnIfConst()));
key_columns[i] = materialized_columns.back().get();
if (ColumnPtr converted = key_columns[i]->convertToFullColumnIfConst())
{
materialized_columns.emplace_back(converted);
key_columns[i] = materialized_columns.back().get();
}
}
/// Keys with NULL value in any column won't join to anything.
@ -710,10 +694,7 @@ void Join::joinBlockImpl(
{
for (size_t i = 0; i < existing_columns; ++i)
{
auto & col = block.getByPosition(i).column;
if (ColumnPtr converted = col->convertToFullColumnIfConst())
col = converted;
block.getByPosition(i).column = block.getByPosition(i).column->convertToFullColumnIfConst();
/// If use_nulls, convert left columns (except keys) to Nullable.
if (use_nulls)

34
dbms/src/Interpreters/Set.cpp
View File

@ -121,15 +121,10 @@ void Set::setHeader(const Block & block)
/// Remember the columns we will work with
for (size_t i = 0; i < keys_size; ++i)
{
key_columns.emplace_back(block.safeGetByPosition(i).column.get());
materialized_columns.emplace_back(block.safeGetByPosition(i).column->convertToFullColumnIfConst());
key_columns.emplace_back(materialized_columns.back().get());
data_types.emplace_back(block.safeGetByPosition(i).type);
if (ColumnPtr converted = key_columns.back()->convertToFullColumnIfConst())
{
materialized_columns.emplace_back(converted);
key_columns.back() = materialized_columns.back().get();
}
/// Convert low cardinality column to full.
if (auto * low_cardinality_type = typeid_cast<const DataTypeLowCardinality *>(data_types.back().get()))
{
@ -175,20 +170,8 @@ bool Set::insertFromBlock(const Block & block)
/// Remember the columns we will work with
for (size_t i = 0; i < keys_size; ++i)
{
key_columns.emplace_back(block.safeGetByPosition(i).column.get());
if (ColumnPtr converted = key_columns.back()->convertToFullColumnIfConst())
{
materialized_columns.emplace_back(converted);
key_columns.back() = materialized_columns.back().get();
}
/// Convert low cardinality column to full.
if (key_columns.back()->lowCardinality())
{
materialized_columns.emplace_back(key_columns.back()->convertToFullColumnIfLowCardinality());
key_columns.back() = materialized_columns.back().get();
}
materialized_columns.emplace_back(block.safeGetByPosition(i).column->convertToFullColumnIfConst()->convertToFullColumnIfLowCardinality());
key_columns.emplace_back(materialized_columns.back().get());
}
size_t rows = block.rows();
@ -365,18 +348,13 @@ ColumnPtr Set::execute(const Block & block, bool negative) const
for (size_t i = 0; i < num_key_columns; ++i)
{
key_columns.push_back(block.safeGetByPosition(i).column.get());
if (!removeNullable(data_types[i])->equals(*removeNullable(block.safeGetByPosition(i).type)))
throw Exception("Types of column " + toString(i + 1) + " in section IN don't match: "
+ data_types[i]->getName() + " on the right, " + block.safeGetByPosition(i).type->getName() +
" on the left.", ErrorCodes::TYPE_MISMATCH);
if (ColumnPtr converted = key_columns.back()->convertToFullColumnIfConst())
{
materialized_columns.emplace_back(converted);
key_columns.back() = materialized_columns.back().get();
}
materialized_columns.emplace_back(block.safeGetByPosition(i).column->convertToFullColumnIfConst());
key_columns.emplace_back() = materialized_columns.back().get();
}
/// We will check existence in Set only for keys, where all components are not NULL.

3
dbms/src/Interpreters/evaluateMissingDefaults.cpp
View File

@ -67,8 +67,7 @@ void evaluateMissingDefaults(Block & block,
if (copy_block.has(col->name))
{
auto evaluated_col = copy_block.getByName(col->name);
if (ColumnPtr converted = evaluated_col.column->convertToFullColumnIfConst())
evaluated_col.column = converted;
evaluated_col.column = evaluated_col.column->convertToFullColumnIfConst();
block.insert(pos, std::move(evaluated_col));
}

4
dbms/src/Storages/VirtualColumnUtils.cpp
View File

@ -166,9 +166,7 @@ void filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & c
/// Filter the block.
String filter_column_name = expression_ast->getColumnName();
ColumnPtr filter_column = block_with_filter.getByName(filter_column_name).column;
if (ColumnPtr converted = filter_column->convertToFullColumnIfConst())
filter_column = converted;
ColumnPtr filter_column = block_with_filter.getByName(filter_column_name).column->convertToFullColumnIfConst();
const IColumn::Filter & filter = typeid_cast<const ColumnUInt8 &>(*filter_column).getData();
for (size_t i = 0; i < block.columns(); ++i)

2
dbms/tests/queries/0_stateless/00808_array_enumerate_segfault.reference Normal file
View File

@ -0,0 +1,2 @@
[]
[1]

4
dbms/tests/queries/0_stateless/00808_array_enumerate_segfault.sql Normal file
View File

@ -0,0 +1,4 @@
SET send_logs_level = 'none';
SELECT arrayEnumerateUniq(anyHeavy([]), []);
SELECT arrayEnumerateDense([], [sequenceCount(NULL)]); -- { serverError 190 }
SELECT arrayEnumerateDense([STDDEV_SAMP(NULL, 910947.571364)], [NULL]);

2
dbms/tests/queries/bugs/fuzzy.sql
View File

@ -3,9 +3,7 @@ SELECT globalNotIn(['"wh'], [NULL]);
SELECT globalIn([''], [NULL])
SELECT ( SELECT toDecimal128([], rowNumberInBlock()) ) , lcm('', [[(CAST(('>A') AS String))]]);
SELECT truncate(895, -16);
SELECT arrayEnumerateUniq(anyHeavy([]), []);
SELECT notIn([['']], [[NULL]]);
SELECT subtractDays((CAST((-5263074.47) AS DateTime)), -737895);
SELECT quantileDeterministic([], findClusterIndex(( SELECT subtractDays((CAST((566450.398706) AS DateTime)), 54) ) )), '\0', [];
SELECT addDays((CAST((96.338) AS DateTime)), -3);
SELECT arrayEnumerateDense([], [sequenceCount(NULL)]);