thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-10-10 02:20:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7062 from ClickHouse/fix-msan-for-low-cardinality

Browse Source 
Fix msan for LowCardinality

(cherry picked from commit a314a36feb)
This commit is contained in:
alexey-milovidov 2019-09-24 02:11:15 +03:00 committed by stavrolia
parent 9533ff12dd
commit af5099b869
3 changed files with 35 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dbms/src/Core/Block.cpp
View File

@ -219,11 +219,14 @@ size_t Block::getPositionByName(const std::string & name) const
} }
void Block::checkNumberOfRows() const void Block::checkNumberOfRows(bool allow_null_columns) const
{ {
ssize_t rows = -1; ssize_t rows = -1;
for (const auto & elem : data) for (const auto & elem : data)
{ {
if (!elem.column && allow_null_columns)
continue;
if (!elem.column) if (!elem.column)
throw Exception("Column " + elem.name + " in block is nullptr, in method checkNumberOfRows." throw Exception("Column " + elem.name + " in block is nullptr, in method checkNumberOfRows."
, ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); , ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);

2
dbms/src/Core/Block.h
View File

@ -90,7 +90,7 @@ public:
size_t columns() const { return data.size(); } size_t columns() const { return data.size(); }
/// Checks that every column in block is not nullptr and has same number of elements. /// Checks that every column in block is not nullptr and has same number of elements.
void checkNumberOfRows() const; void checkNumberOfRows(bool allow_null_columns = false) const;
/// Approximate number of bytes in memory - for profiling and limits. /// Approximate number of bytes in memory - for profiling and limits.
size_t bytes() const; size_t bytes() const;

50
dbms/src/Functions/IFunction.cpp
View File

@ -337,19 +337,43 @@ static ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes(
size_t num_rows = input_rows_count; size_t num_rows = input_rows_count;
ColumnPtr indexes; ColumnPtr indexes;
/// Find first LowCardinality column and replace it to nested dictionary.
for (auto arg : args) for (auto arg : args)
{ {
ColumnWithTypeAndName & column = block.getByPosition(arg); ColumnWithTypeAndName & column = block.getByPosition(arg);
if (auto * low_cardinality_column = checkAndGetColumn<ColumnLowCardinality>(column.column.get())) if (auto * low_cardinality_column = checkAndGetColumn<ColumnLowCardinality>(column.column.get()))
{ {
/// Single LowCardinality column is supported now.
if (indexes) if (indexes)
throw Exception("Expected single dictionary argument for function.", ErrorCodes::LOGICAL_ERROR); throw Exception("Expected single dictionary argument for function.", ErrorCodes::LOGICAL_ERROR);
indexes = low_cardinality_column->getIndexesPtr(); auto * low_cardinality_type = checkAndGetDataType<DataTypeLowCardinality>(column.type.get());
num_rows = low_cardinality_column->getDictionary().size();
if (!low_cardinality_type)
throw Exception("Incompatible type for low cardinality column: " + column.type->getName(),
ErrorCodes::LOGICAL_ERROR);
if (can_be_executed_on_default_arguments)
{
/// Normal case, when function can be executed on values's default.
column.column = low_cardinality_column->getDictionary().getNestedColumn();
indexes = low_cardinality_column->getIndexesPtr();
}
else
{
/// Special case when default value can't be used. Example: 1 % LowCardinality(Int).
/// LowCardinality always contains default, so 1 % 0 will throw exception in normal case.
auto dict_encoded = low_cardinality_column->getMinimalDictionaryEncodedColumn(0, low_cardinality_column->size());
column.column = dict_encoded.dictionary;
indexes = dict_encoded.indexes;
}
num_rows = column.column->size();
column.type = low_cardinality_type->getDictionaryType();
} }
} }
/// Change size of constants.
for (auto arg : args) for (auto arg : args)
{ {
ColumnWithTypeAndName & column = block.getByPosition(arg); ColumnWithTypeAndName & column = block.getByPosition(arg);
@ -358,26 +382,12 @@ static ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes(
column.column = column_const->removeLowCardinality()->cloneResized(num_rows); column.column = column_const->removeLowCardinality()->cloneResized(num_rows);
column.type = removeLowCardinality(column.type); column.type = removeLowCardinality(column.type);
} }
else if (auto * low_cardinality_column = checkAndGetColumn<ColumnLowCardinality>(column.column.get()))
{
auto * low_cardinality_type = checkAndGetDataType<DataTypeLowCardinality>(column.type.get());
if (!low_cardinality_type)
throw Exception("Incompatible type for low cardinality column: " + column.type->getName(),
ErrorCodes::LOGICAL_ERROR);
if (can_be_executed_on_default_arguments)
column.column = low_cardinality_column->getDictionary().getNestedColumn();
else
{
auto dict_encoded = low_cardinality_column->getMinimalDictionaryEncodedColumn(0, low_cardinality_column->size());
column.column = dict_encoded.dictionary;
indexes = dict_encoded.indexes;
}
column.type = low_cardinality_type->getDictionaryType();
}
} }
#ifndef NDEBUG
block.checkNumberOfRows(true);
#endif
return indexes; return indexes;
} }