thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 07:01:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #41458 from ClickHouse/fix-bug-orc

Browse Source 
Fix very strange behavior of Apache ORC
This commit is contained in:
Alexey Milovidov 2022-09-19 10:43:17 +03:00 committed by GitHub
commit b9bd2c8373
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 19 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp
View File

@ -55,7 +55,7 @@ ORCBlockOutputFormat::ORCBlockOutputFormat(WriteBuffer & out_, const Block & hea
data_types.push_back(recursiveRemoveLowCardinality(type));
}
ORC_UNIQUE_PTR<orc::Type> ORCBlockOutputFormat::getORCType(const DataTypePtr & type)
std::unique_ptr<orc::Type> ORCBlockOutputFormat::getORCType(const DataTypePtr & type)
{
switch (type->getTypeId())
{
@ -398,17 +398,21 @@ void ORCBlockOutputFormat::writeColumn(
const auto & list_column = assert_cast<const ColumnArray &>(column);
auto nested_type = assert_cast<const DataTypeArray &>(*type).getNestedType();
const ColumnArray::Offsets & offsets = list_column.getOffsets();
list_orc_column.resize(list_column.size());
size_t column_size = list_column.size();
list_orc_column.resize(column_size);
/// The length of list i in ListVectorBatch is offsets[i+1] - offsets[i].
list_orc_column.offsets[0] = 0;
for (size_t i = 0; i != list_column.size(); ++i)
for (size_t i = 0; i != column_size; ++i)
{
list_orc_column.offsets[i + 1] = offsets[i];
list_orc_column.notNull[i] = 1;
}
orc::ColumnVectorBatch & nested_orc_column = *list_orc_column.elements;
writeColumn(nested_orc_column, list_column.getData(), nested_type, null_bytemap);
list_orc_column.numElements = list_column.size();
list_orc_column.numElements = column_size;
break;
}
case TypeIndex::Tuple:
@ -429,10 +433,12 @@ void ORCBlockOutputFormat::writeColumn(
const auto & map_type = assert_cast<const DataTypeMap &>(*type);
const ColumnArray::Offsets & offsets = list_column.getOffsets();
size_t column_size = list_column.size();
map_orc_column.resize(list_column.size());
/// The length of list i in ListVectorBatch is offsets[i+1] - offsets[i].
map_orc_column.offsets[0] = 0;
for (size_t i = 0; i != list_column.size(); ++i)
for (size_t i = 0; i != column_size; ++i)
{
map_orc_column.offsets[i + 1] = offsets[i];
map_orc_column.notNull[i] = 1;
@ -447,7 +453,7 @@ void ORCBlockOutputFormat::writeColumn(
auto value_type = map_type.getValueType();
writeColumn(values_orc_column, *nested_columns[1], value_type, null_bytemap);
map_orc_column.numElements = list_column.size();
map_orc_column.numElements = column_size;
break;
}
default:
@ -461,8 +467,9 @@ size_t ORCBlockOutputFormat::getColumnSize(const IColumn & column, DataTypePtr &
{
auto nested_type = assert_cast<const DataTypeArray &>(*type).getNestedType();
const IColumn & nested_column = assert_cast<const ColumnArray &>(column).getData();
return getColumnSize(nested_column, nested_type);
return std::max(column.size(), getColumnSize(nested_column, nested_type));
}
return column.size();
}
@ -472,7 +479,6 @@ size_t ORCBlockOutputFormat::getMaxColumnSize(Chunk & chunk)
size_t max_column_size = 0;
for (size_t i = 0; i != columns_num; ++i)
max_column_size = std::max(max_column_size, getColumnSize(*chunk.getColumns()[i], data_types[i]));
return max_column_size;
}
@ -485,7 +491,8 @@ void ORCBlockOutputFormat::consume(Chunk chunk)
size_t rows_num = chunk.getNumRows();
/// getMaxColumnSize is needed to write arrays.
/// The size of the batch must be no less than total amount of array elements.
/// The size of the batch must be no less than total amount of array elements
/// and no less than the number of rows (ORC writes a null bit for every row).
std::unique_ptr<orc::ColumnVectorBatch> batch = writer->createRowBatch(getMaxColumnSize(chunk));
orc::StructVectorBatch & root = dynamic_cast<orc::StructVectorBatch &>(*batch);

BIN
tests/queries/0_stateless/02426_orc_bug.reference Normal file
View File

Binary file not shown.

3
tests/queries/0_stateless/02426_orc_bug.sql Normal file
View File

@ -0,0 +1,3 @@
-- Tags: no-fasttest
SELECT arrayJoin([[], [1]]) FORMAT ORC;