thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 16:50:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix segfault in Avro with LowCardinality(Nullable)

Browse Source
This commit is contained in:
Pavel Kruglov 2021-09-17 16:11:44 +03:00
parent 2d9fa85c0a
commit 48a3a2f75c
3 changed files with 2042 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/Processors/Formats/Impl/AvroRowInputFormat.cpp
View File

@ -35,6 +35,7 @@
#include <Columns/ColumnNullable.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnLowCardinality.h>
#include <avro/Compiler.hh>
#include <avro/DataFile.hh>
@ -178,7 +179,21 @@ static std::string nodeName(avro::NodePtr node)
AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::NodePtr root_node, DataTypePtr target_type)
{
const WhichDataType target = removeLowCardinality(target_type);
if (target_type->lowCardinality())
{
const auto * lc_type = assert_cast<const DataTypeLowCardinality *>(target_type.get());
auto dict_deserialize = createDeserializeFn(root_node, lc_type->getDictionaryType());
return [dict_deserialize](IColumn & column, avro::Decoder & decoder)
{
auto & lc_column = assert_cast<ColumnLowCardinality &>(column);
auto tmp_column = lc_column.getDictionary().getNestedColumn()->cloneEmpty();
dict_deserialize(*tmp_column, decoder);
lc_column.insertFromFullColumn(*tmp_column, 0);
};
}
const WhichDataType target = WhichDataType(target_type);
switch (root_node->type())
{

2010
tests/queries/0_stateless/01543_avro_deserialization_with_lc.reference
View File

File diff suppressed because it is too large Load Diff

18
tests/queries/0_stateless/01543_avro_deserialization_with_lc.sh
View File

@ -5,10 +5,24 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS test_01543 (value LowCardinality(String)) ENGINE=Memory()"
$CLICKHOUSE_CLIENT --query "INSERT INTO test_01543 SELECT toString(number) FROM numbers(1000)"
USER_FILES_PATH=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}')
$CLICKHOUSE_CLIENT --multiquery --query "
SET allow_suspicious_low_cardinality_types=1;
CREATE TABLE IF NOT EXISTS test_01543 (value LowCardinality(String), value2 LowCardinality(UInt64)) ENGINE=Memory();
"
$CLICKHOUSE_CLIENT --query "INSERT INTO test_01543 SELECT toString(number), number FROM numbers(1000)"
$CLICKHOUSE_CLIENT -q "SELECT * FROM test_01543 FORMAT Avro" |
$CLICKHOUSE_CLIENT -q "INSERT INTO test_01543 FORMAT Avro";
$CLICKHOUSE_CLIENT -q "SELECT * FROM test_01543";
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_01543"
$CLICKHOUSE_CLIENT --query "SELECT number % 2 ? number: NULL as x from numbers(10) FORMAT Avro" > $USER_FILES_PATH/test_01543.avro
$CLICKHOUSE_CLIENT --query "SELECT * FROM file('test_01543.avro', 'Avro', 'x LowCardinality(Nullable(UInt64))')"
rm $USER_FILES_PATH/test_01543.avro