From e1236340b5ff81436396f7d30247875a3216c32a Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 16 Nov 2022 12:27:01 -0300 Subject: [PATCH] Flatten list type arrow chunks on parsing --- src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index e9b01ec7dda..7642ce18992 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -329,9 +329,8 @@ static ColumnPtr readOffsetsFromArrowListColumn(std::shared_ptr(*(arrow_column->chunk(chunk_i))); auto arrow_offsets_array = list_chunk.offsets(); auto & arrow_offsets = dynamic_cast(*arrow_offsets_array); - auto start = offsets_data.back(); for (int64_t i = 1; i < arrow_offsets.length(); ++i) - offsets_data.emplace_back(start + arrow_offsets.Value(i)); + offsets_data.emplace_back(arrow_offsets.Value(i)); } return offsets_column; } @@ -467,8 +466,12 @@ static std::shared_ptr getNestedArrowColumn(std::shared_ptr for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i) { arrow::ListArray & list_chunk = dynamic_cast(*(arrow_column->chunk(chunk_i))); - std::shared_ptr chunk = list_chunk.values(); - array_vector.emplace_back(std::move(chunk)); + auto flatten_result = list_chunk.Flatten(); + if (flatten_result.ok()) { + array_vector.emplace_back(flatten_result.ValueOrDie()); + } else { + throw Exception(ErrorCodes::INCORRECT_DATA, "Failed to flatten chunk '{}' of column of type '{}' ", chunk_i, arrow_column->type()->id()); + } } return std::make_shared(array_vector); }