some changes

This commit is contained in:
lgbo-ustc 2022-06-21 11:29:55 +08:00
parent c13bf03fe0
commit fc641d9ce4
3 changed files with 16 additions and 7 deletions

View File

@ -4,6 +4,7 @@
#include <Common/typeid_cast.h> #include <Common/typeid_cast.h>
#include <Common/assert_cast.h> #include <Common/assert_cast.h>
#include <Common/StringUtils/StringUtils.h> #include <Common/StringUtils/StringUtils.h>
#include "Columns/IColumn.h"
#include <DataTypes/DataTypeArray.h> #include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeTuple.h> #include <DataTypes/DataTypeTuple.h>
@ -119,7 +120,11 @@ Block flatten(const Block & block)
{ {
const DataTypes & element_types = type_tuple->getElements(); const DataTypes & element_types = type_tuple->getElements();
const Strings & names = type_tuple->getElementNames(); const Strings & names = type_tuple->getElementNames();
const ColumnTuple * column_tuple = typeid_cast<const ColumnTuple *>(elem.column.get()); const ColumnTuple * column_tuple;
if(isColumnConst(*elem.column))
column_tuple = typeid_cast<const ColumnTuple *>(&assert_cast<const ColumnConst &>(*elem.column).getDataColumn());
else
column_tuple = typeid_cast<const ColumnTuple *>(elem.column.get());
size_t tuple_size = column_tuple->tupleSize(); size_t tuple_size = column_tuple->tupleSize();
for (size_t i = 0; i < tuple_size; ++i) for (size_t i = 0; i < tuple_size; ++i)
{ {
@ -306,7 +311,7 @@ std::optional<ColumnWithTypeAndName> NestedColumnExtractHelper::extractColumn(
{ {
if (nested_table->has(new_column_name_prefix, case_insentive)) if (nested_table->has(new_column_name_prefix, case_insentive))
{ {
ColumnWithTypeAndName column = nested_table->getByName(new_column_name_prefix, case_insentive); ColumnWithTypeAndName column = *nested_table->findByName(new_column_name_prefix, case_insentive);
if (case_insentive) if (case_insentive)
column.name = original_column_name; column.name = original_column_name;
return {column}; return {column};

View File

@ -18,8 +18,9 @@ namespace Nested
/// Returns the prefix of the name to the first '.'. Or the name is unchanged if there is no dot. /// Returns the prefix of the name to the first '.'. Or the name is unchanged if there is no dot.
std::string extractTableName(const std::string & nested_name); std::string extractTableName(const std::string & nested_name);
/// Replace Array(Tuple(...)) columns to a multiple of Array columns in a form of `column_name.element_name`. /// Flat a column of nested type into columns
/// only for named tuples that actually represent Nested structures. /// 1) For named tuplest Tuple(x .., y ..., ...), replace it with t.x ..., t.y ... , ...
/// 2) For an Array with named Tuple element column, a Array(Tuple(x ..., y ..., ...)), replace it with multiple Array Columns, a.x ..., a.y ..., ...
Block flatten(const Block & block); Block flatten(const Block & block);
/// Collect Array columns in a form of `column_name.element_name` to single Array(Tuple(...)) column. /// Collect Array columns in a form of `column_name.element_name` to single Array(Tuple(...)) column.
@ -35,6 +36,9 @@ namespace Nested
std::unordered_set<String> getAllTableNames(const Block & block, bool to_lower_case = false); std::unordered_set<String> getAllTableNames(const Block & block, bool to_lower_case = false);
} }
/// Use this class to extract element columns from columns of nested type in a block, e.g. named Tuple.
/// It can extract a column from a multiple nested type column, e.g. named Tuple in named Tuple
/// Keeps some intermediate datas to avoid rebuild them multi-times.
class NestedColumnExtractHelper class NestedColumnExtractHelper
{ {
public: public:

View File

@ -565,8 +565,8 @@ HiveFiles StorageHive::collectHiveFilesFromPartition(
const ContextPtr & context_, const ContextPtr & context_,
PruneLevel prune_level) const PruneLevel prune_level) const
{ {
//LOG_DEBUG( LOG_DEBUG(
// log, "Collect hive files from partition {}, prune_level:{}", boost::join(partition.values, ","), pruneLevelToString(prune_level)); log, "Collect hive files from partition {}, prune_level:{}", boost::join(partition.values, ","), pruneLevelToString(prune_level));
/// Skip partition "__HIVE_DEFAULT_PARTITION__" /// Skip partition "__HIVE_DEFAULT_PARTITION__"
bool has_default_partition = false; bool has_default_partition = false;
@ -794,7 +794,7 @@ Pipe StorageHive::read(
auto subset_column = nested_columns_extractor.extractColumn(column); auto subset_column = nested_columns_extractor.extractColumn(column);
if (subset_column) if (subset_column)
{ {
sample_block.insert(*subset_column); sample_block.insert(std::move(*subset_column));
continue; continue;
} }
} }