From 6325d4d9b0cf45386dfb5f4f31d941fe4af0ae26 Mon Sep 17 00:00:00 2001 From: feng lv Date: Sat, 5 Feb 2022 08:10:28 +0000 Subject: [PATCH] continue of #34317 fix fix --- src/DataTypes/DataTypeLowCardinality.h | 2 +- src/Storages/FileLog/StorageFileLog.cpp | 5 ++++- src/Storages/HDFS/StorageHDFS.cpp | 19 ++++++++++++------- src/Storages/HDFS/StorageHDFSCluster.cpp | 5 ++--- src/Storages/Hive/StorageHive.cpp | 23 ++++++++++++++++------- src/Storages/Kafka/StorageKafka.cpp | 12 ++++++------ src/Storages/StorageDistributed.cpp | 14 +++++++------- src/Storages/StorageS3.cpp | 24 ++++++++++++++++-------- src/Storages/StorageS3Cluster.cpp | 5 ++--- 9 files changed, 66 insertions(+), 43 deletions(-) diff --git a/src/DataTypes/DataTypeLowCardinality.h b/src/DataTypes/DataTypeLowCardinality.h index 38b2109eec6..57f67ddad7a 100644 --- a/src/DataTypes/DataTypeLowCardinality.h +++ b/src/DataTypes/DataTypeLowCardinality.h @@ -13,7 +13,7 @@ private: DataTypePtr dictionary_type; public: - DataTypeLowCardinality(DataTypePtr dictionary_type_); + explicit DataTypeLowCardinality(DataTypePtr dictionary_type_); const DataTypePtr & getDictionaryType() const { return dictionary_type; } diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index 1ae8a5bb22d..f89caaec685 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -966,7 +967,9 @@ bool StorageFileLog::updateFileInfos() NamesAndTypesList StorageFileLog::getVirtuals() const { - return NamesAndTypesList{{"_filename", std::make_shared()}, {"_offset", std::make_shared()}}; + return NamesAndTypesList{ + {"_filename", std::make_shared(std::make_shared())}, + {"_offset", std::make_shared()}}; } Names StorageFileLog::getVirtualColumnNames() diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 294ab2f6d4e..d40838ad141 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -258,9 +258,15 @@ Block HDFSSource::getHeader(const StorageMetadataPtr & metadata_snapshot, bool n auto header = metadata_snapshot->getSampleBlock(); /// Note: AddingDefaultsBlockInputStream doesn't change header. if (need_path_column) - header.insert({DataTypeString().createColumn(), std::make_shared(), "_path"}); + header.insert( + {DataTypeLowCardinality{std::make_shared()}.createColumn(), + std::make_shared(std::make_shared()), + "_path"}); if (need_file_column) - header.insert({DataTypeString().createColumn(), std::make_shared(), "_file"}); + header.insert( + {DataTypeLowCardinality{std::make_shared()}.createColumn(), + std::make_shared(std::make_shared()), + "_file"}); return header; } @@ -378,7 +384,7 @@ Chunk HDFSSource::generate() /// Enrich with virtual columns. if (need_path_column) { - auto column = DataTypeString().createColumnConst(num_rows, current_path); + auto column = DataTypeLowCardinality{std::make_shared()}.createColumnConst(num_rows, current_path); columns.push_back(column->convertToFullColumnIfConst()); } @@ -387,7 +393,7 @@ Chunk HDFSSource::generate() size_t last_slash_pos = current_path.find_last_of('/'); auto file_name = current_path.substr(last_slash_pos + 1); - auto column = DataTypeString().createColumnConst(num_rows, std::move(file_name)); + auto column = DataTypeLowCardinality{std::make_shared()}.createColumnConst(num_rows, std::move(file_name)); columns.push_back(column->convertToFullColumnIfConst()); } @@ -689,9 +695,8 @@ void registerStorageHDFS(StorageFactory & factory) NamesAndTypesList StorageHDFS::getVirtuals() const { return NamesAndTypesList{ - {"_path", std::make_shared()}, - {"_file", std::make_shared()} - }; + {"_path", std::make_shared(std::make_shared())}, + {"_file", std::make_shared(std::make_shared())}}; } } diff --git a/src/Storages/HDFS/StorageHDFSCluster.cpp b/src/Storages/HDFS/StorageHDFSCluster.cpp index ba1cc045fbf..dfe1ea6ffd3 100644 --- a/src/Storages/HDFS/StorageHDFSCluster.cpp +++ b/src/Storages/HDFS/StorageHDFSCluster.cpp @@ -138,9 +138,8 @@ QueryProcessingStage::Enum StorageHDFSCluster::getQueryProcessingStage( NamesAndTypesList StorageHDFSCluster::getVirtuals() const { return NamesAndTypesList{ - {"_path", std::make_shared()}, - {"_file", std::make_shared()} - }; + {"_path", std::make_shared(std::make_shared())}, + {"_file", std::make_shared(std::make_shared())}}; } diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index af357e13ca7..3040ad23283 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -76,9 +76,15 @@ public: static Block getHeader(Block header, const SourcesInfoPtr & source_info) { if (source_info->need_path_column) - header.insert({DataTypeString().createColumn(), std::make_shared(), "_path"}); + header.insert( + {DataTypeLowCardinality{std::make_shared()}.createColumn(), + std::make_shared(std::make_shared()), + "_path"}); if (source_info->need_file_column) - header.insert({DataTypeString().createColumn(), std::make_shared(), "_file"}); + header.insert( + {DataTypeLowCardinality{std::make_shared()}.createColumn(), + std::make_shared(std::make_shared()), + "_file"}); return header; } @@ -87,9 +93,9 @@ public: { ColumnsDescription columns_description{header.getNamesAndTypesList()}; if (source_info->need_path_column) - columns_description.add({"_path", std::make_shared()}); + columns_description.add({"_path", std::make_shared(std::make_shared())}); if (source_info->need_file_column) - columns_description.add({"_file", std::make_shared()}); + columns_description.add({"_file", std::make_shared(std::make_shared())}); return columns_description; } @@ -211,7 +217,7 @@ public: /// Enrich with virtual columns. if (source_info->need_path_column) { - auto column = DataTypeString().createColumnConst(num_rows, current_path); + auto column = DataTypeLowCardinality{std::make_shared()}.createColumnConst(num_rows, current_path); columns.push_back(column->convertToFullColumnIfConst()); } @@ -220,7 +226,8 @@ public: size_t last_slash_pos = current_path.find_last_of('/'); auto file_name = current_path.substr(last_slash_pos + 1); - auto column = DataTypeString().createColumnConst(num_rows, std::move(file_name)); + auto column + = DataTypeLowCardinality{std::make_shared()}.createColumnConst(num_rows, std::move(file_name)); columns.push_back(column->convertToFullColumnIfConst()); } return Chunk(std::move(columns), num_rows); @@ -633,7 +640,9 @@ SinkToStoragePtr StorageHive::write(const ASTPtr & /*query*/, const StorageMetad NamesAndTypesList StorageHive::getVirtuals() const { - return NamesAndTypesList{{"_path", std::make_shared()}, {"_file", std::make_shared()}}; + return NamesAndTypesList{ + {"_path", std::make_shared(std::make_shared())}, + {"_file", std::make_shared(std::make_shared())}}; } void registerStorageHive(StorageFactory & factory) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 6101eb04af6..30acbcdf62b 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -16,13 +17,15 @@ #include #include #include +#include #include #include #include #include -#include #include #include +#include +#include #include #include #include @@ -36,8 +39,6 @@ #include #include #include -#include -#include namespace DB @@ -807,15 +808,14 @@ void registerStorageKafka(StorageFactory & factory) NamesAndTypesList StorageKafka::getVirtuals() const { auto result = NamesAndTypesList{ - {"_topic", std::make_shared()}, + {"_topic", std::make_shared(std::make_shared())}, {"_key", std::make_shared()}, {"_offset", std::make_shared()}, {"_partition", std::make_shared()}, {"_timestamp", std::make_shared(std::make_shared())}, {"_timestamp_ms", std::make_shared(std::make_shared(3))}, {"_headers.name", std::make_shared(std::make_shared())}, - {"_headers.value", std::make_shared(std::make_shared())} - }; + {"_headers.value", std::make_shared(std::make_shared())}}; if (kafka_settings->kafka_handle_error_mode == HandleKafkaErrorMode::STREAM) { result.push_back({"_raw_message", std::make_shared()}); diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 543ec9ee14a..da648aa4e5c 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -302,13 +302,13 @@ NamesAndTypesList StorageDistributed::getVirtuals() const /// NOTE This is weird. Most of these virtual columns are part of MergeTree /// tables info. But Distributed is general-purpose engine. return NamesAndTypesList{ - NameAndTypePair("_table", std::make_shared()), - NameAndTypePair("_part", std::make_shared()), - NameAndTypePair("_part_index", std::make_shared()), - NameAndTypePair("_part_uuid", std::make_shared()), - NameAndTypePair("_partition_id", std::make_shared()), - NameAndTypePair("_sample_factor", std::make_shared()), - NameAndTypePair("_shard_num", std::make_shared()), /// deprecated + NameAndTypePair("_table", std::make_shared()), + NameAndTypePair("_part", std::make_shared()), + NameAndTypePair("_part_index", std::make_shared()), + NameAndTypePair("_part_uuid", std::make_shared()), + NameAndTypePair("_partition_id", std::make_shared()), + NameAndTypePair("_sample_factor", std::make_shared()), + NameAndTypePair("_shard_num", std::make_shared()), /// deprecated }; } diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 2c373b01c91..77d4952291c 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -209,9 +209,15 @@ String StorageS3Source::KeysIterator::next() Block StorageS3Source::getHeader(Block sample_block, bool with_path_column, bool with_file_column) { if (with_path_column) - sample_block.insert({DataTypeString().createColumn(), std::make_shared(), "_path"}); + sample_block.insert( + {DataTypeLowCardinality{std::make_shared()}.createColumn(), + std::make_shared(std::make_shared()), + "_path"}); if (with_file_column) - sample_block.insert({DataTypeString().createColumn(), std::make_shared(), "_file"}); + sample_block.insert( + {DataTypeLowCardinality{std::make_shared()}.createColumn(), + std::make_shared(std::make_shared()), + "_file"}); return sample_block; } @@ -305,12 +311,15 @@ Chunk StorageS3Source::generate() UInt64 num_rows = chunk.getNumRows(); if (with_path_column) - chunk.addColumn(DataTypeString().createColumnConst(num_rows, file_path)->convertToFullColumnIfConst()); + chunk.addColumn(DataTypeLowCardinality{std::make_shared()} + .createColumnConst(num_rows, file_path) + ->convertToFullColumnIfConst()); if (with_file_column) { size_t last_slash_pos = file_path.find_last_of('/'); - chunk.addColumn(DataTypeString().createColumnConst(num_rows, file_path.substr( - last_slash_pos + 1))->convertToFullColumnIfConst()); + chunk.addColumn(DataTypeLowCardinality{std::make_shared()} + .createColumnConst(num_rows, file_path.substr(last_slash_pos + 1)) + ->convertToFullColumnIfConst()); } return chunk; @@ -961,9 +970,8 @@ void registerStorageCOS(StorageFactory & factory) NamesAndTypesList StorageS3::getVirtuals() const { return NamesAndTypesList{ - {"_path", std::make_shared()}, - {"_file", std::make_shared()} - }; + {"_path", std::make_shared(std::make_shared())}, + {"_file", std::make_shared(std::make_shared())}}; } bool StorageS3::supportsPartitionBy() const diff --git a/src/Storages/StorageS3Cluster.cpp b/src/Storages/StorageS3Cluster.cpp index 659071b392d..762eb079c1c 100644 --- a/src/Storages/StorageS3Cluster.cpp +++ b/src/Storages/StorageS3Cluster.cpp @@ -152,9 +152,8 @@ QueryProcessingStage::Enum StorageS3Cluster::getQueryProcessingStage( NamesAndTypesList StorageS3Cluster::getVirtuals() const { return NamesAndTypesList{ - {"_path", std::make_shared()}, - {"_file", std::make_shared()} - }; + {"_path", std::make_shared(std::make_shared())}, + {"_file", std::make_shared(std::make_shared())}}; }