From 13226c9bbfda03b768d07ee03252cb243f3f45b4 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 24 Nov 2021 16:35:11 +0800 Subject: [PATCH] fix code typo --- src/Interpreters/Context.cpp | 6 +++--- src/Storages/Hive/HiveCommon.cpp | 9 +++++---- src/Storages/Hive/HiveCommon.h | 4 ++-- src/Storages/Hive/HiveFile.cpp | 16 ++++++++-------- src/Storages/Hive/HiveFile.h | 23 ++++++++++------------- src/Storages/Hive/HiveSettings.h | 6 +++--- src/Storages/Hive/StorageHive.cpp | 2 +- 7 files changed, 32 insertions(+), 34 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 34f3b82672a..42ac5e2d44e 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1842,9 +1842,9 @@ HMSClientPtr Context::getHMSClient(const String & name) const if (it == shared->hive_metastore_clients.end() || it->second->isExpired()) { // connect to hive metastore - Poco::URI hms_url(name); - const auto& host = hms_url.getHost(); - auto port = hms_url.getPort(); + Poco::URI hive_metastore_url(name); + const auto & host = hive_metastore_url.getHost(); + auto port = hive_metastore_url.getPort(); std::shared_ptr socket = std::make_shared(host, port); socket->setKeepAlive(true); diff --git a/src/Storages/Hive/HiveCommon.cpp b/src/Storages/Hive/HiveCommon.cpp index 5e76df5328e..ad59c3e2627 100644 --- a/src/Storages/Hive/HiveCommon.cpp +++ b/src/Storages/Hive/HiveCommon.cpp @@ -133,12 +133,13 @@ std::vector HMSClient::HiveTableMeta::getLocationFiles(cons auto fs_builder = createHDFSBuilder(getNameNodeUrl(table->sd.location), getContext()->getGlobalContext()->getConfigRef()); auto fs = createHDFSFS(fs_builder.get()); Poco::URI uri(location); - HDFSFileInfo ls; - ls.file_info = hdfsListDirectory(fs.get(), uri.getPath().c_str(), &ls.length); + HDFSFileInfo dir_info; + dir_info.file_info = hdfsListDirectory(fs.get(), uri.getPath().c_str(), &dir_info.length); auto result = std::make_shared>(); - for (int i = 0; i < ls.length; ++i) + for (int i = 0; i < dir_info.length; ++i) { - auto & finfo = ls.file_info[i]; + auto & finfo = dir_info.file_info[i]; + /// skip directories and empty files, mKind value 'D' represents directory, otherwise file if (finfo.mKind != 'D' && finfo.mSize > 0) result->emplace_back(String(finfo.mName), finfo.mLastMod, finfo.mSize); } diff --git a/src/Storages/Hive/HiveCommon.h b/src/Storages/Hive/HiveCommon.h index db8877471b5..61f59fd5715 100644 --- a/src/Storages/Hive/HiveCommon.h +++ b/src/Storages/Hive/HiveCommon.h @@ -20,9 +20,9 @@ public: struct FileInfo { FileInfo() = default; - FileInfo(const std::string & path_, UInt64 ts_, size_t size_) : path(path_), last_mod_ts(ts_), size(size_) { } + FileInfo(const std::string & path_, UInt64 last_modify_time_, size_t size_) : path(path_), last_modify_time(last_modify_time_), size(size_) { } std::string path; - UInt64 last_mod_ts; // in ms + UInt64 last_modify_time; // in ms size_t size; }; diff --git a/src/Storages/Hive/HiveFile.cpp b/src/Storages/Hive/HiveFile.cpp index 665a086677c..a8c9284c717 100644 --- a/src/Storages/Hive/HiveFile.cpp +++ b/src/Storages/Hive/HiveFile.cpp @@ -24,20 +24,20 @@ namespace DB { -template -Range createRangeFromOrcStatistics(const S * stats) +template +Range createRangeFromOrcStatistics(const StatisticsType * stats) { if (stats->hasMinimum() && stats->hasMaximum()) { - return Range(T(stats->getMinimum()), true, T(stats->getMaximum()), true); + return Range(FieldType(stats->getMinimum()), true, FieldType(stats->getMaximum()), true); } else if (stats->hasMinimum()) { - return Range::createLeftBounded(T(stats->getMinimum()), true); + return Range::createLeftBounded(FieldType(stats->getMinimum()), true); } else if (stats->hasMaximum()) { - return Range::createRightBounded(T(stats->getMaximum()), true); + return Range::createRightBounded(FieldType(stats->getMaximum()), true); } else { @@ -45,12 +45,12 @@ Range createRangeFromOrcStatistics(const S * stats) } } -template -Range createRangeFromParquetStatistics(std::shared_ptr stats) +template +Range createRangeFromParquetStatistics(std::shared_ptr stats) { if (!stats->HasMinMax()) return Range(); - return Range(T(stats->min()), true, T(stats->max()), true); + return Range(FieldType(stats->min()), true, FieldType(stats->max()), true); } Range createRangeFromParquetStatistics(std::shared_ptr stats) diff --git a/src/Storages/Hive/HiveFile.h b/src/Storages/Hive/HiveFile.h index 152c93c7a52..0be2d7a9573 100644 --- a/src/Storages/Hive/HiveFile.h +++ b/src/Storages/Hive/HiveFile.h @@ -54,7 +54,7 @@ public: const FieldVector & values_, const String & namenode_url_, const String & path_, - UInt64 ts_, + UInt64 last_modify_time_, size_t size_, const NamesAndTypesList & index_names_and_types_, const std::shared_ptr & storage_settings_, @@ -63,14 +63,11 @@ public: , partition_values(values_) , namenode_url(namenode_url_) , path(path_) - , last_mod_ts(ts_) + , last_modify_time(last_modify_time_) , size(size_) , index_names_and_types(index_names_and_types_) , storage_settings(storage_settings_) { - // std::cout << "1delim:" << storage_settings->hive_text_field_delimeter << std::endl; - // std::cout << "1disable orc:" << storage_settings->disable_orc_stripe_minmax_index << std::endl; - // std::cout << "1disable parquet:" << storage_settings->disable_parquet_rowgroup_minmax_index << std::endl; } virtual ~IHiveFile() = default; @@ -123,14 +120,14 @@ public: return boost::algorithm::join(strs, "|"); } - inline UInt64 getLastModTs() const { return last_mod_ts; } + inline UInt64 getLastModTs() const { return last_modify_time; } inline size_t getSize() const { return size; } protected: FieldVector partition_values; String namenode_url; String path; - UInt64 last_mod_ts; + UInt64 last_modify_time; size_t size; NamesAndTypesList index_names_and_types; std::shared_ptr minmax_idx; @@ -146,12 +143,12 @@ public: const FieldVector & values_, const String & namenode_url_, const String & path_, - UInt64 ts_, + UInt64 last_modify_time_, size_t size_, const NamesAndTypesList & index_names_and_types_, const std::shared_ptr & hive_settings_, ContextPtr context_) - : IHiveFile(values_, namenode_url_, path_, ts_, size_, index_names_and_types_, hive_settings_, context_) + : IHiveFile(values_, namenode_url_, path_, last_modify_time_, size_, index_names_and_types_, hive_settings_, context_) { } @@ -167,12 +164,12 @@ public: const FieldVector & values_, const String & namenode_url_, const String & path_, - UInt64 ts_, + UInt64 last_modify_time_, size_t size_, const NamesAndTypesList & index_names_and_types_, const std::shared_ptr & hive_settings_, ContextPtr context_) - : IHiveFile(values_, namenode_url_, path_, ts_, size_, index_names_and_types_, hive_settings_, context_) + : IHiveFile(values_, namenode_url_, path_, last_modify_time_, size_, index_names_and_types_, hive_settings_, context_) { } @@ -202,12 +199,12 @@ public: const FieldVector & values_, const String & namenode_url_, const String & path_, - UInt64 ts_, + UInt64 last_modify_time_, size_t size_, const NamesAndTypesList & index_names_and_types_, const std::shared_ptr & hive_settings_, ContextPtr context_) - : IHiveFile(values_, namenode_url_, path_, ts_, size_, index_names_and_types_, hive_settings_, context_) + : IHiveFile(values_, namenode_url_, path_, last_modify_time_, size_, index_names_and_types_, hive_settings_, context_) { } diff --git a/src/Storages/Hive/HiveSettings.h b/src/Storages/Hive/HiveSettings.h index 36ec566448d..406595b4741 100644 --- a/src/Storages/Hive/HiveSettings.h +++ b/src/Storages/Hive/HiveSettings.h @@ -14,9 +14,9 @@ class ASTStorage; #define HIVE_RELATED_SETTINGS(M) \ M(Char, hive_text_field_delimeter, '\x01', "How to split one row of hive data with format text", 0) \ - M(Bool, disable_orc_stripe_minmax_index, true, "Disable using ORC stripe level minmax index.", 0) \ - M(Bool, disable_parquet_rowgroup_minmax_index, true, "Disable using Parquet row-group level minmax index.", 0) \ - M(Bool, disable_orc_file_minmax_index, false, "Disable using ORC file level minmax index.", 0) + M(Bool, enable_orc_stripe_minmax_index, false, "Enable using ORC stripe level minmax index.", 0) \ + M(Bool, enable_parquet_rowgroup_minmax_index, false, "Enable using Parquet row-group level minmax index.", 0) \ + M(Bool, enable_orc_file_minmax_index, true, "Enable using ORC file level minmax index.", 0) #define LIST_OF_HIVE_SETTINGS(M) \ HIVE_RELATED_SETTINGS(M) \ diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index 4c6e331adb6..435f9762b6d 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -410,7 +410,7 @@ Pipe StorageHive::read( fields, hdfs_namenode_url, hfile.path, - hfile.last_mod_ts, + hfile.last_modify_time, hfile.size, hivefile_name_types, storage_settings,