thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-30 19:42:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

add prefetch for hive text

Browse Source
This commit is contained in:
taiyang-li 2022-05-26 11:04:35 +08:00
parent a7a816dcb6
commit 561c87222d
3 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/Storages/HDFS/AsynchronousReadBufferFromHDFS.cpp
View File

@ -42,9 +42,12 @@ AsynchronousReadBufferFromHDFS::AsynchronousReadBufferFromHDFS(
, impl(std::move(impl_)) , impl(std::move(impl_))
, prefetch_buffer(settings_.remote_fs_buffer_size) , prefetch_buffer(settings_.remote_fs_buffer_size)
, read_until_position(impl->getFileSize()) , read_until_position(impl->getFileSize())
, use_prefetch(settings_.remote_fs_prefetch)
, log(&Poco::Logger::get("AsynchronousReadBufferFromHDFS")) , log(&Poco::Logger::get("AsynchronousReadBufferFromHDFS"))
{ {
ProfileEvents::increment(ProfileEvents::RemoteFSBuffers); ProfileEvents::increment(ProfileEvents::RemoteFSBuffers);
if (use_prefetch)
prefetch();
} }
bool AsynchronousReadBufferFromHDFS::hasPendingDataToRead() bool AsynchronousReadBufferFromHDFS::hasPendingDataToRead()
@ -142,6 +145,10 @@ bool AsynchronousReadBufferFromHDFS::nextImpl()
file_offset_of_buffer_end = impl->getFileOffsetOfBufferEnd(); file_offset_of_buffer_end = impl->getFileOffsetOfBufferEnd();
prefetch_future = {}; prefetch_future = {};
if (use_prefetch)
prefetch();
return size; return size;
} }

1
src/Storages/HDFS/AsynchronousReadBufferFromHDFS.h
View File

@ -57,6 +57,7 @@ private:
size_t file_offset_of_buffer_end = 0; size_t file_offset_of_buffer_end = 0;
std::optional<size_t> read_until_position; std::optional<size_t> read_until_position;
bool use_prefetch;
Poco::Logger * log; Poco::Logger * log;
}; };

3
src/Storages/Hive/StorageHive.cpp
View File

@ -133,6 +133,9 @@ public:
if (to_read_block.has(name_type.name)) if (to_read_block.has(name_type.name))
to_read_block.erase(name_type.name); to_read_block.erase(name_type.name);
} }
/// Apply read buffer prefetch for HiveText format, because it is read sequentially
read_settings.remote_fs_prefetch = format == "HiveText";
} }
FormatSettings updateFormatSettings(const HiveFilePtr & hive_file) FormatSettings updateFormatSettings(const HiveFilePtr & hive_file)