thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 09:32:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #51386 from bigo-sg/fix_hdfs_read_buffer_heap_overflow

Browse Source 
Fix heap overflow in read buffer from hdfs when read_until_position is not zero
This commit is contained in:
Kseniia Sumarokova 2023-07-22 15:00:58 +02:00 committed by GitHub
commit d34f2bed07
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 32 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/IO/examples/CMakeLists.txt
View File

@ -73,3 +73,9 @@ target_link_libraries (snappy_read_buffer PRIVATE clickhouse_common_io)
clickhouse_add_executable (hadoop_snappy_read_buffer hadoop_snappy_read_buffer.cpp)
target_link_libraries (hadoop_snappy_read_buffer PRIVATE clickhouse_common_io)
if (TARGET ch_contrib::hdfs)
clickhouse_add_executable (read_buffer_from_hdfs read_buffer_from_hdfs.cpp)
target_link_libraries (read_buffer_from_hdfs PRIVATE dbms ch_contrib::hdfs)
endif ()

25
src/IO/examples/read_buffer_from_hdfs.cpp Normal file
View File

@ -0,0 +1,25 @@
#include <iostream>
#include <memory>
#include <string>
#include <IO/WriteBufferFromFile.h>
#include <IO/copyData.h>
#include <Storages/HDFS/ReadBufferFromHDFS.h>
#include <base/types.h>
#include <Common/Config/ConfigProcessor.h>
using namespace DB;
int main()
{
setenv("LIBHDFS3_CONF", "/path/to/hdfs-site.xml", true); /// NOLINT
String hdfs_uri = "hdfs://cluster_name";
String hdfs_file_path = "/path/to/hdfs/file";
ConfigurationPtr config = Poco::AutoPtr(new Poco::Util::MapConfiguration());
ReadSettings read_settings;
ReadBufferFromHDFS read_buffer(hdfs_uri, hdfs_file_path, *config, read_settings, 2097152UL, false);
String download_path = "./download";
WriteBufferFromFile write_buffer(download_path);
copyData(read_buffer, write_buffer);
return 0;
}

2
src/Storages/HDFS/ReadBufferFromHDFS.cpp
View File

@ -89,7 +89,7 @@ struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl : public BufferWithOwnMemory<S
if (read_until_position < file_offset)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read beyond right offset ({} > {})", file_offset, read_until_position - 1);
num_bytes_to_read = read_until_position - file_offset;
num_bytes_to_read = std::min<size_t>(read_until_position - file_offset, internal_buffer.size());
}
else
{