thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-29 21:20:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix readbuffer

Browse Source
This commit is contained in:
kssenii 2021-06-18 14:32:08 +00:00
parent f33387837e
commit 7f356a5fbc
3 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/Disks/DiskWebServer.h
View File

@ -22,8 +22,6 @@ struct DiskWebServerSettings
}; };
/// Storage to store data on a web server and metadata on the local disk.
class DiskWebServer : public IDisk, WithContext class DiskWebServer : public IDisk, WithContext
{ {
using SettingsPtr = std::unique_ptr<DiskWebServerSettings>; using SettingsPtr = std::unique_ptr<DiskWebServerSettings>;
@ -64,7 +62,6 @@ public:
const String & getName() const final override { return name; } const String & getName() const final override { return name; }
/// ???
const String & getPath() const final override { return metadata_path; } const String & getPath() const final override { return metadata_path; }
UInt64 getTotalSpace() const final override { return std::numeric_limits<UInt64>::max(); } UInt64 getTotalSpace() const final override { return std::numeric_limits<UInt64>::max(); }

2
src/Disks/IDiskRemote.h
View File

@ -155,7 +155,7 @@ struct RemoteMetadata
const String & remote_fs_root_path; const String & remote_fs_root_path;
/// Relative path to metadata file on local FS. /// Relative path to metadata file on local FS.
const String & metadata_file_path; const String metadata_file_path;
RemoteMetadata(const String & remote_fs_root_path_, const String & metadata_file_path_) RemoteMetadata(const String & remote_fs_root_path_, const String & metadata_file_path_)
: remote_fs_root_path(remote_fs_root_path_), metadata_file_path(metadata_file_path_) {} : remote_fs_root_path(remote_fs_root_path_), metadata_file_path(metadata_file_path_) {}

14
src/IO/ReadIndirectBufferFromWebServer.cpp
View File

@ -35,6 +35,10 @@ ReadIndirectBufferFromWebServer::ReadIndirectBufferFromWebServer(const String &
std::unique_ptr<ReadBuffer> ReadIndirectBufferFromWebServer::initialize() std::unique_ptr<ReadBuffer> ReadIndirectBufferFromWebServer::initialize()
{ {
Poco::URI uri(url); Poco::URI uri(url);
ReadWriteBufferFromHTTP::HTTPHeaderEntries headers;
headers.emplace_back(std::make_pair("Range:", fmt::format("bytes:{}-", offset)));
return std::make_unique<ReadWriteBufferFromHTTP>( return std::make_unique<ReadWriteBufferFromHTTP>(
uri, uri,
Poco::Net::HTTPRequest::HTTP_GET, Poco::Net::HTTPRequest::HTTP_GET,
@ -51,8 +55,6 @@ bool ReadIndirectBufferFromWebServer::nextImpl()
if (!impl) if (!impl)
impl = initialize(); impl = initialize();
pos = impl->position();
bool ret = false, successful_read = false; bool ret = false, successful_read = false;
auto sleep_milliseconds = std::chrono::milliseconds(100); auto sleep_milliseconds = std::chrono::milliseconds(100);
@ -67,6 +69,9 @@ bool ReadIndirectBufferFromWebServer::nextImpl()
catch (const Exception & e) catch (const Exception & e)
{ {
LOG_WARNING(log, "Read attempt {}/{} failed from {}. ({})", try_num, max_read_tries, url, e.message()); LOG_WARNING(log, "Read attempt {}/{} failed from {}. ({})", try_num, max_read_tries, url, e.message());
impl.reset();
impl = initialize();
} }
std::this_thread::sleep_for(sleep_milliseconds); std::this_thread::sleep_for(sleep_milliseconds);
@ -79,7 +84,8 @@ bool ReadIndirectBufferFromWebServer::nextImpl()
if (ret) if (ret)
{ {
working_buffer = internal_buffer = impl->buffer(); working_buffer = internal_buffer = impl->buffer();
/// Do not update pos here, because it is anyway overwritten after nextImpl() in ReadBuffer::next(). pos = working_buffer.begin();
offset += working_buffer.size();
} }
return ret; return ret;
@ -105,7 +111,7 @@ off_t ReadIndirectBufferFromWebServer::seek(off_t offset_, int whence)
off_t ReadIndirectBufferFromWebServer::getPosition() off_t ReadIndirectBufferFromWebServer::getPosition()
{ {
return offset + count(); return offset - available();
} }
} }