mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-29 21:20:49 +00:00
Fix readbuffer
This commit is contained in:
parent
f33387837e
commit
7f356a5fbc
@ -22,8 +22,6 @@ struct DiskWebServerSettings
|
|||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
/// Storage to store data on a web server and metadata on the local disk.
|
|
||||||
|
|
||||||
class DiskWebServer : public IDisk, WithContext
|
class DiskWebServer : public IDisk, WithContext
|
||||||
{
|
{
|
||||||
using SettingsPtr = std::unique_ptr<DiskWebServerSettings>;
|
using SettingsPtr = std::unique_ptr<DiskWebServerSettings>;
|
||||||
@ -64,7 +62,6 @@ public:
|
|||||||
|
|
||||||
const String & getName() const final override { return name; }
|
const String & getName() const final override { return name; }
|
||||||
|
|
||||||
/// ???
|
|
||||||
const String & getPath() const final override { return metadata_path; }
|
const String & getPath() const final override { return metadata_path; }
|
||||||
|
|
||||||
UInt64 getTotalSpace() const final override { return std::numeric_limits<UInt64>::max(); }
|
UInt64 getTotalSpace() const final override { return std::numeric_limits<UInt64>::max(); }
|
||||||
|
@ -155,7 +155,7 @@ struct RemoteMetadata
|
|||||||
const String & remote_fs_root_path;
|
const String & remote_fs_root_path;
|
||||||
|
|
||||||
/// Relative path to metadata file on local FS.
|
/// Relative path to metadata file on local FS.
|
||||||
const String & metadata_file_path;
|
const String metadata_file_path;
|
||||||
|
|
||||||
RemoteMetadata(const String & remote_fs_root_path_, const String & metadata_file_path_)
|
RemoteMetadata(const String & remote_fs_root_path_, const String & metadata_file_path_)
|
||||||
: remote_fs_root_path(remote_fs_root_path_), metadata_file_path(metadata_file_path_) {}
|
: remote_fs_root_path(remote_fs_root_path_), metadata_file_path(metadata_file_path_) {}
|
||||||
|
@ -35,6 +35,10 @@ ReadIndirectBufferFromWebServer::ReadIndirectBufferFromWebServer(const String &
|
|||||||
std::unique_ptr<ReadBuffer> ReadIndirectBufferFromWebServer::initialize()
|
std::unique_ptr<ReadBuffer> ReadIndirectBufferFromWebServer::initialize()
|
||||||
{
|
{
|
||||||
Poco::URI uri(url);
|
Poco::URI uri(url);
|
||||||
|
|
||||||
|
ReadWriteBufferFromHTTP::HTTPHeaderEntries headers;
|
||||||
|
headers.emplace_back(std::make_pair("Range:", fmt::format("bytes:{}-", offset)));
|
||||||
|
|
||||||
return std::make_unique<ReadWriteBufferFromHTTP>(
|
return std::make_unique<ReadWriteBufferFromHTTP>(
|
||||||
uri,
|
uri,
|
||||||
Poco::Net::HTTPRequest::HTTP_GET,
|
Poco::Net::HTTPRequest::HTTP_GET,
|
||||||
@ -51,8 +55,6 @@ bool ReadIndirectBufferFromWebServer::nextImpl()
|
|||||||
if (!impl)
|
if (!impl)
|
||||||
impl = initialize();
|
impl = initialize();
|
||||||
|
|
||||||
pos = impl->position();
|
|
||||||
|
|
||||||
bool ret = false, successful_read = false;
|
bool ret = false, successful_read = false;
|
||||||
auto sleep_milliseconds = std::chrono::milliseconds(100);
|
auto sleep_milliseconds = std::chrono::milliseconds(100);
|
||||||
|
|
||||||
@ -67,6 +69,9 @@ bool ReadIndirectBufferFromWebServer::nextImpl()
|
|||||||
catch (const Exception & e)
|
catch (const Exception & e)
|
||||||
{
|
{
|
||||||
LOG_WARNING(log, "Read attempt {}/{} failed from {}. ({})", try_num, max_read_tries, url, e.message());
|
LOG_WARNING(log, "Read attempt {}/{} failed from {}. ({})", try_num, max_read_tries, url, e.message());
|
||||||
|
|
||||||
|
impl.reset();
|
||||||
|
impl = initialize();
|
||||||
}
|
}
|
||||||
|
|
||||||
std::this_thread::sleep_for(sleep_milliseconds);
|
std::this_thread::sleep_for(sleep_milliseconds);
|
||||||
@ -79,7 +84,8 @@ bool ReadIndirectBufferFromWebServer::nextImpl()
|
|||||||
if (ret)
|
if (ret)
|
||||||
{
|
{
|
||||||
working_buffer = internal_buffer = impl->buffer();
|
working_buffer = internal_buffer = impl->buffer();
|
||||||
/// Do not update pos here, because it is anyway overwritten after nextImpl() in ReadBuffer::next().
|
pos = working_buffer.begin();
|
||||||
|
offset += working_buffer.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
return ret;
|
return ret;
|
||||||
@ -105,7 +111,7 @@ off_t ReadIndirectBufferFromWebServer::seek(off_t offset_, int whence)
|
|||||||
|
|
||||||
off_t ReadIndirectBufferFromWebServer::getPosition()
|
off_t ReadIndirectBufferFromWebServer::getPosition()
|
||||||
{
|
{
|
||||||
return offset + count();
|
return offset - available();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user