thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-27 10:02:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed multipart mechanism and added a warning about 10k parts.

Browse Source
This commit is contained in:
Vladimir Chebotarev 2019-06-22 08:58:05 +03:00
parent 97b7635c8a
commit ab456262d9
2 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dbms/src/IO/WriteBufferFromS3.cpp
View File

@ -12,6 +12,7 @@
#define DEFAULT_S3_MAX_FOLLOW_PUT_REDIRECT 2 #define DEFAULT_S3_MAX_FOLLOW_PUT_REDIRECT 2
#define S3_SOFT_MAX_PARTS 10000
namespace DB namespace DB
{ {
@ -34,7 +35,6 @@ WriteBufferFromS3::WriteBufferFromS3(
, timeouts {timeouts_} , timeouts {timeouts_}
, auth_request {Poco::Net::HTTPRequest::HTTP_PUT, uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1} , auth_request {Poco::Net::HTTPRequest::HTTP_PUT, uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1}
, temporary_buffer {std::make_unique<WriteBufferFromString>(buffer_string)} , temporary_buffer {std::make_unique<WriteBufferFromString>(buffer_string)}
, part_number {1}
, last_part_size {0} , last_part_size {0}
{ {
if (!credentials.getUsername().empty()) if (!credentials.getUsername().empty())
@ -69,7 +69,6 @@ void WriteBufferFromS3::finalize()
if (!buffer_string.empty()) if (!buffer_string.empty())
{ {
writePart(buffer_string); writePart(buffer_string);
++part_number;
} }
complete(); complete();
@ -154,9 +153,14 @@ void WriteBufferFromS3::writePart(const String & data)
HTTPSessionPtr session; HTTPSessionPtr session;
std::istream * istr = nullptr; /// owned by session std::istream * istr = nullptr; /// owned by session
Poco::URI part_uri = uri; Poco::URI part_uri = uri;
part_uri.addQueryParameter("partNumber", std::to_string(part_number)); part_uri.addQueryParameter("partNumber", std::to_string(part_tags.size() + 1));
part_uri.addQueryParameter("uploadId", upload_id); part_uri.addQueryParameter("uploadId", upload_id);
if (part_tags.size() == S3_SOFT_MAX_PARTS)
{
LOG_WARNING(&Logger::get("WriteBufferFromS3"), "Maximum part number in S3 protocol has reached.");
}
for (int i = 0; i < DEFAULT_S3_MAX_FOLLOW_PUT_REDIRECT; ++i) for (int i = 0; i < DEFAULT_S3_MAX_FOLLOW_PUT_REDIRECT; ++i)
{ {
session = makeHTTPSession(part_uri, timeouts); session = makeHTTPSession(part_uri, timeouts);

1
dbms/src/IO/WriteBufferFromS3.h
View File

@ -35,7 +35,6 @@ private:
Poco::Net::HTTPRequest auth_request; Poco::Net::HTTPRequest auth_request;
String buffer_string; String buffer_string;
std::unique_ptr<WriteBufferFromString> temporary_buffer; std::unique_ptr<WriteBufferFromString> temporary_buffer;
size_t part_number;
size_t last_part_size; size_t last_part_size;
String upload_id; String upload_id;
std::vector<String> part_tags; std::vector<String> part_tags;