thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 16:12:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Minimum block size to configuration.

Browse Source
This commit is contained in:
Vladimir Chebotarev 2019-06-21 08:24:01 +03:00
parent 395560df1b
commit 97b7635c8a
3 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
dbms/src/IO/WriteBufferFromS3.cpp
View File

@ -12,7 +12,6 @@
#define DEFAULT_S3_MAX_FOLLOW_PUT_REDIRECT 2
#define DEFAULT_S3_MINIMUM_PART_SIZE 100'000'000
namespace DB
{
@ -24,10 +23,14 @@ namespace ErrorCodes
WriteBufferFromS3::WriteBufferFromS3(
const Poco::URI & uri_, const ConnectionTimeouts & timeouts_,
const Poco::Net::HTTPBasicCredentials & credentials, size_t buffer_size_)
const Poco::URI & uri_,
size_t minimum_upload_part_size_,
const ConnectionTimeouts & timeouts_,
const Poco::Net::HTTPBasicCredentials & credentials, size_t buffer_size_
)
: BufferWithOwnMemory<WriteBuffer>(buffer_size_, nullptr, 0)
, uri {uri_}
, minimum_upload_part_size {minimum_upload_part_size_}
, timeouts {timeouts_}
, auth_request {Poco::Net::HTTPRequest::HTTP_PUT, uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1}
, temporary_buffer {std::make_unique<WriteBufferFromString>(buffer_string)}
@ -50,7 +53,7 @@ void WriteBufferFromS3::nextImpl()
last_part_size += offset();
if (last_part_size > DEFAULT_S3_MINIMUM_PART_SIZE)
if (last_part_size > minimum_upload_part_size)
{
temporary_buffer->finish();
writePart(buffer_string);

2
dbms/src/IO/WriteBufferFromS3.h
View File

@ -30,6 +30,7 @@ class WriteBufferFromS3 : public BufferWithOwnMemory<WriteBuffer>
{
private:
Poco::URI uri;
size_t minimum_upload_part_size;
ConnectionTimeouts timeouts;
Poco::Net::HTTPRequest auth_request;
String buffer_string;
@ -41,6 +42,7 @@ private:
public:
explicit WriteBufferFromS3(const Poco::URI & uri,
size_t minimum_upload_part_size_,
const ConnectionTimeouts & timeouts = {},
const Poco::Net::HTTPBasicCredentials & credentials = {},
size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE);

3
dbms/src/Storages/StorageS3.cpp
View File

@ -84,7 +84,8 @@ namespace
const ConnectionTimeouts & timeouts)
: sample_block(sample_block_)
{
write_buf = std::make_unique<WriteBufferFromS3>(uri, timeouts);
auto minimum_upload_part_size = context.getConfigRef().getUInt64("s3_minimum_upload_part_size", 512'000'000);
write_buf = std::make_unique<WriteBufferFromS3>(uri, minimum_upload_part_size, timeouts);
writer = FormatFactory::instance().getOutput(format, *write_buf, sample_block, context);
}