thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-03 13:02:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updated names in BufferAllocationPolicy

Browse Source
This commit is contained in:
Smita Kulkarni 2024-03-18 15:59:05 +01:00
parent 8a11afeba1
commit 0530055842
6 changed files with 39 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/Common/BufferAllocationPolicy.cpp
View File

@ -11,7 +11,7 @@ class FixedSizeBufferAllocationPolicy : public IBufferAllocationPolicy
size_t buffer_number = 0;
public:
explicit FixedSizeBufferAllocationPolicy(const BufferAllocationSettings & settings_)
explicit FixedSizeBufferAllocationPolicy(const IBufferAllocationPolicy::Settings & settings_)
: buffer_size(settings_.strict_upload_part_size)
{
chassert(buffer_size > 0);
@ -45,7 +45,7 @@ class ExpBufferAllocationPolicy : public DB::IBufferAllocationPolicy
size_t buffer_number = 0;
public:
explicit ExpBufferAllocationPolicy(const BufferAllocationSettings & settings_)
explicit ExpBufferAllocationPolicy(const IBufferAllocationPolicy::Settings & settings_)
: first_size(std::max(settings_.max_single_part_upload_size, settings_.min_upload_part_size))
, second_size(settings_.min_upload_part_size)
, multiply_factor(settings_.upload_part_size_multiply_factor)
@ -91,7 +91,7 @@ public:
IBufferAllocationPolicy::~IBufferAllocationPolicy() = default;
IBufferAllocationPolicyPtr ChooseBufferPolicy(BufferAllocationSettings settings_)
IBufferAllocationPolicy::IBufferAllocationPolicyPtr IBufferAllocationPolicy::create(IBufferAllocationPolicy::Settings settings_)
{
if (settings_.strict_upload_part_size > 0)
return std::make_unique<FixedSizeBufferAllocationPolicy>(settings_);

41
src/Common/BufferAllocationPolicy.h
View File

@ -9,27 +9,30 @@
namespace DB
{
struct BufferAllocationSettings
{
size_t strict_upload_part_size = 0;
size_t min_upload_part_size = 16 * 1024 * 1024;
size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024;
size_t upload_part_size_multiply_factor = 2;
size_t upload_part_size_multiply_parts_count_threshold = 500;
size_t max_single_part_upload_size = 32 * 1024 * 1024;
};
class IBufferAllocationPolicy
{
public:
virtual size_t getBufferNumber() const = 0;
virtual size_t getBufferSize() const = 0;
virtual void nextBuffer() = 0;
virtual ~IBufferAllocationPolicy() = 0;
public:
struct Settings
{
size_t strict_upload_part_size = 0;
size_t min_upload_part_size = 16 * 1024 * 1024;
size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024;
size_t upload_part_size_multiply_factor = 2;
size_t upload_part_size_multiply_parts_count_threshold = 500;
size_t max_single_part_upload_size = 32 * 1024 * 1024;
};
virtual size_t getBufferNumber() const = 0;
virtual size_t getBufferSize() const = 0;
virtual void nextBuffer() = 0;
virtual ~IBufferAllocationPolicy() = 0;
using IBufferAllocationPolicyPtr = std::unique_ptr<IBufferAllocationPolicy>;
static IBufferAllocationPolicyPtr create(Settings settings_);
};
using IBufferAllocationPolicyPtr = std::unique_ptr<IBufferAllocationPolicy>;
IBufferAllocationPolicyPtr ChooseBufferPolicy(BufferAllocationSettings settings_);
}

12
src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp
View File

@ -33,12 +33,12 @@ WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage(
ThreadPoolCallbackRunner<void> schedule_)
: WriteBufferFromFileBase(buf_size_, nullptr, 0)
, log(getLogger("WriteBufferFromAzureBlobStorage"))
, buffer_allocation_policy(ChooseBufferPolicy({settings_->strict_upload_part_size,
settings_->min_upload_part_size,
settings_->max_upload_part_size,
settings_->upload_part_size_multiply_factor,
settings_->upload_part_size_multiply_parts_count_threshold,
settings_->max_single_part_upload_size}))
, buffer_allocation_policy(IBufferAllocationPolicy::create({settings_->strict_upload_part_size,
settings_->min_upload_part_size,
settings_->max_upload_part_size,
settings_->upload_part_size_multiply_factor,
settings_->upload_part_size_multiply_parts_count_threshold,
settings_->max_single_part_upload_size}))
, max_single_part_upload_size(settings_->max_single_part_upload_size)
, max_unexpected_write_error_retries(settings_->max_unexpected_write_error_retries)
, blob_path(blob_path_)

2
src/Disks/IO/WriteBufferFromAzureBlobStorage.h
View File

@ -62,7 +62,7 @@ private:
LoggerPtr log;
LogSeriesLimiterPtr limitedLog = std::make_shared<LogSeriesLimiter>(log, 1, 5);
IBufferAllocationPolicyPtr buffer_allocation_policy;
IBufferAllocationPolicy::IBufferAllocationPolicyPtr buffer_allocation_policy;
const size_t max_single_part_upload_size;
const size_t max_unexpected_write_error_retries;

12
src/IO/WriteBufferFromS3.cpp
View File

@ -93,12 +93,12 @@ WriteBufferFromS3::WriteBufferFromS3(
, write_settings(write_settings_)
, client_ptr(std::move(client_ptr_))
, object_metadata(std::move(object_metadata_))
, buffer_allocation_policy(ChooseBufferPolicy({upload_settings.strict_upload_part_size,
upload_settings.min_upload_part_size,
upload_settings.max_upload_part_size,
upload_settings.upload_part_size_multiply_factor,
upload_settings.upload_part_size_multiply_parts_count_threshold,
upload_settings.max_single_part_upload_size}))
, buffer_allocation_policy(IBufferAllocationPolicy::create({upload_settings.strict_upload_part_size,
upload_settings.min_upload_part_size,
upload_settings.max_upload_part_size,
upload_settings.upload_part_size_multiply_factor,
upload_settings.upload_part_size_multiply_parts_count_threshold,
upload_settings.max_single_part_upload_size}))
, task_tracker(
std::make_unique<TaskTracker>(
std::move(schedule_),

2
src/IO/WriteBufferFromS3.h
View File

@ -86,7 +86,7 @@ private:
LoggerPtr log = getLogger("WriteBufferFromS3");
LogSeriesLimiterPtr limitedLog = std::make_shared<LogSeriesLimiter>(log, 1, 5);
IBufferAllocationPolicyPtr buffer_allocation_policy;
IBufferAllocationPolicy::IBufferAllocationPolicyPtr buffer_allocation_policy;
/// Upload in S3 is made in parts.
/// We initiate upload, then upload each part and get ETag as a response, and then finalizeImpl() upload with listing all our parts.