diff --git a/src/Common/BufferAllocationPolicy.cpp b/src/Common/BufferAllocationPolicy.cpp index 1456233eb03..2b00d4f84d0 100644 --- a/src/Common/BufferAllocationPolicy.cpp +++ b/src/Common/BufferAllocationPolicy.cpp @@ -11,7 +11,7 @@ class FixedSizeBufferAllocationPolicy : public IBufferAllocationPolicy size_t buffer_number = 0; public: - explicit FixedSizeBufferAllocationPolicy(const BufferAllocationSettings & settings_) + explicit FixedSizeBufferAllocationPolicy(const IBufferAllocationPolicy::Settings & settings_) : buffer_size(settings_.strict_upload_part_size) { chassert(buffer_size > 0); @@ -45,7 +45,7 @@ class ExpBufferAllocationPolicy : public DB::IBufferAllocationPolicy size_t buffer_number = 0; public: - explicit ExpBufferAllocationPolicy(const BufferAllocationSettings & settings_) + explicit ExpBufferAllocationPolicy(const IBufferAllocationPolicy::Settings & settings_) : first_size(std::max(settings_.max_single_part_upload_size, settings_.min_upload_part_size)) , second_size(settings_.min_upload_part_size) , multiply_factor(settings_.upload_part_size_multiply_factor) @@ -91,7 +91,7 @@ public: IBufferAllocationPolicy::~IBufferAllocationPolicy() = default; -IBufferAllocationPolicyPtr ChooseBufferPolicy(BufferAllocationSettings settings_) +IBufferAllocationPolicy::IBufferAllocationPolicyPtr IBufferAllocationPolicy::create(IBufferAllocationPolicy::Settings settings_) { if (settings_.strict_upload_part_size > 0) return std::make_unique(settings_); diff --git a/src/Common/BufferAllocationPolicy.h b/src/Common/BufferAllocationPolicy.h index b759d22ede6..3ada9e52fdf 100644 --- a/src/Common/BufferAllocationPolicy.h +++ b/src/Common/BufferAllocationPolicy.h @@ -9,27 +9,30 @@ namespace DB { -struct BufferAllocationSettings -{ - size_t strict_upload_part_size = 0; - size_t min_upload_part_size = 16 * 1024 * 1024; - size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024; - size_t upload_part_size_multiply_factor = 2; - size_t upload_part_size_multiply_parts_count_threshold = 500; - size_t max_single_part_upload_size = 32 * 1024 * 1024; -}; - class IBufferAllocationPolicy { - public: - virtual size_t getBufferNumber() const = 0; - virtual size_t getBufferSize() const = 0; - virtual void nextBuffer() = 0; - virtual ~IBufferAllocationPolicy() = 0; +public: + + struct Settings + { + size_t strict_upload_part_size = 0; + size_t min_upload_part_size = 16 * 1024 * 1024; + size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024; + size_t upload_part_size_multiply_factor = 2; + size_t upload_part_size_multiply_parts_count_threshold = 500; + size_t max_single_part_upload_size = 32 * 1024 * 1024; + }; + + + virtual size_t getBufferNumber() const = 0; + virtual size_t getBufferSize() const = 0; + virtual void nextBuffer() = 0; + virtual ~IBufferAllocationPolicy() = 0; + + using IBufferAllocationPolicyPtr = std::unique_ptr; + + static IBufferAllocationPolicyPtr create(Settings settings_); + }; -using IBufferAllocationPolicyPtr = std::unique_ptr; - -IBufferAllocationPolicyPtr ChooseBufferPolicy(BufferAllocationSettings settings_); - } diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index bc11d445a51..41d7187afda 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -33,12 +33,12 @@ WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage( ThreadPoolCallbackRunner schedule_) : WriteBufferFromFileBase(buf_size_, nullptr, 0) , log(getLogger("WriteBufferFromAzureBlobStorage")) - , buffer_allocation_policy(ChooseBufferPolicy({settings_->strict_upload_part_size, - settings_->min_upload_part_size, - settings_->max_upload_part_size, - settings_->upload_part_size_multiply_factor, - settings_->upload_part_size_multiply_parts_count_threshold, - settings_->max_single_part_upload_size})) + , buffer_allocation_policy(IBufferAllocationPolicy::create({settings_->strict_upload_part_size, + settings_->min_upload_part_size, + settings_->max_upload_part_size, + settings_->upload_part_size_multiply_factor, + settings_->upload_part_size_multiply_parts_count_threshold, + settings_->max_single_part_upload_size})) , max_single_part_upload_size(settings_->max_single_part_upload_size) , max_unexpected_write_error_retries(settings_->max_unexpected_write_error_retries) , blob_path(blob_path_) diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h index 7223f66693e..f047dd53724 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h @@ -62,7 +62,7 @@ private: LoggerPtr log; LogSeriesLimiterPtr limitedLog = std::make_shared(log, 1, 5); - IBufferAllocationPolicyPtr buffer_allocation_policy; + IBufferAllocationPolicy::IBufferAllocationPolicyPtr buffer_allocation_policy; const size_t max_single_part_upload_size; const size_t max_unexpected_write_error_retries; diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 60fa828d6c4..c907c4cb6b6 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -93,12 +93,12 @@ WriteBufferFromS3::WriteBufferFromS3( , write_settings(write_settings_) , client_ptr(std::move(client_ptr_)) , object_metadata(std::move(object_metadata_)) - , buffer_allocation_policy(ChooseBufferPolicy({upload_settings.strict_upload_part_size, - upload_settings.min_upload_part_size, - upload_settings.max_upload_part_size, - upload_settings.upload_part_size_multiply_factor, - upload_settings.upload_part_size_multiply_parts_count_threshold, - upload_settings.max_single_part_upload_size})) + , buffer_allocation_policy(IBufferAllocationPolicy::create({upload_settings.strict_upload_part_size, + upload_settings.min_upload_part_size, + upload_settings.max_upload_part_size, + upload_settings.upload_part_size_multiply_factor, + upload_settings.upload_part_size_multiply_parts_count_threshold, + upload_settings.max_single_part_upload_size})) , task_tracker( std::make_unique( std::move(schedule_), diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 840274c8ace..201743e7f9c 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -86,7 +86,7 @@ private: LoggerPtr log = getLogger("WriteBufferFromS3"); LogSeriesLimiterPtr limitedLog = std::make_shared(log, 1, 5); - IBufferAllocationPolicyPtr buffer_allocation_policy; + IBufferAllocationPolicy::IBufferAllocationPolicyPtr buffer_allocation_policy; /// Upload in S3 is made in parts. /// We initiate upload, then upload each part and get ETag as a response, and then finalizeImpl() upload with listing all our parts.