thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-27 01:51:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing tests.

Browse Source
This commit is contained in:
Nikolai Kochetov 2022-12-17 16:02:34 +00:00
parent 98ebef7914
commit b2355a2212
4 changed files with 53 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/Backups/BackupIO_S3.cpp
View File

@ -129,6 +129,7 @@ BackupWriterS3::BackupWriterS3(
, request_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString()).request_settings)
, log(&Poco::Logger::get("BackupWriterS3"))
{
request_settings.updateFromSettings(context_->getSettingsRef());
request_settings.max_single_read_retries = context_->getSettingsRef().s3_max_single_read_retries; // FIXME: Avoid taking value for endpoint
}

1
src/Storages/StorageS3.cpp
View File

@ -1197,6 +1197,7 @@ void StorageS3::updateS3Configuration(ContextPtr ctx, StorageS3::S3Configuration
{
auto settings = ctx->getStorageS3Settings().getSettings(upd.uri.uri.toString());
upd.request_settings = settings.request_settings;
upd.request_settings.updateFromSettings(ctx->getSettings());
if (upd.client)
{

53
src/Storages/StorageS3Settings.cpp
View File

@ -21,11 +21,7 @@ namespace ErrorCodes
S3Settings::RequestSettings::PartUploadSettings::PartUploadSettings(const Settings & settings)
: PartUploadSettings()
{
min_upload_part_size = settings.s3_min_upload_part_size;
upload_part_size_multiply_factor = settings.s3_upload_part_size_multiply_factor;
upload_part_size_multiply_parts_count_threshold = settings.s3_upload_part_size_multiply_parts_count_threshold;
max_single_part_upload_size = settings.s3_max_single_part_upload_size;
updateFromSettingsImpl(settings, false);
validate();
}
@ -46,6 +42,21 @@ S3Settings::RequestSettings::PartUploadSettings::PartUploadSettings(
validate();
}
void S3Settings::RequestSettings::PartUploadSettings::updateFromSettingsImpl(const Settings & settings, bool if_changed)
{
if (!if_changed || settings.s3_min_upload_part_size.changed)
min_upload_part_size = settings.s3_min_upload_part_size;
if (!if_changed || settings.s3_upload_part_size_multiply_factor.changed)
upload_part_size_multiply_factor = settings.s3_upload_part_size_multiply_factor;
if (!if_changed || settings.s3_upload_part_size_multiply_parts_count_threshold.changed)
upload_part_size_multiply_parts_count_threshold = settings.s3_upload_part_size_multiply_parts_count_threshold;
if (!if_changed || settings.s3_max_single_part_upload_size.changed)
max_single_part_upload_size = settings.s3_max_single_part_upload_size;
}
void S3Settings::RequestSettings::PartUploadSettings::validate()
{
static constexpr size_t min_upload_part_size_limit = 5 * 1024 * 1024;
@ -118,18 +129,38 @@ void S3Settings::RequestSettings::PartUploadSettings::validate()
S3Settings::RequestSettings::RequestSettings(const Settings & settings)
: upload_settings(settings)
{
max_single_read_retries = settings.s3_max_single_read_retries;
max_connections = settings.s3_max_connections;
check_objects_after_upload = settings.s3_check_objects_after_upload;
max_unexpected_write_error_retries = settings.s3_max_unexpected_write_error_retries;
if (settings.s3_max_get_rps)
updateFromSettingsImpl(settings, false);
}
void S3Settings::RequestSettings::updateFromSettingsImpl(const Settings & settings, bool if_changed)
{
if (!if_changed || settings.s3_max_single_read_retries.changed)
max_single_read_retries = settings.s3_max_single_read_retries;
if (!if_changed || settings.s3_max_connections.changed)
max_connections = settings.s3_max_connections;
if (!if_changed || settings.s3_check_objects_after_upload.changed)
check_objects_after_upload = settings.s3_check_objects_after_upload;
if (!if_changed || settings.s3_max_unexpected_write_error_retries.changed)
max_unexpected_write_error_retries = settings.s3_max_unexpected_write_error_retries;
if ((!if_changed || settings.s3_max_get_rps.changed || settings.s3_max_get_burst.changed) && settings.s3_max_get_rps)
get_request_throttler = std::make_shared<Throttler>(
settings.s3_max_get_rps, settings.s3_max_get_burst ? settings.s3_max_get_burst : Throttler::default_burst_seconds * settings.s3_max_get_rps);
if (settings.s3_max_put_rps)
if ((!if_changed || settings.s3_max_put_rps.changed || settings.s3_max_put_burst.changed) && settings.s3_max_put_rps)
put_request_throttler = std::make_shared<Throttler>(
settings.s3_max_put_rps, settings.s3_max_put_burst ? settings.s3_max_put_burst : Throttler::default_burst_seconds * settings.s3_max_put_rps);
}
void S3Settings::RequestSettings::updateFromSettings(const Settings & settings)
{
updateFromSettingsImpl(settings, true);
upload_settings.updateFromSettings(settings);
}
S3Settings::RequestSettings::RequestSettings(const NamedCollection & collection)
: upload_settings(collection)
{

10
src/Storages/StorageS3Settings.h
View File

@ -48,13 +48,16 @@ struct S3Settings
&& max_single_operation_copy_size == other.max_single_operation_copy_size;
}
void updateFromSettings(const Settings & settings) { updateFromSettingsImpl(settings, true); }
void validate();
private:
PartUploadSettings() = default;
explicit PartUploadSettings(const Settings & settings);
explicit PartUploadSettings(const NamedCollection & collection);
PartUploadSettings(const Poco::Util::AbstractConfiguration & config, const String & key, const Settings & settings);
void validate();
void updateFromSettingsImpl(const Settings & settings, bool if_changed);
friend struct RequestSettings;
};
@ -87,6 +90,11 @@ struct S3Settings
explicit RequestSettings(const Settings & settings);
explicit RequestSettings(const NamedCollection & collection);
RequestSettings(const Poco::Util::AbstractConfiguration & config, const String & key, const Settings & settings);
void updateFromSettings(const Settings & settings);
private:
void updateFromSettingsImpl(const Settings & settings, bool if_changed);
};
S3::AuthSettings auth_settings;